CommonCrawlLinks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T21:49:04.030438Z",
     "start_time": "2019-04-19T21:49:03.717299Z"
    }
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import time\n",
    "import json\n",
    "import csv\n",
    "import codecs\n",
    "from bs4 import BeautifulSoup\n",
    "from io import BytesIO\n",
    "\n",
    "import gzip\n",
    "import random\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T19:40:18.049397Z",
     "start_time": "2019-04-18T19:40:18.045646Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T21:49:51.080385Z",
     "start_time": "2019-04-19T21:49:51.074660Z"
    }
   },
   "outputs": [],
   "source": [
    "def domainSearch(domain):\n",
    "    recordList = []\n",
    "    for ind in indexList:\n",
    "        #print (\"Index : %s\" % ind)\n",
    "        url  = \"http://index.commoncrawl.org/CC-MAIN-%s-index?\" % ind\n",
    "        url += \"url=%s&matchType=domain&output=json\" % domain\n",
    "        \n",
    "        jsonResponse = requests.get(url)\n",
    "        print (\"json\" + str(jsonResponse))\n",
    "        if jsonResponse.status_code == 200:\n",
    "            recs = jsonResponse.content.splitlines()\n",
    "            for rec in recs:\n",
    "                recordList.append(json.loads(rec))\n",
    "    print (\"Total hits : %s\" % len(recordList))\n",
    "    return recordList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T21:49:51.387784Z",
     "start_time": "2019-04-19T21:49:51.379515Z"
    }
   },
   "outputs": [],
   "source": [
    "def page(rec):\n",
    "    as3 = \"https://commoncrawl.s3.amazonaws.com/\"\n",
    "    #print (rec['filename'])\n",
    "    offset, length = int(rec['offset']), int(rec['length'])\n",
    "    offset_end = offset + length - 1\n",
    "    resp = requests.get(as3 + rec['filename'], headers={'Range': 'bytes={}-{}'.format(offset, offset_end)})\n",
    "    #print (\"Resp : %s\" %resp)\n",
    "    #print(resp.content)\n",
    "    data = BytesIO(resp.content)\n",
    "    #print (\"Data : %s\" %data.getvalue)\n",
    "    f = gzip.GzipFile(fileobj=data)\n",
    "    #print (f)\n",
    "    try:\n",
    "        actData = f.read().decode(\"utf-8\")\n",
    "    except:\n",
    "        return None\n",
    "    #print (actData)\n",
    "    response = \"\"\n",
    "    if len(actData):\n",
    "        try:\n",
    "            warc, header, response = actData.strip().split('\\r\\n\\r\\n',2)\n",
    "        except:\n",
    "            pass\n",
    "    return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T21:49:51.693747Z",
     "start_time": "2019-04-19T21:49:51.687347Z"
    }
   },
   "outputs": [],
   "source": [
    "def getLinks(content, linkList):\n",
    "    parser = BeautifulSoup(content)\n",
    "    links = parser.find_all(\"a\")\n",
    "    if links:\n",
    "        for link in links:\n",
    "            #print(link)\n",
    "            href = link.attrs.get(\"href\")\n",
    "            if href is not None:\n",
    "                if domain not in href:\n",
    "                    if href not in linkList and href.startswith(\"http\"):\n",
    "                        #print (\"Adding link : %s\" %href)\n",
    "                        linkList.append(href)\n",
    "    return linkList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-20T06:16:03.103103Z",
     "start_time": "2019-04-20T06:16:00.994629Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "json<Response [200]>\n",
      "json<Response [404]>\n",
      "Total hits : 13344\n"
     ]
    }
   ],
   "source": [
    "choice = 4\n",
    "loc = \"./Dic/Lab 2/CommonCrawlData/\"\n",
    "domainNames = [\"gamespot.com/news/\", \"https://www.billboard.com/news\" , \"https://www.newyorker.com/books\", \"https://www.cinemablend.com/television/\", \"https://www.cinemablend.com/news\"]\n",
    "fileName = [\"CC-Games_Gamespot_New.csv\", \"CC-Music-Billboard-New.csv\", \"CC-Books-NewYorker-New.csv\", \"CC-Television-Cinemablend.csv\", \"CC-Movies-Cinemablend.csv\"]\n",
    "lookFor = [\"article\",\"\",\"books\", \"television\",\"news\"]\n",
    "indexList = [\"2019-13\", \"2019-07\"]\n",
    "domain = domainNames[choice]\n",
    "recordList = domainSearch(domain)\n",
    "linksList = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-20T06:16:03.398304Z",
     "start_time": "2019-04-20T06:16:03.377191Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'news'"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.seed(25)\n",
    "recordListInd = random.sample(range(0,len(recordList)), 10000)\n",
    "len(recordListInd)\n",
    "lookFor[choice]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-18T22:25:50.381327Z",
     "start_time": "2019-04-18T22:25:50.373130Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-20T07:10:58.134841Z",
     "start_time": "2019-04-20T06:16:05.949260Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding link 0: https://www.cinemablend.com/television/2436699/the-opposition-with-jordan-klepper-cancelled-at-comedy-central-but-theres-some-good-news\n",
      "Adding link 1: http://www.cinemablend.com/dvdnews/Indiana-Jones-And-The-Double-Dip-DVD-9261.html\n",
      "Adding link 2: http://www.cinemablend.com/news/1614500/cinemablends-star-ratings-explained\n",
      "Adding link 3: http://www.cinemablend.com/news.php?tag=to%203d%20or%20not%20to%203d\n",
      "Adding link 4: http://www.cinemablend.com/news.php?tag=Comic%20Book%20Guides\n",
      "Adding link 5: http://www.cinemablend.com/news/1532990/ubisoft-backtracks-statement-about-the-assassins-creed-movie-not-making-money\n",
      "Adding link 6: http://www.cinemablend.com/news/tag/razer/\n",
      "Adding link 7: http://www.cinemablend.com/news/1560420/one-huge-way-alicia-vikanders-tomb-raider-will-differ-from-angelina-jolies-version\n",
      "Adding link 8: http://www.cinemablend.com/news/1600830/how-many-assassins-creed-movies-michael-fassbender-thinks-they-will-make\n",
      "Adding link 9: http://www.cinemablend.com/news/tag/arkane%20studios/\n",
      "Adding link 10: http://www.cinemablend.com/news.php?tag=300\n",
      "Adding link 11: http://www.cinemablend.com/news.php?tag=cinemacon%202014\n",
      "Adding link 12: http://www.cinemablend.com/news/1545020/could-tron-3-still-happen-heres-what-the-producer-says\n",
      "Adding link 13: https://www.cinemablend.com/gallery/dvdnews/Disney-Opens-Vault-Brings-Dumbo-Blu-Ray-31673.html?tid=16911\n",
      "Adding link 14: http://www.cinemablend.com/news.php?tag=CinemaCon2013\n",
      "Adding link 15: http://www.cinemablend.com/news/1601580/keep-deadshot-out-of-gotham-city-sirens\n",
      "Adding link 16: http://www.cinemablend.com/dvdnews/dvds/?page=2\n",
      "Adding link 17: http://cinemablend.com/dvdnews/Dillinger-s-Crime-Spree-Comes-Home-With-Public-Enemies-On-Blu-Ray-And-DVD-20653.html\n",
      "Adding link 18: http://www.cinemablend.com/news/tag/mercurysteam/\n",
      "Adding link 19: http://www.cinemablend.com/news/Tomb-Raider-Looking-Rising-Star-Lara-Croft-She-Seems-Perfect-117607.html\n",
      "Adding link 20: http://www.cinemablend.com/news/Tomb-Raider-Reboot-Happening-Here-What-We-Know-94807.html\n",
      "Adding link 21: http://www.cinemablend.com/news/Popular-Video-Game-Uncharted-Drake-Fortune-Getting-Adapted-Get-Details-68186.html\n",
      "Adding link 22: http://cinemablend.com/dvdnews/Netflix-Pays-Big-Money-To-Add-Paramount-And-Lionsgate-Titles-To-Instant-Watch-26277.html\n",
      "Adding link 23: http://www.cinemablend.com/news/1591370/how-much-it-would-cost-to-power-the-death-star-per-day\n",
      "Adding link 24: http://www.cinemablend.com/news/1596800/epic-rogue-one-trailer-explores-star-wars-history-with-the-death-star\n",
      "Adding link 25: http://www.cinemablend.com/news/1541780/resident-evil-the-final-chapters-trailer-is-coming-very-soon-heres-what-we-know\n",
      "Adding link 26: http://www.cinemablend.com/news.php?tag=best%20of%202010\n",
      "Adding link 27: http://www.cinemablend.com/news/1576479/the-watch-dogs-movie-wants-to-change-how-you-think-about-hackers\n",
      "Adding link 28: http://www.cinemablend.com/news/1536559/a-live-action-pokemon-movie-is-officially-happening-get-the-details\n",
      "Adding link 29: http://www.cinemablend.com/news/1681179/watch-the-epic-star-wars-the-last-jedi-video-from-disneys-d-23\n",
      "Adding link 30: http://www.cinemablend.com/news/1680460/how-avengers-4-could-connect-to-netflixs-marvel-shows\n",
      "Adding link 31: http://www.cinemablend.com/news/1681160/toy-story-4-just-lost-a-director-get-the-details\n",
      "Adding link 32: http://www.cinemablend.com/television/1552979/narcos-just-got-big-renewal-news-from-netflix-get-the-details\n",
      "Adding link 33: http://www.cinemablend.com/news/1639139/30-best-sci-fi-movies-of-all-time\n",
      "Adding link 34: http://www.cinemablend.com/news/1538840/why-aliens-is-such-a-great-movie-according-to-james-cameron\n",
      "Adding link 35: http://www.cinemablend.com/news/1533949/mortal-kombat-3-may-happen-with-this-incredible-sounding-twist\n",
      "Adding link 36: http://www.cinemablend.com/news/1646090/one-major-fast-and-furious-mystery-the-series-promises-to-confront-according-to-a-producer\n",
      "Adding link 37: http://www.cinemablend.com/sections.php?cid=6&pageNum_news=0&tag=metal%20gear%20solid%204\n",
      "Adding link 38: https://amp.cinemablend.com/news/1563679/the-gears-of-war-movie-is-finally-moving-forward-heres-what-we-know\n",
      "Adding link 39: http://www.cinemablend.com/news/1642120/is-blumhouse-doing-a-five-nights-at-freddys-movie\n",
      "Adding link 40: http://www.cinemablend.com/news/1591940/the-live-action-pokemon-movie-has-found-its-director\n",
      "Adding link 41: http://cinemablend.com/dvdnews/DVD-Trailer-For-After-Life-Is-Creepy-Fun-For-Death-Fetishists-25726.html\n",
      "Adding link 42: http://cinemablend.com/dvdnews/Liam-Neeson-And-Christina-Ricci-Enter-The-After-Life-On-DVD-25669.html\n",
      "Adding link 43: http://www.cinemablend.com/news/1625619/the-fate-of-the-furious-who-helen-mirren-is-playing\n",
      "Adding link 44: http://www.cinemablend.com/news/1591330/a-back-to-the-future-reboot-heres-what-the-producer-thinks\n",
      "Adding link 45: http://www.cinemablend.com/news/1602550/vin-diesels-touching-message-about-paul-walker-and-fate-of-the-furious\n",
      "Adding link 46: http://www.cinemablend.com/news/1608640/how-carrie-fisher-felt-about-that-rogue-one-cameo\n",
      "Adding link 47: http://www.cinemablend.com/news/1529940/why-disneys-moana-is-already-in-trouble-with-activists\n",
      "Adding link 48: http://www.cinemablend.com/dvdnews/CB-DVD-Staff-Lists-Their-Favorite-Scary-Movies-13151.html\n",
      "Adding link 49: http://www.cinemablend.com/news/1528189/dungeons--dragons-may-have-found-its-first-lead\n",
      "Adding link 50: https://www.cinemablend.com/dvdnews/Madness-Sparta-4894.html\n",
      "Adding link 51: https://www.cinemablend.com/dvdnews/Important-News-Canadians-With-Netflix-Who-Like-Paramount-Pictures-31017.html\n",
      "Adding link 52: http://www.cinemablend.com/news/1563679/the-gears-of-war-movie-is-finally-moving-forward-heres-what-we-know\n",
      "Adding link 53: https://www.cinemablend.com/gallery/dvdnews/DVD-Season-Sets-Burn-Notice-White-Collar-June-30913.html?tid=16549\n",
      "Adding link 54: http://www.cinemablend.com/news.php?tag=best%20of%202011\n",
      "Adding link 55: http://www.cinemablend.com/news/1609660/extreme-cinematic-universe-what-you-need-to-know\n",
      "Adding link 56: http://www.cinemablend.com/news/tag/volition/\n",
      "Adding link 57: http://www.cinemablend.com/news/1551980/dwayne-johnsons-hard-r-rated-saints-row-movie-sounds-like-it-would-have-been-awesome\n",
      "Adding link 58: http://www.cinemablend.com/news.php?tag=The%20Thing%20Set%20Report\n",
      "Adding link 59: http://www.cinemablend.com/news/Steven-Spielberg-Making-Halo-14282.html\n",
      "Adding link 60: http://www.cinemablend.com/dvdnews/Jackass-3-5-Offending-March-28492.html\n",
      "Adding link 61: http://www.cinemablend.com/dvdnews/The-Battle-Between-Man-And-Machine-Rages-On-With-Terminator-Salvation-On-DVD-And-Blu-Ray-19811.html\n",
      "Adding link 62: http://www.cinemablend.com/news/1522800/pixars-latest-description-of-cars-3-has-us-more-than-a-little-excited\n",
      "Adding link 63: http://www.cinemablend.com/news/1566590/the-minecraft-movie-is-being-rewritten-again\n",
      "Adding link 64: http://www.cinemablend.com/news/1585730/the-minecraft-movie-may-have-found-its-first-star\n",
      "Adding link 65: https://www.cinemablend.com/gallery/dvdnews/DVD-Season-Sets-Burn-Notice-White-Collar-June-30913.html?tid=16555\n",
      "Adding link 66: http://www.cinemablend.com/news/1584560/jurassic-world-2-will-be-both-a-jurassic-world-sequel-and-jurassic-park-5-says-ja-bayona\n",
      "Adding link 67: http://www.cinemablend.com/news/1655489/the-10-funniest-moments-in-guardians-of-the-galaxy-2\n",
      "Adding link 68: http://www.cinemablend.com/news/1563049/how-nightcrawler-should-change-in-the-next-x-men-movie-according-to-kodi-smit-mcphee\n",
      "Adding link 69: http://www.cinemablend.com/news/1591680/nic-cage-using-virtual-reality-to-make-a-global-warming-sci-fi-thriller-yes-please\n",
      "Adding link 70: http://www.cinemablend.com/news/1636920/the-matrix-reboot-may-not-actually-be-a-reboot-at-all\n",
      "Adding link 71: http://www.cinemablend.com/news/1614160/total-recall-ending-was-it-a-dream-the-whole-time\n",
      "Adding link 72: http://www.cinemablend.com/news/1581320/starship-troopers-is-getting-rebooted-find-out-who-is-involved\n",
      "Adding link 73: http://www.cinemablend.com/dvdnews/?tag=flixworthy\n",
      "Adding link 74: http://www.cinemablend.com/The-Hobbit-The-Battle-of-the-Five-Armies-6003.html#news\n",
      "Adding link 75: http://www.cinemablend.com/news/1559789/how-fantastic-beasts-katherine-waterston-feels-about-being-compared-to-hermione\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding link 76: http://www.cinemablend.com/news/tag/saber%20interactive/\n",
      "Adding link 77: http://www.cinemablend.com/dvdnews/FlixWorthy-Party-Down-Because-It-Might-Get-Loud-24278.html\n",
      "Adding link 78: http://www.cinemablend.com/news/1647420/the-last-jedi-trailer-has-secret-whispers-heres-what-we-can-hear\n",
      "Adding link 79: http://www.cinemablend.com/news/1591010/lin-manuel-miranda-has-ambitious-plans-for-the-kingkiller-chronicles-trilogy\n",
      "Adding link 80: http://www.cinemablend.com/news/1522139/the-inside-joke-that-was-snuck-into-ferris-buellers-day-off\n",
      "Adding link 81: http://www.cinemablend.com/news/1554369/sony-is-making-its-own-mulan-movie-heres-what-we-know\n",
      "Adding link 82: https://www.cinemablend.com/television/2461004/bryan-cranston-responds-to-breaking-bad-movie-news\n",
      "Adding link 83: http://cinemablend.com/dvdnews/Mystery-Team-Comes-Out-Next-Week-Celebrate-With-Four-New-Clips-24713.html\n",
      "Adding link 84: http://www.cinemablend.com/news/1612770/6-dc-villains-whod-be-perfect-for-green-lantern-corps\n",
      "Adding link 85: http://www.cinemablend.com/news/1565159/a-live-action-aladdin-movie-is-happening-with-an-unexpected-director\n",
      "Adding link 86: http://www.cinemablend.com/news/1587330/the-live-action-winnie-the-pooh-movie-has-found-its-director\n",
      "Adding link 87: http://www.cinemablend.com/news/1627789/angelina-jolie-might-do-maleficent-2-as-her-next-project\n",
      "Adding link 88: http://www.cinemablend.com/news/1633799/danny-devito-is-being-eyed-for-dumbo-heres-who-he-would-play\n",
      "Adding link 89: http://www.cinemablend.com/news/1555839/why-chloe-moretz-abruptly-pulled-the-plug-on-all-of-her-upcoming-film-projects\n",
      "Adding link 90: http://www.cinemablend.com/news/1611370/the-major-actor-who-might-play-the-live-action-movies-villain\n",
      "Adding link 91: http://www.cinemablend.com/news/1560869/disney-is-doing-a-live-action-lion-king-movie-get-the-details\n",
      "Adding link 92: http://www.cinemablend.com/news/1600072/emma-stones-cruella-de-vil-movie-is-one-step-closer-to-happening\n",
      "Adding link 93: http://www.cinemablend.com/news/1623819/why-will-smith-has-passed-on-disneys-live-action-dumbo\n",
      "Adding link 94: http://www.cinemablend.com/news/1546469/disneys-live-action-little-mermaid-just-added-two-huge-musical-talents-get-the-details\n",
      "Adding link 95: http://www.cinemablend.com/news/1675689/rosario-dawson-exits-new-mutants-has-already-been-replaced\n",
      "Adding link 96: http://www.cinemablend.com/news/1682559/5-reveals-dc-needs-to-make-at-san-diego-comic-con\n",
      "Adding link 97: http://www.cinemablend.com/news.php\n",
      "Adding link 98: http://www.cinemablend.com/news.php?tag=Comic%20Con%202010\n",
      "Adding link 99: http://www.cinemablend.com/news/1538999/the-division-movie-may-have-found-its-female-lead\n",
      "Adding link 100: http://www.cinemablend.com/news/1674459/apparently-peter-parker-is-in-iron-man-2\n",
      "Adding link 101: http://www.cinemablend.com/news/1613130/why-kingpin-probably-wont-appear-in-marvel-movies-according-to-vincent-donofrio\n",
      "Adding link 102: http://www.cinemablend.com/news/1650859/7-ways-the-mcu-can-change-moving-forward\n",
      "Adding link 103: http://www.cinemablend.com/news/1542990/thor-ragnarok-may-feature-this-mcu-vehicle\n",
      "Adding link 104: http://www.cinemablend.com/news.php?tag=Oscars%202014\n",
      "Adding link 105: http://www.cinemablend.com/news/1683839/how-the-lego-ninjago-movie-was-influenced-by-phil-lord-and-chris-miller\n",
      "Adding link 106: http://www.cinemablend.com/news/1622560/the-lego-ninjago-movie-trailer-is-exciting-and-hysterical-watch-it-now\n",
      "Adding link 107: http://www.cinemablend.com/news/1684299/thor-ragnarok-showed-3-full-scenes-at-comic-con-heres-what-we-saw\n",
      "Adding link 108: http://www.cinemablend.com/news/1522460/to-3d-or-not-to-3d-buy-the-right-warcraft-ticket\n",
      "Adding link 109: https://www.cinemablend.com/gallery/dvdnews/DVD-Season-Sets-Burn-Notice-White-Collar-June-30913.html?tid=16545\n",
      "Adding link 110: http://www.cinemablend.com/news/tag/quantic%20dream/\n",
      "Adding link 111: http://cinemablend.com/dvdnews/Avatar-Extended-Collector-s-Edition-Confirmed-For-November-16th-27402.html\n",
      "Adding link 112: http://cinemablend.com/dvdnews/Rumor-The-Abyss-Coming-To-Blu-Ray-In-2011-27322.html\n",
      "Adding link 113: http://www.cinemablend.com/news.php?tag=now%20streaming\n",
      "Adding link 114: http://www.cinemablend.com/news/Dead-Rising-Watchtower-Movie-Coming-Who-It-Star-67819.html\n",
      "Adding link 115: https://www.cinemablend.com/gallery/dvdnews/Dinoshark-Trailer-May-Best-Thing-You-Watch-All-Day-30996.html?tid=16561\n",
      "Adding link 116: http://www.cinemablend.com/news.php?tag=top%2010%202012\n",
      "No. of links : 117\n"
     ]
    }
   ],
   "source": [
    "for recInd in recordListInd:\n",
    "    content = page(recordList[recInd])\n",
    "    if content == None:\n",
    "        continue\n",
    "    #print (\"Received %d bytes for %s\" % (len(content), recordList[recInd]['url']))\n",
    "    parser = BeautifulSoup(content)\n",
    "    links = parser.find_all(\"a\")\n",
    "    if links:\n",
    "        \n",
    "        numOfLinks = len(links)\n",
    "#         if (numOfLinks > 10):\n",
    "#             numOfLinks = 10\n",
    "        random.seed(10)\n",
    "        linksInd = random.sample(range(0,len(links)), numOfLinks)\n",
    "        #print(linksInd)\n",
    "        for linkInd in linksInd:\n",
    "            #print(link)\n",
    "            href = links[linkInd].attrs.get(\"href\")\n",
    "            if href is not None:\n",
    "                if domain not in href:\n",
    "                    if href not in linksList and href.startswith(\"http\") and \"cinemablend\" in href and \"game\" not in href and \"facebook\" not in href and \"support\" not in href and \"t.co\" not in href and \"google\" not in href and \"twitter\" not in href and \"youtube\" not in href and \"instagram\" not in href and \".jpg\" not in href and lookFor[choice] in href:\n",
    "                        print (\"Adding link \"+ str(len(linksList)) + \": %s\" %href)\n",
    "                        linksList.append(href)\n",
    "                        if len(linksList)>130:\n",
    "                            break\n",
    "    \n",
    "print (\"No. of links : %d\" % len(linksList))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-20T07:11:21.965412Z",
     "start_time": "2019-04-20T07:11:21.959330Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "117"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(linksList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-20T06:15:37.871466Z",
     "start_time": "2019-04-20T06:15:37.864688Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['http://www.cinemablend.com/news/1591330/a-back-to-the-future-reboot-heres-what-the-producer-thinks',\n",
       " 'http://www.cinemablend.com/news/1529940/why-disneys-moana-is-already-in-trouble-with-activists',\n",
       " 'http://www.cinemablend.com/news/1680460/how-avengers-4-could-connect-to-netflixs-marvel-shows',\n",
       " 'https://amp.cinemablend.com/news/1563679/the-gears-of-war-movie-is-finally-moving-forward-heres-what-we-know',\n",
       " 'http://www.cinemablend.com/news/1576479/the-watch-dogs-movie-wants-to-change-how-you-think-about-hackers',\n",
       " 'https://www.cinemablend.com/television/2411501/rick-and-morty-season-4-just-got-some-positive-news-for-once',\n",
       " 'http://www.cinemablend.com/news.php',\n",
       " 'http://www.cinemablend.com/news/tag/mercurysteam/',\n",
       " 'http://www.cinemablend.com/news/1566590/the-minecraft-movie-is-being-rewritten-again',\n",
       " 'http://www.cinemablend.com/news/1612770/6-dc-villains-whod-be-perfect-for-green-lantern-corps',\n",
       " 'http://www.cinemablend.com/news/1528189/dungeons--dragons-may-have-found-its-first-lead',\n",
       " 'http://www.cinemablend.com/news/1554369/sony-is-making-its-own-mulan-movie-heres-what-we-know',\n",
       " 'http://www.cinemablend.com/news/1538999/the-division-movie-may-have-found-its-female-lead',\n",
       " 'http://www.cinemablend.com/news/1560420/one-huge-way-alicia-vikanders-tomb-raider-will-differ-from-angelina-jolies-version',\n",
       " 'http://www.cinemablend.com/news/1591370/how-much-it-would-cost-to-power-the-death-star-per-day',\n",
       " 'http://cinemablend.com/dvdnews/Avatar-Extended-Collector-s-Edition-Confirmed-For-November-16th-27402.html',\n",
       " 'http://www.cinemablend.com/news/1614500/cinemablends-star-ratings-explained',\n",
       " 'http://www.cinemablend.com/news/Dead-Rising-Watchtower-Movie-Coming-Who-It-Star-67819.html',\n",
       " 'http://www.cinemablend.com/news/1636920/the-matrix-reboot-may-not-actually-be-a-reboot-at-all',\n",
       " 'http://www.cinemablend.com/news/1579639/what-mel-gibson-thought-of-mad-max-fury-road',\n",
       " 'http://www.cinemablend.com/dvdnews/FlixWorthy-Party-Down-Because-It-Might-Get-Loud-24278.html',\n",
       " 'http://www.cinemablend.com/news.php?tag=Comic%20Con%202010',\n",
       " 'http://www.cinemablend.com/news/1647420/the-last-jedi-trailer-has-secret-whispers-heres-what-we-can-hear',\n",
       " 'http://www.cinemablend.com/news/Steven-Spielberg-Making-Halo-14282.html',\n",
       " 'http://www.cinemablend.com/news/1533949/mortal-kombat-3-may-happen-with-this-incredible-sounding-twist',\n",
       " 'http://www.cinemablend.com/news/1542990/thor-ragnarok-may-feature-this-mcu-vehicle',\n",
       " 'http://www.cinemablend.com/news/1674459/apparently-peter-parker-is-in-iron-man-2',\n",
       " 'http://www.cinemablend.com/news/tag/volition/',\n",
       " 'https://www.cinemablend.com/television/2461004/bryan-cranston-responds-to-breaking-bad-movie-news',\n",
       " 'http://cinemablend.com/dvdnews/DVD-Trailer-For-After-Life-Is-Creepy-Fun-For-Death-Fetishists-25726.html',\n",
       " 'http://www.cinemablend.com/news/1522139/the-inside-joke-that-was-snuck-into-ferris-buellers-day-off',\n",
       " 'http://www.cinemablend.com/news/1642120/is-blumhouse-doing-a-five-nights-at-freddys-movie',\n",
       " 'http://www.cinemablend.com/news.php?tag=now%20streaming',\n",
       " 'https://www.cinemablend.com/gallery/dvdnews/Disney-Opens-Vault-Brings-Dumbo-Blu-Ray-31673.html?tid=16911',\n",
       " 'http://www.cinemablend.com/news/1565159/a-live-action-aladdin-movie-is-happening-with-an-unexpected-director',\n",
       " 'http://www.cinemablend.com/news/1624129/how-the-new-van-helsing-will-be-unlike-superhero-movies',\n",
       " 'http://www.cinemablend.com/news/tag/quantic%20dream/',\n",
       " 'http://www.cinemablend.com/news/652479/Spider-Man-Homecoming-What-We-Know-So-Far',\n",
       " 'http://www.cinemablend.com/news/1563049/how-nightcrawler-should-change-in-the-next-x-men-movie-according-to-kodi-smit-mcphee',\n",
       " 'http://www.cinemablend.com/news/1522800/pixars-latest-description-of-cars-3-has-us-more-than-a-little-excited',\n",
       " 'http://www.cinemablend.com/news/tag/darksiders%202/',\n",
       " 'http://www.cinemablend.com/news.php?tag=Comic%20Book%20Guides',\n",
       " 'http://www.cinemablend.com/dvdnews/Indiana-Jones-And-The-Double-Dip-DVD-9261.html',\n",
       " 'https://www.cinemablend.com/gallery/dvdnews/DVD-Season-Sets-Burn-Notice-White-Collar-June-30913.html?tid=16555',\n",
       " 'http://www.cinemablend.com/The-Hobbit-The-Battle-of-the-Five-Armies-6003.html#news',\n",
       " 'https://www.cinemablend.com/gallery/dvdnews/Disney-Opens-Vault-Brings-Dumbo-Blu-Ray-31673.html?tid=16907',\n",
       " 'http://www.cinemablend.com/news/1600830/how-many-assassins-creed-movies-michael-fassbender-thinks-they-will-make',\n",
       " 'http://www.cinemablend.com/news.php?tag=cinemacon%202014',\n",
       " 'https://www.cinemablend.com/dvdnews/Madness-Sparta-4894.html',\n",
       " 'http://cinemablend.com/dvdnews/Mystery-Team-Comes-Out-Next-Week-Celebrate-With-Four-New-Clips-24713.html',\n",
       " 'http://www.cinemablend.com/news/1584560/jurassic-world-2-will-be-both-a-jurassic-world-sequel-and-jurassic-park-5-says-ja-bayona',\n",
       " 'https://www.cinemablend.com/gallery/dvdnews/DVD-Season-Sets-Burn-Notice-White-Collar-June-30913.html?tid=16549',\n",
       " 'http://www.cinemablend.com/news/1655489/the-10-funniest-moments-in-guardians-of-the-galaxy-2',\n",
       " 'http://www.cinemablend.com/news/1620839/the-subtle-way-star-wars-the-force-awakens-ties-back-to-darth-mauls-past',\n",
       " 'https://www.cinemablend.com/dvdnews/Important-News-Canadians-With-Netflix-Who-Like-Paramount-Pictures-31017.html',\n",
       " 'http://www.cinemablend.com/news.php?tag=best%20of%202011',\n",
       " 'http://cinemablend.com/dvdnews/Netflix-Pays-Big-Money-To-Add-Paramount-And-Lionsgate-Titles-To-Instant-Watch-26277.html',\n",
       " 'http://www.cinemablend.com/news/1682559/5-reveals-dc-needs-to-make-at-san-diego-comic-con',\n",
       " 'http://www.cinemablend.com/news.php?tag=top%2010%202012',\n",
       " 'http://www.cinemablend.com/news/1608940/Upcoming-Disney-Movies-Every-Film-The-Studio-Plans-To-Release']"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l = linksList[222:]\n",
    "l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-20T07:13:54.796444Z",
     "start_time": "2019-04-20T07:13:54.790695Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(fileName[choice], 'a') as f:\n",
    "    for item in l:\n",
    "        f.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T21:59:13.352253Z",
     "start_time": "2019-04-19T21:59:13.341823Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T21:59:50.275647Z",
     "start_time": "2019-04-19T21:59:50.264834Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'rec' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-2a40257c84b9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'rec' is not defined"
     ]
    }
   ],
   "source": [
    "\"display\" not in href and \"daily\" not in href and \"archive\" not in href and \"disqus\" not in href and \"t.co\" not in href and \"google\" not in href and \"twitter\" not in href and \"youtube\" not in href and \"instagram\" not in href and "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"display\" not in href and \"daily\" not in href and \"archive\" not in href and \"disqus\" not in href and \"t.co\" not in href and \"google\" not in href and \"twitter\" not in href and \"youtube\" not in href and \"instagram\" not in href and "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}