extract_citations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import os\n",
    "import re, sys\n",
    "from collections import defaultdict, Counter\n",
    "from operator import itemgetter\n",
    "import pandas as pd\n",
    "import math\n",
    "import urllib.request\n",
    "from IPython.core.debugger import Tracer; debug_here = Tracer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31429\n",
      "44262\n"
     ]
    }
   ],
   "source": [
    "# Initialize for aggregating citations for top sources (also a script) or editor counts (should be made a script)\n",
    "#citations = Counter()\n",
    "#adds = Counter()\n",
    "#dels = Counter()\n",
    "ec = defaultdict(Counter) # ec[name][citation]\n",
    "srcs = set()\n",
    "citedir = '/home/michael/school/research/wp/wp_articles/citediffs/'\n",
    "#outpath = '/home/michael/school/research/wp/wp_articles/citations_add_del.csv'\n",
    "ecmatpath = '/home/michael/school/research/wp/wp_articles/editor_citations_matrix.csv'\n",
    "fps = []\n",
    "for p in sorted(os.listdir(citedir)):\n",
    "    if not p.startswith('.'): fps.append(os.path.join(citedir, p))\n",
    "\n",
    "# Aggregate citations from citediffs\n",
    "for fp in fps:\n",
    "    with open(fp) as f:\n",
    "        rows = [row for row in csv.DictReader(f)]\n",
    "        \n",
    "    for i, row in enumerate(rows):\n",
    "        if row['add/delete'] == 'add':\n",
    "            if row['vandalism_comment'] == '0':\n",
    "                #adds[row['citation']] += 1\n",
    "                #citations[row['citation']] += 1\n",
    "                ec[row['editor']][row['citation']] += 1\n",
    "                srcs.add(row['citation'])\n",
    "        elif row['add/delete'] == 'delete':\n",
    "            if row['vandalism_comment'] == '0':\n",
    "                #dels[row['citation']] += 1\n",
    "                #citations[row['citation']] += 1\n",
    "                ec[row['editor']][row['citation']] -= 1\n",
    "    \n",
    "srcs = sorted(list(srcs))\n",
    "\n",
    "#print(len(adds))\n",
    "#print(len(dels))\n",
    "#print(len(citations))\n",
    "print(len(ec))\n",
    "print(len(srcs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get 100 most frequent citations\n",
    "freqpath = '/home/michael/school/research/wp/wikipedia/data/citations_annotated.csv'\n",
    "freq = pd.read_csv(freqpath)\n",
    "#crit = freq['name'].map(lambda x: isinstance(x, str))\n",
    "#freq100 = freq[crit]\n",
    "freq100 = freq.loc[:99,:]\n",
    "\n",
    "freqsrcs = {}\n",
    "for _,row in freq100.iterrows():\n",
    "    if isinstance(row['variants'], str):\n",
    "        freqsrcs[row['name']] = [row['citation']]+ [var[1:-1] for var in row['variants'][1:-1].split(', ')]\n",
    "    else:\n",
    "        freqsrcs[row['name']] = [row['citation']]\n",
    "len(freqsrcs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1484"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get bot list\n",
    "bots = []\n",
    "cnum = \"\"\n",
    "done = False\n",
    "\n",
    "# Get bot category page\n",
    "category_base_url = \"https://en.wikipedia.org/w/api.php?\" + \\\n",
    "    \"action=query&\" +  \\\n",
    "    \"list=categorymembers&\" + \\\n",
    "    \"format=json&\" + \\\n",
    "    \"cmtitle=Category%3A{:s}&\" +  \\\n",
    "    \"cmprop=title&\" + \\\n",
    "    \"cmlimit=500\" + \\\n",
    "    \"&cmcontinue={:s}\"\n",
    "    \n",
    "while not done:\n",
    "    category_url = category_base_url.format(urllib.parse.quote(\"All_Wikipedia_bots\"), cnum)\n",
    "    #print(category_url)\n",
    "\n",
    "    page =  urllib.request.urlopen(category_url).read().decode('utf-8')\n",
    "    #print(page)\n",
    "\n",
    "    # Extract bot names\n",
    "    botsp = re.findall(r'\"User\\:(?P<username>[\\w\\ /\\.]*)\"\\}', page)\n",
    "    botsp = [re.sub(r'/.*', '', ed) for ed in botsp]\n",
    "    #print(botsp[:10])\n",
    "    bots += botsp\n",
    "#     print(len(bots))\n",
    "\n",
    "    # Get continue number\n",
    "    #cnum = re.search(r'cmcontinue\":\"(?P<cnum>[^\"]*)\"', page)\n",
    "    cnum = re.search(r'cmcontinue\":\"([^\"]*)\"', page)\n",
    "    if cnum:\n",
    "        cnum = cnum.group(1)\n",
    "#         print(cnum)\n",
    "    else:\n",
    "        done = True\n",
    "\n",
    "# Add other bots\n",
    "other_bots = ['SmackBot']\n",
    "bots += other_bots\n",
    "len(bots)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Print out editor citation matrix\n",
    "ec100 = {}\n",
    "variants = [x for sublist in [val for val in freqsrcs.values()] for x in sublist]\n",
    "\n",
    "# Reconstruct a combined one\n",
    "for ed in [e for e in ec if e not in bots and e != '']:\n",
    "    for src in ec[ed]:\n",
    "        if src in variants:\n",
    "            # Find which is name\n",
    "            selected_variants = [val for val in freqsrcs.values() if src in val][0]\n",
    "            name = list(freqsrcs.keys())[list(freqsrcs.values()).index(selected_variants)]\n",
    "            #if not ed in ec100: ec100[ed] = Counter()\n",
    "            if not ed in ec100: ec100[ed] = {}\n",
    "            if not name in ec100[ed]:\n",
    "                ec100[ed][name] = ec[ed][src]\n",
    "            else:\n",
    "                ec100[ed][name] += ec[ed][src]\n",
    "\n",
    "ecmat100 = pd.DataFrame(ec100)\n",
    "\n",
    "# Write editor-citation matrix\n",
    "ecmat100.to_csv(ecmatpath, na_rep=0)\n",
    "\n",
    "ecmat100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48927\n"
     ]
    }
   ],
   "source": [
    "# Print input for SVDFeature--sources and editors sorted alphabetically\n",
    "ratings = []\n",
    "for i, ed in enumerate(sorted(ec100.keys())):\n",
    "    for j, src in enumerate(sorted(freqsrcs.keys())):\n",
    "        if src in ec100[ed]:\n",
    "            rating = [ec100[ed][src], 0, 1, 1, \"{:d}:{:d}\".format(i+1,1), \"{:d}:{:d}\".format(j+1,1)]\n",
    "            ratings.append(rating)\n",
    "    \n",
    "print(len(ratings))\n",
    "\n",
    "csvpath = '/home/michael/school/research/wp/wp_articles/editor_citation_svdfeature.txt'\n",
    "\n",
    "with open(csvpath, 'w') as f:\n",
    "    w = csv.writer(f, delimiter=' ')\n",
    "    for r in ratings:\n",
    "        w.writerow(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14814"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ec100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'The Atlantic': -1})"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ec100['0DavidBerg0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>citation</th>\n",
       "      <th>additions</th>\n",
       "      <th>deletions</th>\n",
       "      <th>total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0    </th>\n",
       "      <td>                                       haaretz.com</td>\n",
       "      <td> 4659</td>\n",
       "      <td> 2803</td>\n",
       "      <td> 7462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1    </th>\n",
       "      <td>                                    news.bbc.co.uk</td>\n",
       "      <td> 4100</td>\n",
       "      <td> 3156</td>\n",
       "      <td> 7256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2    </th>\n",
       "      <td>                                         jpost.com</td>\n",
       "      <td> 3630</td>\n",
       "      <td> 2335</td>\n",
       "      <td> 5965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3    </th>\n",
       "      <td>                                    guardian.co.uk</td>\n",
       "      <td> 2315</td>\n",
       "      <td> 1692</td>\n",
       "      <td> 4007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4    </th>\n",
       "      <td>                                      ynetnews.com</td>\n",
       "      <td> 2494</td>\n",
       "      <td> 1507</td>\n",
       "      <td> 4001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5    </th>\n",
       "      <td>                                       nytimes.com</td>\n",
       "      <td> 2277</td>\n",
       "      <td> 1480</td>\n",
       "      <td> 3757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6    </th>\n",
       "      <td>                                  books.google.com</td>\n",
       "      <td> 1764</td>\n",
       "      <td> 1299</td>\n",
       "      <td> 3063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7    </th>\n",
       "      <td>                                        mfa.gov.il</td>\n",
       "      <td> 1592</td>\n",
       "      <td> 1097</td>\n",
       "      <td> 2689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8    </th>\n",
       "      <td>                          jewishvirtuallibrary.org</td>\n",
       "      <td> 1358</td>\n",
       "      <td> 1007</td>\n",
       "      <td> 2365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9    </th>\n",
       "      <td>                                     domino.un.org</td>\n",
       "      <td> 1102</td>\n",
       "      <td>  788</td>\n",
       "      <td> 1890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10   </th>\n",
       "      <td>                                            un.org</td>\n",
       "      <td> 1030</td>\n",
       "      <td>  753</td>\n",
       "      <td> 1783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11   </th>\n",
       "      <td>                                washingtonpost.com</td>\n",
       "      <td> 1009</td>\n",
       "      <td>  750</td>\n",
       "      <td> 1759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12   </th>\n",
       "      <td>                             english.aljazeera.net</td>\n",
       "      <td>  963</td>\n",
       "      <td>  659</td>\n",
       "      <td> 1622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13   </th>\n",
       "      <td>                                   telegraph.co.uk</td>\n",
       "      <td>  830</td>\n",
       "      <td>  597</td>\n",
       "      <td> 1427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14   </th>\n",
       "      <td>                                         bbc.co.uk</td>\n",
       "      <td>  867</td>\n",
       "      <td>  498</td>\n",
       "      <td> 1365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15   </th>\n",
       "      <td>                                       reuters.com</td>\n",
       "      <td>  819</td>\n",
       "      <td>  527</td>\n",
       "      <td> 1346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16   </th>\n",
       "      <td>                                           cnn.com</td>\n",
       "      <td>  732</td>\n",
       "      <td>  604</td>\n",
       "      <td> 1336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17   </th>\n",
       "      <td>                                           hrw.org</td>\n",
       "      <td>  730</td>\n",
       "      <td>  578</td>\n",
       "      <td> 1308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18   </th>\n",
       "      <td>                                            Morris</td>\n",
       "      <td>  942</td>\n",
       "      <td>  342</td>\n",
       "      <td> 1284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19   </th>\n",
       "      <td>                                   edition.cnn.com</td>\n",
       "      <td>  721</td>\n",
       "      <td>  521</td>\n",
       "      <td> 1242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20   </th>\n",
       "      <td>                                    news.yahoo.com</td>\n",
       "      <td>  637</td>\n",
       "      <td>  534</td>\n",
       "      <td> 1171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21   </th>\n",
       "      <td>                                        google.com</td>\n",
       "      <td>  674</td>\n",
       "      <td>  489</td>\n",
       "      <td> 1163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22   </th>\n",
       "      <td>                                       youtube.com</td>\n",
       "      <td>  638</td>\n",
       "      <td>  519</td>\n",
       "      <td> 1157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23   </th>\n",
       "      <td>                                       btselem.org</td>\n",
       "      <td>  659</td>\n",
       "      <td>  471</td>\n",
       "      <td> 1130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24   </th>\n",
       "      <td>                                      Benny Morris</td>\n",
       "      <td>  721</td>\n",
       "      <td>  408</td>\n",
       "      <td> 1129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25   </th>\n",
       "      <td>                                 timesonline.co.uk</td>\n",
       "      <td>  609</td>\n",
       "      <td>  484</td>\n",
       "      <td> 1093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26   </th>\n",
       "      <td>                                   web.archive.org</td>\n",
       "      <td>  632</td>\n",
       "      <td>  450</td>\n",
       "      <td> 1082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27   </th>\n",
       "      <td>                                          time.com</td>\n",
       "      <td>  620</td>\n",
       "      <td>  459</td>\n",
       "      <td> 1079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28   </th>\n",
       "      <td>                                          jcpa.org</td>\n",
       "      <td>  594</td>\n",
       "      <td>  478</td>\n",
       "      <td> 1072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29   </th>\n",
       "      <td>                            israelnationalnews.com</td>\n",
       "      <td>  627</td>\n",
       "      <td>  420</td>\n",
       "      <td> 1047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44359</th>\n",
       "      <td>                                  InAntalyaFollow </td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44360</th>\n",
       "      <td>                                        Wilson1880</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44361</th>\n",
       "      <td>            Brooklyn Journal of International Law </td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44362</th>\n",
       "      <td>                                            Dishon</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44363</th>\n",
       "      <td>                                    falmouth.ac.uk</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44364</th>\n",
       "      <td>                                             SPON1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44365</th>\n",
       "      <td>                              avigdorliberman.info</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44366</th>\n",
       "      <td>                                              Name</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44367</th>\n",
       "      <td>                                 beitjala-city.org</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44368</th>\n",
       "      <td>                                            mfa.am</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44369</th>\n",
       "      <td>                                   eng.sapir.ac.il</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44370</th>\n",
       "      <td>                                            John A</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44371</th>\n",
       "      <td>                                    GuardianJuly04</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44372</th>\n",
       "      <td>                                    secure.ssa.gov</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44373</th>\n",
       "      <td>                                         contactus</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44374</th>\n",
       "      <td>                                           myanmar</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44375</th>\n",
       "      <td>                                    ppe-agency.com</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44376</th>\n",
       "      <td>                                                BC</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44377</th>\n",
       "      <td>                                           UN 2006</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44378</th>\n",
       "      <td>                                        rhr.org.il</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44379</th>\n",
       "      <td>                         maineworldnewsservice.com</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44380</th>\n",
       "      <td>                                            Sulley</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44381</th>\n",
       "      <td>                      CenturyOne Foundation, Inc. </td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44382</th>\n",
       "      <td>                                  Ivan Van Sertima</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44383</th>\n",
       "      <td>                                       Oren 2002 p</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44384</th>\n",
       "      <td> INTERNATIONAL COMMISSION OF JURISTS and LAW IN...</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44385</th>\n",
       "      <td>                                      Barry Turner</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44386</th>\n",
       "      <td>                                     dailynews.com</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44387</th>\n",
       "      <td>                                    eurohockey.com</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44388</th>\n",
       "      <td>                              languagemagazine.com</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44389 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                citation  additions  \\\n",
       "0                                            haaretz.com       4659   \n",
       "1                                         news.bbc.co.uk       4100   \n",
       "2                                              jpost.com       3630   \n",
       "3                                         guardian.co.uk       2315   \n",
       "4                                           ynetnews.com       2494   \n",
       "5                                            nytimes.com       2277   \n",
       "6                                       books.google.com       1764   \n",
       "7                                             mfa.gov.il       1592   \n",
       "8                               jewishvirtuallibrary.org       1358   \n",
       "9                                          domino.un.org       1102   \n",
       "10                                                un.org       1030   \n",
       "11                                    washingtonpost.com       1009   \n",
       "12                                 english.aljazeera.net        963   \n",
       "13                                       telegraph.co.uk        830   \n",
       "14                                             bbc.co.uk        867   \n",
       "15                                           reuters.com        819   \n",
       "16                                               cnn.com        732   \n",
       "17                                               hrw.org        730   \n",
       "18                                                Morris        942   \n",
       "19                                       edition.cnn.com        721   \n",
       "20                                        news.yahoo.com        637   \n",
       "21                                            google.com        674   \n",
       "22                                           youtube.com        638   \n",
       "23                                           btselem.org        659   \n",
       "24                                          Benny Morris        721   \n",
       "25                                     timesonline.co.uk        609   \n",
       "26                                       web.archive.org        632   \n",
       "27                                              time.com        620   \n",
       "28                                              jcpa.org        594   \n",
       "29                                israelnationalnews.com        627   \n",
       "...                                                  ...        ...   \n",
       "44359                                   InAntalyaFollow           1   \n",
       "44360                                         Wilson1880          1   \n",
       "44361             Brooklyn Journal of International Law           1   \n",
       "44362                                             Dishon          1   \n",
       "44363                                     falmouth.ac.uk          1   \n",
       "44364                                              SPON1          1   \n",
       "44365                               avigdorliberman.info          1   \n",
       "44366                                               Name          1   \n",
       "44367                                  beitjala-city.org          1   \n",
       "44368                                             mfa.am          1   \n",
       "44369                                    eng.sapir.ac.il          1   \n",
       "44370                                             John A          1   \n",
       "44371                                     GuardianJuly04          1   \n",
       "44372                                     secure.ssa.gov          1   \n",
       "44373                                          contactus          1   \n",
       "44374                                            myanmar          1   \n",
       "44375                                     ppe-agency.com          1   \n",
       "44376                                                 BC          1   \n",
       "44377                                            UN 2006          1   \n",
       "44378                                         rhr.org.il          1   \n",
       "44379                          maineworldnewsservice.com          1   \n",
       "44380                                             Sulley          1   \n",
       "44381                       CenturyOne Foundation, Inc.           1   \n",
       "44382                                   Ivan Van Sertima          1   \n",
       "44383                                        Oren 2002 p          1   \n",
       "44384  INTERNATIONAL COMMISSION OF JURISTS and LAW IN...          1   \n",
       "44385                                       Barry Turner          1   \n",
       "44386                                      dailynews.com          1   \n",
       "44387                                     eurohockey.com          1   \n",
       "44388                               languagemagazine.com          1   \n",
       "\n",
       "       deletions  total  \n",
       "0           2803   7462  \n",
       "1           3156   7256  \n",
       "2           2335   5965  \n",
       "3           1692   4007  \n",
       "4           1507   4001  \n",
       "5           1480   3757  \n",
       "6           1299   3063  \n",
       "7           1097   2689  \n",
       "8           1007   2365  \n",
       "9            788   1890  \n",
       "10           753   1783  \n",
       "11           750   1759  \n",
       "12           659   1622  \n",
       "13           597   1427  \n",
       "14           498   1365  \n",
       "15           527   1346  \n",
       "16           604   1336  \n",
       "17           578   1308  \n",
       "18           342   1284  \n",
       "19           521   1242  \n",
       "20           534   1171  \n",
       "21           489   1163  \n",
       "22           519   1157  \n",
       "23           471   1130  \n",
       "24           408   1129  \n",
       "25           484   1093  \n",
       "26           450   1082  \n",
       "27           459   1079  \n",
       "28           478   1072  \n",
       "29           420   1047  \n",
       "...          ...    ...  \n",
       "44359          0      1  \n",
       "44360          0      1  \n",
       "44361          0      1  \n",
       "44362          0      1  \n",
       "44363          0      1  \n",
       "44364          0      1  \n",
       "44365          0      1  \n",
       "44366          0      1  \n",
       "44367          0      1  \n",
       "44368          0      1  \n",
       "44369          0      1  \n",
       "44370          0      1  \n",
       "44371          0      1  \n",
       "44372          0      1  \n",
       "44373          0      1  \n",
       "44374          0      1  \n",
       "44375          0      1  \n",
       "44376          0      1  \n",
       "44377          0      1  \n",
       "44378          0      1  \n",
       "44379          0      1  \n",
       "44380          0      1  \n",
       "44381          0      1  \n",
       "44382          0      1  \n",
       "44383          0      1  \n",
       "44384          0      1  \n",
       "44385          0      1  \n",
       "44386          0      1  \n",
       "44387          0      1  \n",
       "44388          0      1  \n",
       "\n",
       "[44389 rows x 4 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Annotate add_del figures\n",
    "raw_csvpath = '/home/michael/school/research/wp/wp_articles/citations_add_del.csv'\n",
    "raw = pd.read_csv(raw_csvpath)\n",
    "raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>additions</th>\n",
       "      <th>citation</th>\n",
       "      <th>deletions</th>\n",
       "      <th>total</th>\n",
       "      <th>variants</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BBC</th>\n",
       "      <td> 5075</td>\n",
       "      <td>                                    news.bbc.co.uk</td>\n",
       "      <td> 3741</td>\n",
       "      <td> 8816</td>\n",
       "      <td>                      [bbc.co.uk, BBC]</td>\n",
       "      <td>                                               BBC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Haaretz</th>\n",
       "      <td> 4948</td>\n",
       "      <td>                                       haaretz.com</td>\n",
       "      <td> 3011</td>\n",
       "      <td> 7959</td>\n",
       "      <td>              [haaretz.co.il, Haaretz]</td>\n",
       "      <td>                                           Haaretz</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jerusalem Post</th>\n",
       "      <td> 4038</td>\n",
       "      <td>                                         jpost.com</td>\n",
       "      <td> 2598</td>\n",
       "      <td> 6636</td>\n",
       "      <td>        [fr.jpost.com, Jerusalem Post]</td>\n",
       "      <td>                                    Jerusalem Post</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York Times</th>\n",
       "      <td> 2837</td>\n",
       "      <td>                                       nytimes.com</td>\n",
       "      <td> 1903</td>\n",
       "      <td> 4740</td>\n",
       "      <td>   [query.nytimes.com, New York Times]</td>\n",
       "      <td>                                    New York Times</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ynet News</th>\n",
       "      <td> 2924</td>\n",
       "      <td>                                      ynetnews.com</td>\n",
       "      <td> 1786</td>\n",
       "      <td> 4710</td>\n",
       "      <td>                          [ynet.co.il]</td>\n",
       "      <td>                                         Ynet News</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Guardian</th>\n",
       "      <td> 2331</td>\n",
       "      <td>                                    guardian.co.uk</td>\n",
       "      <td> 1706</td>\n",
       "      <td> 4037</td>\n",
       "      <td>                        [The Guardian]</td>\n",
       "      <td>                                      The Guardian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UN</th>\n",
       "      <td> 2167</td>\n",
       "      <td>                                     domino.un.org</td>\n",
       "      <td> 1566</td>\n",
       "      <td> 3733</td>\n",
       "      <td>                          [un.org, UN]</td>\n",
       "      <td>                                                UN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>books.google.com</th>\n",
       "      <td> 1764</td>\n",
       "      <td>                                  books.google.com</td>\n",
       "      <td> 1299</td>\n",
       "      <td> 3063</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                  books.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Israel Ministry of Foreign Affairs</th>\n",
       "      <td> 1613</td>\n",
       "      <td>                                        mfa.gov.il</td>\n",
       "      <td> 1113</td>\n",
       "      <td> 2726</td>\n",
       "      <td>  [Israel Ministry of Foreign Affairs]</td>\n",
       "      <td>                Israel Ministry of Foreign Affairs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CNN</th>\n",
       "      <td> 1481</td>\n",
       "      <td>                                           cnn.com</td>\n",
       "      <td> 1137</td>\n",
       "      <td> 2618</td>\n",
       "      <td>                [edition.cnn.com, CNN]</td>\n",
       "      <td>                                               CNN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Benny Morris</th>\n",
       "      <td> 1663</td>\n",
       "      <td>                                            Morris</td>\n",
       "      <td>  750</td>\n",
       "      <td> 2413</td>\n",
       "      <td>                        [Benny Morris]</td>\n",
       "      <td>                                      Benny Morris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jewish Virtual Library</th>\n",
       "      <td> 1367</td>\n",
       "      <td>                          jewishvirtuallibrary.org</td>\n",
       "      <td> 1016</td>\n",
       "      <td> 2383</td>\n",
       "      <td>              [Jewish Virtual Library]</td>\n",
       "      <td>                            Jewish Virtual Library</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Al Jazeera</th>\n",
       "      <td> 1378</td>\n",
       "      <td>                             english.aljazeera.net</td>\n",
       "      <td>  902</td>\n",
       "      <td> 2280</td>\n",
       "      <td>           [aljazeera.com, Al Jazeera]</td>\n",
       "      <td>                                        Al Jazeera</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Washington Post</th>\n",
       "      <td> 1020</td>\n",
       "      <td>                                washingtonpost.com</td>\n",
       "      <td>  760</td>\n",
       "      <td> 1780</td>\n",
       "      <td>                     [Washington Post]</td>\n",
       "      <td>                                   Washington Post</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reuters</th>\n",
       "      <td> 1064</td>\n",
       "      <td>                                       reuters.com</td>\n",
       "      <td>  711</td>\n",
       "      <td> 1775</td>\n",
       "      <td>             [uk.reuters.com, Reuters]</td>\n",
       "      <td>                                           Reuters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Telegraph</th>\n",
       "      <td>  830</td>\n",
       "      <td>                                   telegraph.co.uk</td>\n",
       "      <td>  597</td>\n",
       "      <td> 1427</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                     The Telegraph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Human Rights Watch</th>\n",
       "      <td>  753</td>\n",
       "      <td>                                           hrw.org</td>\n",
       "      <td>  594</td>\n",
       "      <td> 1347</td>\n",
       "      <td>                  [Human Rights Watch]</td>\n",
       "      <td>                                Human Rights Watch</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yahoo News</th>\n",
       "      <td>  637</td>\n",
       "      <td>                                    news.yahoo.com</td>\n",
       "      <td>  534</td>\n",
       "      <td> 1171</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                        Yahoo News</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>google.com</th>\n",
       "      <td>  674</td>\n",
       "      <td>                                        google.com</td>\n",
       "      <td>  489</td>\n",
       "      <td> 1163</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                        google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>youtube.com</th>\n",
       "      <td>  638</td>\n",
       "      <td>                                       youtube.com</td>\n",
       "      <td>  519</td>\n",
       "      <td> 1157</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                       youtube.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B'Tselem</th>\n",
       "      <td>  665</td>\n",
       "      <td>                                       btselem.org</td>\n",
       "      <td>  475</td>\n",
       "      <td> 1140</td>\n",
       "      <td>                            [B'Tselem]</td>\n",
       "      <td>                                          B'Tselem</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Time</th>\n",
       "      <td>  651</td>\n",
       "      <td>                                          time.com</td>\n",
       "      <td>  481</td>\n",
       "      <td> 1132</td>\n",
       "      <td>                                [Time]</td>\n",
       "      <td>                                              Time</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Times of London</th>\n",
       "      <td>  609</td>\n",
       "      <td>                                 timesonline.co.uk</td>\n",
       "      <td>  484</td>\n",
       "      <td> 1093</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                   Times of London</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>web.archive.org</th>\n",
       "      <td>  632</td>\n",
       "      <td>                                   web.archive.org</td>\n",
       "      <td>  450</td>\n",
       "      <td> 1082</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                   web.archive.org</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jerusalem Center for Public Affairs</th>\n",
       "      <td>  595</td>\n",
       "      <td>                                          jcpa.org</td>\n",
       "      <td>  478</td>\n",
       "      <td> 1073</td>\n",
       "      <td> [Jerusalem Center for Public Affairs]</td>\n",
       "      <td>               Jerusalem Center for Public Affairs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Arutz Sheva (Israel News)</th>\n",
       "      <td>  627</td>\n",
       "      <td>                            israelnationalnews.com</td>\n",
       "      <td>  420</td>\n",
       "      <td> 1047</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                         Arutz Sheva (Israel News)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>US State Dept</th>\n",
       "      <td>  549</td>\n",
       "      <td>                                         state.gov</td>\n",
       "      <td>  456</td>\n",
       "      <td> 1005</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                     US State Dept</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unispal.un.org</th>\n",
       "      <td>  610</td>\n",
       "      <td>                                    unispal.un.org</td>\n",
       "      <td>  320</td>\n",
       "      <td>  930</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                    unispal.un.org</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Palestine Remembered</th>\n",
       "      <td>  587</td>\n",
       "      <td>                           palestineremembered.com</td>\n",
       "      <td>  327</td>\n",
       "      <td>  914</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                              Palestine Remembered</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rashid Khalidi</th>\n",
       "      <td>  672</td>\n",
       "      <td>                                           Khalidi</td>\n",
       "      <td>  235</td>\n",
       "      <td>  907</td>\n",
       "      <td>                      [Rashid Khalidi]</td>\n",
       "      <td>                                    Rashid Khalidi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Opposition claim</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                  Opposition claim</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                  Opposition claim</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Presence</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                          Presence</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                          Presence</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shmtemple.org</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                     shmtemple.org</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                     shmtemple.org</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Govrin</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                            Govrin</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                            Govrin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tamar Levy</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                        Tamar Levy</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                        Tamar Levy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>khalidi128</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                        khalidi128</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                        khalidi128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Al-Taee</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                           Al-Taee</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                           Al-Taee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fund</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                              fund</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                              fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>miscamble51</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                       miscamble51</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                       miscamble51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Viktor Suvorov</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                    Viktor Suvorov</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                    Viktor Suvorov</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Korman</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                            Korman</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                            Korman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fruchter-Ronen</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                    Fruchter-Ronen</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                    Fruchter-Ronen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pitcher</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                           Pitcher</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                           Pitcher</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sonne Isaiah (1962) </th>\n",
       "      <td>    1</td>\n",
       "      <td>                              Sonne Isaiah (1962) </td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                              Sonne Isaiah (1962) </td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spelling given in Morris</th>\n",
       "      <td>    1</td>\n",
       "      <td>                          Spelling given in Morris</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                          Spelling given in Morris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Out of the Ashes: Destruction and Reconstruction of East Timor</th>\n",
       "      <td>    1</td>\n",
       "      <td> Out of the Ashes: Destruction and Reconstructi...</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td> Out of the Ashes: Destruction and Reconstructi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mati Alon</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                         Mati Alon</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                         Mati Alon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bandy.or.jp</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                       bandy.or.jp</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                       bandy.or.jp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th> The Weekly Standard </th>\n",
       "      <td>    1</td>\n",
       "      <td>                              The Weekly Standard </td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                              The Weekly Standard </td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>izrus.co.il</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                       izrus.co.il</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                       izrus.co.il</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Heng</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                              Heng</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                              Heng</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Megillah (Talmud)</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                 Megillah (Talmud)</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                 Megillah (Talmud)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Keating (2005)</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                    Keating (2005)</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                    Keating (2005)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gail D. Hershenzon</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                Gail D. Hershenzon</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                Gail D. Hershenzon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colour code</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                       Colour code</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                       Colour code</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yusuf Mazhar</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                      Yusuf Mazhar</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                      Yusuf Mazhar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Theological dictionary of the Old Testament</th>\n",
       "      <td>    1</td>\n",
       "      <td>       Theological dictionary of the Old Testament</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>       Theological dictionary of the Old Testament</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[[Thant Myint-U]]</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                 [[Thant Myint-U]]</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                 [[Thant Myint-U]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bardin</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                            Bardin</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                            Bardin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gobarev pages 1-29</th>\n",
       "      <td>    1</td>\n",
       "      <td>                                Gobarev pages 1-29</td>\n",
       "      <td>    0</td>\n",
       "      <td>    1</td>\n",
       "      <td>                                   NaN</td>\n",
       "      <td>                                Gobarev pages 1-29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44326 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                               additions  \\\n",
       "BBC                                                                 5075   \n",
       "Haaretz                                                             4948   \n",
       "Jerusalem Post                                                      4038   \n",
       "New York Times                                                      2837   \n",
       "Ynet News                                                           2924   \n",
       "The Guardian                                                        2331   \n",
       "UN                                                                  2167   \n",
       "books.google.com                                                    1764   \n",
       "Israel Ministry of Foreign Affairs                                  1613   \n",
       "CNN                                                                 1481   \n",
       "Benny Morris                                                        1663   \n",
       "Jewish Virtual Library                                              1367   \n",
       "Al Jazeera                                                          1378   \n",
       "Washington Post                                                     1020   \n",
       "Reuters                                                             1064   \n",
       "The Telegraph                                                        830   \n",
       "Human Rights Watch                                                   753   \n",
       "Yahoo News                                                           637   \n",
       "google.com                                                           674   \n",
       "youtube.com                                                          638   \n",
       "B'Tselem                                                             665   \n",
       "Time                                                                 651   \n",
       "Times of London                                                      609   \n",
       "web.archive.org                                                      632   \n",
       "Jerusalem Center for Public Affairs                                  595   \n",
       "Arutz Sheva (Israel News)                                            627   \n",
       "US State Dept                                                        549   \n",
       "unispal.un.org                                                       610   \n",
       "Palestine Remembered                                                 587   \n",
       "Rashid Khalidi                                                       672   \n",
       "...                                                                  ...   \n",
       "Opposition claim                                                       1   \n",
       "Presence                                                               1   \n",
       "shmtemple.org                                                          1   \n",
       "Govrin                                                                 1   \n",
       "Tamar Levy                                                             1   \n",
       "khalidi128                                                             1   \n",
       "Al-Taee                                                                1   \n",
       "fund                                                                   1   \n",
       "miscamble51                                                            1   \n",
       "Viktor Suvorov                                                         1   \n",
       "Korman                                                                 1   \n",
       "Fruchter-Ronen                                                         1   \n",
       "Pitcher                                                                1   \n",
       "Sonne Isaiah (1962)                                                    1   \n",
       "Spelling given in Morris                                               1   \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor         1   \n",
       "Mati Alon                                                              1   \n",
       "bandy.or.jp                                                            1   \n",
       " The Weekly Standard                                                   1   \n",
       "izrus.co.il                                                            1   \n",
       "Heng                                                                   1   \n",
       "Megillah (Talmud)                                                      1   \n",
       "Keating (2005)                                                         1   \n",
       "Gail D. Hershenzon                                                     1   \n",
       "Colour code                                                            1   \n",
       "Yusuf Mazhar                                                           1   \n",
       "Theological dictionary of the Old Testament                            1   \n",
       "[[Thant Myint-U]]                                                      1   \n",
       "Bardin                                                                 1   \n",
       "Gobarev pages 1-29                                                     1   \n",
       "\n",
       "                                                                                                         citation  \\\n",
       "BBC                                                                                                news.bbc.co.uk   \n",
       "Haaretz                                                                                               haaretz.com   \n",
       "Jerusalem Post                                                                                          jpost.com   \n",
       "New York Times                                                                                        nytimes.com   \n",
       "Ynet News                                                                                            ynetnews.com   \n",
       "The Guardian                                                                                       guardian.co.uk   \n",
       "UN                                                                                                  domino.un.org   \n",
       "books.google.com                                                                                 books.google.com   \n",
       "Israel Ministry of Foreign Affairs                                                                     mfa.gov.il   \n",
       "CNN                                                                                                       cnn.com   \n",
       "Benny Morris                                                                                               Morris   \n",
       "Jewish Virtual Library                                                                   jewishvirtuallibrary.org   \n",
       "Al Jazeera                                                                                  english.aljazeera.net   \n",
       "Washington Post                                                                                washingtonpost.com   \n",
       "Reuters                                                                                               reuters.com   \n",
       "The Telegraph                                                                                     telegraph.co.uk   \n",
       "Human Rights Watch                                                                                        hrw.org   \n",
       "Yahoo News                                                                                         news.yahoo.com   \n",
       "google.com                                                                                             google.com   \n",
       "youtube.com                                                                                           youtube.com   \n",
       "B'Tselem                                                                                              btselem.org   \n",
       "Time                                                                                                     time.com   \n",
       "Times of London                                                                                 timesonline.co.uk   \n",
       "web.archive.org                                                                                   web.archive.org   \n",
       "Jerusalem Center for Public Affairs                                                                      jcpa.org   \n",
       "Arutz Sheva (Israel News)                                                                  israelnationalnews.com   \n",
       "US State Dept                                                                                           state.gov   \n",
       "unispal.un.org                                                                                     unispal.un.org   \n",
       "Palestine Remembered                                                                      palestineremembered.com   \n",
       "Rashid Khalidi                                                                                            Khalidi   \n",
       "...                                                                                                           ...   \n",
       "Opposition claim                                                                                 Opposition claim   \n",
       "Presence                                                                                                 Presence   \n",
       "shmtemple.org                                                                                       shmtemple.org   \n",
       "Govrin                                                                                                     Govrin   \n",
       "Tamar Levy                                                                                             Tamar Levy   \n",
       "khalidi128                                                                                             khalidi128   \n",
       "Al-Taee                                                                                                   Al-Taee   \n",
       "fund                                                                                                         fund   \n",
       "miscamble51                                                                                           miscamble51   \n",
       "Viktor Suvorov                                                                                     Viktor Suvorov   \n",
       "Korman                                                                                                     Korman   \n",
       "Fruchter-Ronen                                                                                     Fruchter-Ronen   \n",
       "Pitcher                                                                                                   Pitcher   \n",
       "Sonne Isaiah (1962)                                                                          Sonne Isaiah (1962)    \n",
       "Spelling given in Morris                                                                 Spelling given in Morris   \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor  Out of the Ashes: Destruction and Reconstructi...   \n",
       "Mati Alon                                                                                               Mati Alon   \n",
       "bandy.or.jp                                                                                           bandy.or.jp   \n",
       " The Weekly Standard                                                                         The Weekly Standard    \n",
       "izrus.co.il                                                                                           izrus.co.il   \n",
       "Heng                                                                                                         Heng   \n",
       "Megillah (Talmud)                                                                               Megillah (Talmud)   \n",
       "Keating (2005)                                                                                     Keating (2005)   \n",
       "Gail D. Hershenzon                                                                             Gail D. Hershenzon   \n",
       "Colour code                                                                                           Colour code   \n",
       "Yusuf Mazhar                                                                                         Yusuf Mazhar   \n",
       "Theological dictionary of the Old Testament                           Theological dictionary of the Old Testament   \n",
       "[[Thant Myint-U]]                                                                               [[Thant Myint-U]]   \n",
       "Bardin                                                                                                     Bardin   \n",
       "Gobarev pages 1-29                                                                             Gobarev pages 1-29   \n",
       "\n",
       "                                                               deletions  \\\n",
       "BBC                                                                 3741   \n",
       "Haaretz                                                             3011   \n",
       "Jerusalem Post                                                      2598   \n",
       "New York Times                                                      1903   \n",
       "Ynet News                                                           1786   \n",
       "The Guardian                                                        1706   \n",
       "UN                                                                  1566   \n",
       "books.google.com                                                    1299   \n",
       "Israel Ministry of Foreign Affairs                                  1113   \n",
       "CNN                                                                 1137   \n",
       "Benny Morris                                                         750   \n",
       "Jewish Virtual Library                                              1016   \n",
       "Al Jazeera                                                           902   \n",
       "Washington Post                                                      760   \n",
       "Reuters                                                              711   \n",
       "The Telegraph                                                        597   \n",
       "Human Rights Watch                                                   594   \n",
       "Yahoo News                                                           534   \n",
       "google.com                                                           489   \n",
       "youtube.com                                                          519   \n",
       "B'Tselem                                                             475   \n",
       "Time                                                                 481   \n",
       "Times of London                                                      484   \n",
       "web.archive.org                                                      450   \n",
       "Jerusalem Center for Public Affairs                                  478   \n",
       "Arutz Sheva (Israel News)                                            420   \n",
       "US State Dept                                                        456   \n",
       "unispal.un.org                                                       320   \n",
       "Palestine Remembered                                                 327   \n",
       "Rashid Khalidi                                                       235   \n",
       "...                                                                  ...   \n",
       "Opposition claim                                                       0   \n",
       "Presence                                                               0   \n",
       "shmtemple.org                                                          0   \n",
       "Govrin                                                                 0   \n",
       "Tamar Levy                                                             0   \n",
       "khalidi128                                                             0   \n",
       "Al-Taee                                                                0   \n",
       "fund                                                                   0   \n",
       "miscamble51                                                            0   \n",
       "Viktor Suvorov                                                         0   \n",
       "Korman                                                                 0   \n",
       "Fruchter-Ronen                                                         0   \n",
       "Pitcher                                                                0   \n",
       "Sonne Isaiah (1962)                                                    0   \n",
       "Spelling given in Morris                                               0   \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor         0   \n",
       "Mati Alon                                                              0   \n",
       "bandy.or.jp                                                            0   \n",
       " The Weekly Standard                                                   0   \n",
       "izrus.co.il                                                            0   \n",
       "Heng                                                                   0   \n",
       "Megillah (Talmud)                                                      0   \n",
       "Keating (2005)                                                         0   \n",
       "Gail D. Hershenzon                                                     0   \n",
       "Colour code                                                            0   \n",
       "Yusuf Mazhar                                                           0   \n",
       "Theological dictionary of the Old Testament                            0   \n",
       "[[Thant Myint-U]]                                                      0   \n",
       "Bardin                                                                 0   \n",
       "Gobarev pages 1-29                                                     0   \n",
       "\n",
       "                                                               total  \\\n",
       "BBC                                                             8816   \n",
       "Haaretz                                                         7959   \n",
       "Jerusalem Post                                                  6636   \n",
       "New York Times                                                  4740   \n",
       "Ynet News                                                       4710   \n",
       "The Guardian                                                    4037   \n",
       "UN                                                              3733   \n",
       "books.google.com                                                3063   \n",
       "Israel Ministry of Foreign Affairs                              2726   \n",
       "CNN                                                             2618   \n",
       "Benny Morris                                                    2413   \n",
       "Jewish Virtual Library                                          2383   \n",
       "Al Jazeera                                                      2280   \n",
       "Washington Post                                                 1780   \n",
       "Reuters                                                         1775   \n",
       "The Telegraph                                                   1427   \n",
       "Human Rights Watch                                              1347   \n",
       "Yahoo News                                                      1171   \n",
       "google.com                                                      1163   \n",
       "youtube.com                                                     1157   \n",
       "B'Tselem                                                        1140   \n",
       "Time                                                            1132   \n",
       "Times of London                                                 1093   \n",
       "web.archive.org                                                 1082   \n",
       "Jerusalem Center for Public Affairs                             1073   \n",
       "Arutz Sheva (Israel News)                                       1047   \n",
       "US State Dept                                                   1005   \n",
       "unispal.un.org                                                   930   \n",
       "Palestine Remembered                                             914   \n",
       "Rashid Khalidi                                                   907   \n",
       "...                                                              ...   \n",
       "Opposition claim                                                   1   \n",
       "Presence                                                           1   \n",
       "shmtemple.org                                                      1   \n",
       "Govrin                                                             1   \n",
       "Tamar Levy                                                         1   \n",
       "khalidi128                                                         1   \n",
       "Al-Taee                                                            1   \n",
       "fund                                                               1   \n",
       "miscamble51                                                        1   \n",
       "Viktor Suvorov                                                     1   \n",
       "Korman                                                             1   \n",
       "Fruchter-Ronen                                                     1   \n",
       "Pitcher                                                            1   \n",
       "Sonne Isaiah (1962)                                                1   \n",
       "Spelling given in Morris                                           1   \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor     1   \n",
       "Mati Alon                                                          1   \n",
       "bandy.or.jp                                                        1   \n",
       " The Weekly Standard                                               1   \n",
       "izrus.co.il                                                        1   \n",
       "Heng                                                               1   \n",
       "Megillah (Talmud)                                                  1   \n",
       "Keating (2005)                                                     1   \n",
       "Gail D. Hershenzon                                                 1   \n",
       "Colour code                                                        1   \n",
       "Yusuf Mazhar                                                       1   \n",
       "Theological dictionary of the Old Testament                        1   \n",
       "[[Thant Myint-U]]                                                  1   \n",
       "Bardin                                                             1   \n",
       "Gobarev pages 1-29                                                 1   \n",
       "\n",
       "                                                                                             variants  \\\n",
       "BBC                                                                                  [bbc.co.uk, BBC]   \n",
       "Haaretz                                                                      [haaretz.co.il, Haaretz]   \n",
       "Jerusalem Post                                                         [fr.jpost.com, Jerusalem Post]   \n",
       "New York Times                                                    [query.nytimes.com, New York Times]   \n",
       "Ynet News                                                                                [ynet.co.il]   \n",
       "The Guardian                                                                           [The Guardian]   \n",
       "UN                                                                                       [un.org, UN]   \n",
       "books.google.com                                                                                  NaN   \n",
       "Israel Ministry of Foreign Affairs                               [Israel Ministry of Foreign Affairs]   \n",
       "CNN                                                                            [edition.cnn.com, CNN]   \n",
       "Benny Morris                                                                           [Benny Morris]   \n",
       "Jewish Virtual Library                                                       [Jewish Virtual Library]   \n",
       "Al Jazeera                                                                [aljazeera.com, Al Jazeera]   \n",
       "Washington Post                                                                     [Washington Post]   \n",
       "Reuters                                                                     [uk.reuters.com, Reuters]   \n",
       "The Telegraph                                                                                     NaN   \n",
       "Human Rights Watch                                                               [Human Rights Watch]   \n",
       "Yahoo News                                                                                        NaN   \n",
       "google.com                                                                                        NaN   \n",
       "youtube.com                                                                                       NaN   \n",
       "B'Tselem                                                                                   [B'Tselem]   \n",
       "Time                                                                                           [Time]   \n",
       "Times of London                                                                                   NaN   \n",
       "web.archive.org                                                                                   NaN   \n",
       "Jerusalem Center for Public Affairs                             [Jerusalem Center for Public Affairs]   \n",
       "Arutz Sheva (Israel News)                                                                         NaN   \n",
       "US State Dept                                                                                     NaN   \n",
       "unispal.un.org                                                                                    NaN   \n",
       "Palestine Remembered                                                                              NaN   \n",
       "Rashid Khalidi                                                                       [Rashid Khalidi]   \n",
       "...                                                                                               ...   \n",
       "Opposition claim                                                                                  NaN   \n",
       "Presence                                                                                          NaN   \n",
       "shmtemple.org                                                                                     NaN   \n",
       "Govrin                                                                                            NaN   \n",
       "Tamar Levy                                                                                        NaN   \n",
       "khalidi128                                                                                        NaN   \n",
       "Al-Taee                                                                                           NaN   \n",
       "fund                                                                                              NaN   \n",
       "miscamble51                                                                                       NaN   \n",
       "Viktor Suvorov                                                                                    NaN   \n",
       "Korman                                                                                            NaN   \n",
       "Fruchter-Ronen                                                                                    NaN   \n",
       "Pitcher                                                                                           NaN   \n",
       "Sonne Isaiah (1962)                                                                               NaN   \n",
       "Spelling given in Morris                                                                          NaN   \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor                                    NaN   \n",
       "Mati Alon                                                                                         NaN   \n",
       "bandy.or.jp                                                                                       NaN   \n",
       " The Weekly Standard                                                                              NaN   \n",
       "izrus.co.il                                                                                       NaN   \n",
       "Heng                                                                                              NaN   \n",
       "Megillah (Talmud)                                                                                 NaN   \n",
       "Keating (2005)                                                                                    NaN   \n",
       "Gail D. Hershenzon                                                                                NaN   \n",
       "Colour code                                                                                       NaN   \n",
       "Yusuf Mazhar                                                                                      NaN   \n",
       "Theological dictionary of the Old Testament                                                       NaN   \n",
       "[[Thant Myint-U]]                                                                                 NaN   \n",
       "Bardin                                                                                            NaN   \n",
       "Gobarev pages 1-29                                                                                NaN   \n",
       "\n",
       "                                                                                                             name  \n",
       "BBC                                                                                                           BBC  \n",
       "Haaretz                                                                                                   Haaretz  \n",
       "Jerusalem Post                                                                                     Jerusalem Post  \n",
       "New York Times                                                                                     New York Times  \n",
       "Ynet News                                                                                               Ynet News  \n",
       "The Guardian                                                                                         The Guardian  \n",
       "UN                                                                                                             UN  \n",
       "books.google.com                                                                                 books.google.com  \n",
       "Israel Ministry of Foreign Affairs                                             Israel Ministry of Foreign Affairs  \n",
       "CNN                                                                                                           CNN  \n",
       "Benny Morris                                                                                         Benny Morris  \n",
       "Jewish Virtual Library                                                                     Jewish Virtual Library  \n",
       "Al Jazeera                                                                                             Al Jazeera  \n",
       "Washington Post                                                                                   Washington Post  \n",
       "Reuters                                                                                                   Reuters  \n",
       "The Telegraph                                                                                       The Telegraph  \n",
       "Human Rights Watch                                                                             Human Rights Watch  \n",
       "Yahoo News                                                                                             Yahoo News  \n",
       "google.com                                                                                             google.com  \n",
       "youtube.com                                                                                           youtube.com  \n",
       "B'Tselem                                                                                                 B'Tselem  \n",
       "Time                                                                                                         Time  \n",
       "Times of London                                                                                   Times of London  \n",
       "web.archive.org                                                                                   web.archive.org  \n",
       "Jerusalem Center for Public Affairs                                           Jerusalem Center for Public Affairs  \n",
       "Arutz Sheva (Israel News)                                                               Arutz Sheva (Israel News)  \n",
       "US State Dept                                                                                       US State Dept  \n",
       "unispal.un.org                                                                                     unispal.un.org  \n",
       "Palestine Remembered                                                                         Palestine Remembered  \n",
       "Rashid Khalidi                                                                                     Rashid Khalidi  \n",
       "...                                                                                                           ...  \n",
       "Opposition claim                                                                                 Opposition claim  \n",
       "Presence                                                                                                 Presence  \n",
       "shmtemple.org                                                                                       shmtemple.org  \n",
       "Govrin                                                                                                     Govrin  \n",
       "Tamar Levy                                                                                             Tamar Levy  \n",
       "khalidi128                                                                                             khalidi128  \n",
       "Al-Taee                                                                                                   Al-Taee  \n",
       "fund                                                                                                         fund  \n",
       "miscamble51                                                                                           miscamble51  \n",
       "Viktor Suvorov                                                                                     Viktor Suvorov  \n",
       "Korman                                                                                                     Korman  \n",
       "Fruchter-Ronen                                                                                     Fruchter-Ronen  \n",
       "Pitcher                                                                                                   Pitcher  \n",
       "Sonne Isaiah (1962)                                                                          Sonne Isaiah (1962)   \n",
       "Spelling given in Morris                                                                 Spelling given in Morris  \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor  Out of the Ashes: Destruction and Reconstructi...  \n",
       "Mati Alon                                                                                               Mati Alon  \n",
       "bandy.or.jp                                                                                           bandy.or.jp  \n",
       " The Weekly Standard                                                                         The Weekly Standard   \n",
       "izrus.co.il                                                                                           izrus.co.il  \n",
       "Heng                                                                                                         Heng  \n",
       "Megillah (Talmud)                                                                               Megillah (Talmud)  \n",
       "Keating (2005)                                                                                     Keating (2005)  \n",
       "Gail D. Hershenzon                                                                             Gail D. Hershenzon  \n",
       "Colour code                                                                                           Colour code  \n",
       "Yusuf Mazhar                                                                                         Yusuf Mazhar  \n",
       "Theological dictionary of the Old Testament                           Theological dictionary of the Old Testament  \n",
       "[[Thant Myint-U]]                                                                               [[Thant Myint-U]]  \n",
       "Bardin                                                                                                     Bardin  \n",
       "Gobarev pages 1-29                                                                             Gobarev pages 1-29  \n",
       "\n",
       "[44326 rows x 6 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "variants = [x for sublist in [val for val in freqsrcs.values()] for x in sublist]\n",
    "\n",
    "# Reconstruct a combined one\n",
    "counts = {}\n",
    "for i, row in raw.iterrows():\n",
    "    if row['citation'] in variants:\n",
    "        # Find which is name\n",
    "        selected_variants = [val for val in freqsrcs.values() if row['citation'] in val][0]\n",
    "        name = list(freqsrcs.keys())[list(freqsrcs.values()).index(selected_variants)]\n",
    "        #print(row['citation'], name)\n",
    "        if name in counts:\n",
    "            counts[name]['additions'] += row['additions']\n",
    "            counts[name]['deletions'] += row['deletions']\n",
    "            counts[name]['total'] += row['total']\n",
    "            if 'variants' in counts[name]:\n",
    "                counts[name]['variants'].append(row['citation'])\n",
    "            else:\n",
    "                counts[name]['variants'] = [row['citation']]\n",
    "                \n",
    "        else:\n",
    "            counts[name] = {}\n",
    "            counts[name]['citation'] = row['citation']\n",
    "            counts[name]['additions'] = row['additions']\n",
    "            counts[name]['deletions'] = row['deletions']\n",
    "            counts[name]['total'] = row['total']\n",
    "            \n",
    "    elif row['citation'] in counts: # name of the source\n",
    "        counts[row['citation']]['additions'] += row['additions']\n",
    "        counts[row['citation']]['deletions'] += row['deletions']\n",
    "        counts[row['citation']]['total'] += row['total']\n",
    "        if 'variants' in counts[row['citation']]:\n",
    "            counts[row['citation']]['variants'].append(row['citation'])\n",
    "        else:\n",
    "            counts[row['citation']]['variants'] = [row['citation']]\n",
    "        \n",
    "    else:\n",
    "        counts[row['citation']] = {}\n",
    "        counts[row['citation']]['citation'] = row['citation']\n",
    "        counts[row['citation']]['additions'] = row['additions']\n",
    "        counts[row['citation']]['deletions'] = row['deletions']\n",
    "        counts[row['citation']]['total'] = row['total']\n",
    "\n",
    "annotated = pd.DataFrame(counts).transpose()\n",
    "annotated['name'] = annotated.index\n",
    "annotated = annotated.sort(columns=['total'], ascending=False)\n",
    "annotated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>total</th>\n",
       "      <th>additions</th>\n",
       "      <th>deletions</th>\n",
       "      <th>citation</th>\n",
       "      <th>variants</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BBC</th>\n",
       "      <td>                                               BBC</td>\n",
       "      <td> 8816</td>\n",
       "      <td> 5075</td>\n",
       "      <td> 3741</td>\n",
       "      <td>                                    news.bbc.co.uk</td>\n",
       "      <td>                      [bbc.co.uk, BBC]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Haaretz</th>\n",
       "      <td>                                           Haaretz</td>\n",
       "      <td> 7959</td>\n",
       "      <td> 4948</td>\n",
       "      <td> 3011</td>\n",
       "      <td>                                       haaretz.com</td>\n",
       "      <td>              [haaretz.co.il, Haaretz]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jerusalem Post</th>\n",
       "      <td>                                    Jerusalem Post</td>\n",
       "      <td> 6636</td>\n",
       "      <td> 4038</td>\n",
       "      <td> 2598</td>\n",
       "      <td>                                         jpost.com</td>\n",
       "      <td>        [fr.jpost.com, Jerusalem Post]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York Times</th>\n",
       "      <td>                                    New York Times</td>\n",
       "      <td> 4740</td>\n",
       "      <td> 2837</td>\n",
       "      <td> 1903</td>\n",
       "      <td>                                       nytimes.com</td>\n",
       "      <td>   [query.nytimes.com, New York Times]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ynet News</th>\n",
       "      <td>                                         Ynet News</td>\n",
       "      <td> 4710</td>\n",
       "      <td> 2924</td>\n",
       "      <td> 1786</td>\n",
       "      <td>                                      ynetnews.com</td>\n",
       "      <td>                          [ynet.co.il]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Guardian</th>\n",
       "      <td>                                      The Guardian</td>\n",
       "      <td> 4037</td>\n",
       "      <td> 2331</td>\n",
       "      <td> 1706</td>\n",
       "      <td>                                    guardian.co.uk</td>\n",
       "      <td>                        [The Guardian]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UN</th>\n",
       "      <td>                                                UN</td>\n",
       "      <td> 3733</td>\n",
       "      <td> 2167</td>\n",
       "      <td> 1566</td>\n",
       "      <td>                                     domino.un.org</td>\n",
       "      <td>                          [un.org, UN]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>books.google.com</th>\n",
       "      <td>                                  books.google.com</td>\n",
       "      <td> 3063</td>\n",
       "      <td> 1764</td>\n",
       "      <td> 1299</td>\n",
       "      <td>                                  books.google.com</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Israel Ministry of Foreign Affairs</th>\n",
       "      <td>                Israel Ministry of Foreign Affairs</td>\n",
       "      <td> 2726</td>\n",
       "      <td> 1613</td>\n",
       "      <td> 1113</td>\n",
       "      <td>                                        mfa.gov.il</td>\n",
       "      <td>  [Israel Ministry of Foreign Affairs]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CNN</th>\n",
       "      <td>                                               CNN</td>\n",
       "      <td> 2618</td>\n",
       "      <td> 1481</td>\n",
       "      <td> 1137</td>\n",
       "      <td>                                           cnn.com</td>\n",
       "      <td>                [edition.cnn.com, CNN]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Benny Morris</th>\n",
       "      <td>                                      Benny Morris</td>\n",
       "      <td> 2413</td>\n",
       "      <td> 1663</td>\n",
       "      <td>  750</td>\n",
       "      <td>                                            Morris</td>\n",
       "      <td>                        [Benny Morris]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jewish Virtual Library</th>\n",
       "      <td>                            Jewish Virtual Library</td>\n",
       "      <td> 2383</td>\n",
       "      <td> 1367</td>\n",
       "      <td> 1016</td>\n",
       "      <td>                          jewishvirtuallibrary.org</td>\n",
       "      <td>              [Jewish Virtual Library]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Al Jazeera</th>\n",
       "      <td>                                        Al Jazeera</td>\n",
       "      <td> 2280</td>\n",
       "      <td> 1378</td>\n",
       "      <td>  902</td>\n",
       "      <td>                             english.aljazeera.net</td>\n",
       "      <td>           [aljazeera.com, Al Jazeera]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Washington Post</th>\n",
       "      <td>                                   Washington Post</td>\n",
       "      <td> 1780</td>\n",
       "      <td> 1020</td>\n",
       "      <td>  760</td>\n",
       "      <td>                                washingtonpost.com</td>\n",
       "      <td>                     [Washington Post]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reuters</th>\n",
       "      <td>                                           Reuters</td>\n",
       "      <td> 1775</td>\n",
       "      <td> 1064</td>\n",
       "      <td>  711</td>\n",
       "      <td>                                       reuters.com</td>\n",
       "      <td>             [uk.reuters.com, Reuters]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Telegraph</th>\n",
       "      <td>                                     The Telegraph</td>\n",
       "      <td> 1427</td>\n",
       "      <td>  830</td>\n",
       "      <td>  597</td>\n",
       "      <td>                                   telegraph.co.uk</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Human Rights Watch</th>\n",
       "      <td>                                Human Rights Watch</td>\n",
       "      <td> 1347</td>\n",
       "      <td>  753</td>\n",
       "      <td>  594</td>\n",
       "      <td>                                           hrw.org</td>\n",
       "      <td>                  [Human Rights Watch]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yahoo News</th>\n",
       "      <td>                                        Yahoo News</td>\n",
       "      <td> 1171</td>\n",
       "      <td>  637</td>\n",
       "      <td>  534</td>\n",
       "      <td>                                    news.yahoo.com</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>google.com</th>\n",
       "      <td>                                        google.com</td>\n",
       "      <td> 1163</td>\n",
       "      <td>  674</td>\n",
       "      <td>  489</td>\n",
       "      <td>                                        google.com</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>youtube.com</th>\n",
       "      <td>                                       youtube.com</td>\n",
       "      <td> 1157</td>\n",
       "      <td>  638</td>\n",
       "      <td>  519</td>\n",
       "      <td>                                       youtube.com</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B'Tselem</th>\n",
       "      <td>                                          B'Tselem</td>\n",
       "      <td> 1140</td>\n",
       "      <td>  665</td>\n",
       "      <td>  475</td>\n",
       "      <td>                                       btselem.org</td>\n",
       "      <td>                            [B'Tselem]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Time</th>\n",
       "      <td>                                              Time</td>\n",
       "      <td> 1132</td>\n",
       "      <td>  651</td>\n",
       "      <td>  481</td>\n",
       "      <td>                                          time.com</td>\n",
       "      <td>                                [Time]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Times of London</th>\n",
       "      <td>                                   Times of London</td>\n",
       "      <td> 1093</td>\n",
       "      <td>  609</td>\n",
       "      <td>  484</td>\n",
       "      <td>                                 timesonline.co.uk</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>web.archive.org</th>\n",
       "      <td>                                   web.archive.org</td>\n",
       "      <td> 1082</td>\n",
       "      <td>  632</td>\n",
       "      <td>  450</td>\n",
       "      <td>                                   web.archive.org</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jerusalem Center for Public Affairs</th>\n",
       "      <td>               Jerusalem Center for Public Affairs</td>\n",
       "      <td> 1073</td>\n",
       "      <td>  595</td>\n",
       "      <td>  478</td>\n",
       "      <td>                                          jcpa.org</td>\n",
       "      <td> [Jerusalem Center for Public Affairs]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Arutz Sheva (Israel News)</th>\n",
       "      <td>                         Arutz Sheva (Israel News)</td>\n",
       "      <td> 1047</td>\n",
       "      <td>  627</td>\n",
       "      <td>  420</td>\n",
       "      <td>                            israelnationalnews.com</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>US State Dept</th>\n",
       "      <td>                                     US State Dept</td>\n",
       "      <td> 1005</td>\n",
       "      <td>  549</td>\n",
       "      <td>  456</td>\n",
       "      <td>                                         state.gov</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unispal.un.org</th>\n",
       "      <td>                                    unispal.un.org</td>\n",
       "      <td>  930</td>\n",
       "      <td>  610</td>\n",
       "      <td>  320</td>\n",
       "      <td>                                    unispal.un.org</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Palestine Remembered</th>\n",
       "      <td>                              Palestine Remembered</td>\n",
       "      <td>  914</td>\n",
       "      <td>  587</td>\n",
       "      <td>  327</td>\n",
       "      <td>                           palestineremembered.com</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rashid Khalidi</th>\n",
       "      <td>                                    Rashid Khalidi</td>\n",
       "      <td>  907</td>\n",
       "      <td>  672</td>\n",
       "      <td>  235</td>\n",
       "      <td>                                           Khalidi</td>\n",
       "      <td>                      [Rashid Khalidi]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Opposition claim</th>\n",
       "      <td>                                  Opposition claim</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                  Opposition claim</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Presence</th>\n",
       "      <td>                                          Presence</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                          Presence</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shmtemple.org</th>\n",
       "      <td>                                     shmtemple.org</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                     shmtemple.org</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Govrin</th>\n",
       "      <td>                                            Govrin</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                            Govrin</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tamar Levy</th>\n",
       "      <td>                                        Tamar Levy</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                        Tamar Levy</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>khalidi128</th>\n",
       "      <td>                                        khalidi128</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                        khalidi128</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Al-Taee</th>\n",
       "      <td>                                           Al-Taee</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                           Al-Taee</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fund</th>\n",
       "      <td>                                              fund</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                              fund</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>miscamble51</th>\n",
       "      <td>                                       miscamble51</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                       miscamble51</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Viktor Suvorov</th>\n",
       "      <td>                                    Viktor Suvorov</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                    Viktor Suvorov</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Korman</th>\n",
       "      <td>                                            Korman</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                            Korman</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fruchter-Ronen</th>\n",
       "      <td>                                    Fruchter-Ronen</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                    Fruchter-Ronen</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pitcher</th>\n",
       "      <td>                                           Pitcher</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                           Pitcher</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sonne Isaiah (1962) </th>\n",
       "      <td>                              Sonne Isaiah (1962) </td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                              Sonne Isaiah (1962) </td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spelling given in Morris</th>\n",
       "      <td>                          Spelling given in Morris</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                          Spelling given in Morris</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Out of the Ashes: Destruction and Reconstruction of East Timor</th>\n",
       "      <td> Out of the Ashes: Destruction and Reconstructi...</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td> Out of the Ashes: Destruction and Reconstructi...</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mati Alon</th>\n",
       "      <td>                                         Mati Alon</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                         Mati Alon</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bandy.or.jp</th>\n",
       "      <td>                                       bandy.or.jp</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                       bandy.or.jp</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th> The Weekly Standard </th>\n",
       "      <td>                              The Weekly Standard </td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                              The Weekly Standard </td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>izrus.co.il</th>\n",
       "      <td>                                       izrus.co.il</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                       izrus.co.il</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Heng</th>\n",
       "      <td>                                              Heng</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                              Heng</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Megillah (Talmud)</th>\n",
       "      <td>                                 Megillah (Talmud)</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                 Megillah (Talmud)</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Keating (2005)</th>\n",
       "      <td>                                    Keating (2005)</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                    Keating (2005)</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gail D. Hershenzon</th>\n",
       "      <td>                                Gail D. Hershenzon</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                Gail D. Hershenzon</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colour code</th>\n",
       "      <td>                                       Colour code</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                       Colour code</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yusuf Mazhar</th>\n",
       "      <td>                                      Yusuf Mazhar</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                      Yusuf Mazhar</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Theological dictionary of the Old Testament</th>\n",
       "      <td>       Theological dictionary of the Old Testament</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>       Theological dictionary of the Old Testament</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[[Thant Myint-U]]</th>\n",
       "      <td>                                 [[Thant Myint-U]]</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                 [[Thant Myint-U]]</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bardin</th>\n",
       "      <td>                                            Bardin</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                            Bardin</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gobarev pages 1-29</th>\n",
       "      <td>                                Gobarev pages 1-29</td>\n",
       "      <td>    1</td>\n",
       "      <td>    1</td>\n",
       "      <td>    0</td>\n",
       "      <td>                                Gobarev pages 1-29</td>\n",
       "      <td>                                   NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44326 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                             name  \\\n",
       "BBC                                                                                                           BBC   \n",
       "Haaretz                                                                                                   Haaretz   \n",
       "Jerusalem Post                                                                                     Jerusalem Post   \n",
       "New York Times                                                                                     New York Times   \n",
       "Ynet News                                                                                               Ynet News   \n",
       "The Guardian                                                                                         The Guardian   \n",
       "UN                                                                                                             UN   \n",
       "books.google.com                                                                                 books.google.com   \n",
       "Israel Ministry of Foreign Affairs                                             Israel Ministry of Foreign Affairs   \n",
       "CNN                                                                                                           CNN   \n",
       "Benny Morris                                                                                         Benny Morris   \n",
       "Jewish Virtual Library                                                                     Jewish Virtual Library   \n",
       "Al Jazeera                                                                                             Al Jazeera   \n",
       "Washington Post                                                                                   Washington Post   \n",
       "Reuters                                                                                                   Reuters   \n",
       "The Telegraph                                                                                       The Telegraph   \n",
       "Human Rights Watch                                                                             Human Rights Watch   \n",
       "Yahoo News                                                                                             Yahoo News   \n",
       "google.com                                                                                             google.com   \n",
       "youtube.com                                                                                           youtube.com   \n",
       "B'Tselem                                                                                                 B'Tselem   \n",
       "Time                                                                                                         Time   \n",
       "Times of London                                                                                   Times of London   \n",
       "web.archive.org                                                                                   web.archive.org   \n",
       "Jerusalem Center for Public Affairs                                           Jerusalem Center for Public Affairs   \n",
       "Arutz Sheva (Israel News)                                                               Arutz Sheva (Israel News)   \n",
       "US State Dept                                                                                       US State Dept   \n",
       "unispal.un.org                                                                                     unispal.un.org   \n",
       "Palestine Remembered                                                                         Palestine Remembered   \n",
       "Rashid Khalidi                                                                                     Rashid Khalidi   \n",
       "...                                                                                                           ...   \n",
       "Opposition claim                                                                                 Opposition claim   \n",
       "Presence                                                                                                 Presence   \n",
       "shmtemple.org                                                                                       shmtemple.org   \n",
       "Govrin                                                                                                     Govrin   \n",
       "Tamar Levy                                                                                             Tamar Levy   \n",
       "khalidi128                                                                                             khalidi128   \n",
       "Al-Taee                                                                                                   Al-Taee   \n",
       "fund                                                                                                         fund   \n",
       "miscamble51                                                                                           miscamble51   \n",
       "Viktor Suvorov                                                                                     Viktor Suvorov   \n",
       "Korman                                                                                                     Korman   \n",
       "Fruchter-Ronen                                                                                     Fruchter-Ronen   \n",
       "Pitcher                                                                                                   Pitcher   \n",
       "Sonne Isaiah (1962)                                                                          Sonne Isaiah (1962)    \n",
       "Spelling given in Morris                                                                 Spelling given in Morris   \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor  Out of the Ashes: Destruction and Reconstructi...   \n",
       "Mati Alon                                                                                               Mati Alon   \n",
       "bandy.or.jp                                                                                           bandy.or.jp   \n",
       " The Weekly Standard                                                                         The Weekly Standard    \n",
       "izrus.co.il                                                                                           izrus.co.il   \n",
       "Heng                                                                                                         Heng   \n",
       "Megillah (Talmud)                                                                               Megillah (Talmud)   \n",
       "Keating (2005)                                                                                     Keating (2005)   \n",
       "Gail D. Hershenzon                                                                             Gail D. Hershenzon   \n",
       "Colour code                                                                                           Colour code   \n",
       "Yusuf Mazhar                                                                                         Yusuf Mazhar   \n",
       "Theological dictionary of the Old Testament                           Theological dictionary of the Old Testament   \n",
       "[[Thant Myint-U]]                                                                               [[Thant Myint-U]]   \n",
       "Bardin                                                                                                     Bardin   \n",
       "Gobarev pages 1-29                                                                             Gobarev pages 1-29   \n",
       "\n",
       "                                                               total  \\\n",
       "BBC                                                             8816   \n",
       "Haaretz                                                         7959   \n",
       "Jerusalem Post                                                  6636   \n",
       "New York Times                                                  4740   \n",
       "Ynet News                                                       4710   \n",
       "The Guardian                                                    4037   \n",
       "UN                                                              3733   \n",
       "books.google.com                                                3063   \n",
       "Israel Ministry of Foreign Affairs                              2726   \n",
       "CNN                                                             2618   \n",
       "Benny Morris                                                    2413   \n",
       "Jewish Virtual Library                                          2383   \n",
       "Al Jazeera                                                      2280   \n",
       "Washington Post                                                 1780   \n",
       "Reuters                                                         1775   \n",
       "The Telegraph                                                   1427   \n",
       "Human Rights Watch                                              1347   \n",
       "Yahoo News                                                      1171   \n",
       "google.com                                                      1163   \n",
       "youtube.com                                                     1157   \n",
       "B'Tselem                                                        1140   \n",
       "Time                                                            1132   \n",
       "Times of London                                                 1093   \n",
       "web.archive.org                                                 1082   \n",
       "Jerusalem Center for Public Affairs                             1073   \n",
       "Arutz Sheva (Israel News)                                       1047   \n",
       "US State Dept                                                   1005   \n",
       "unispal.un.org                                                   930   \n",
       "Palestine Remembered                                             914   \n",
       "Rashid Khalidi                                                   907   \n",
       "...                                                              ...   \n",
       "Opposition claim                                                   1   \n",
       "Presence                                                           1   \n",
       "shmtemple.org                                                      1   \n",
       "Govrin                                                             1   \n",
       "Tamar Levy                                                         1   \n",
       "khalidi128                                                         1   \n",
       "Al-Taee                                                            1   \n",
       "fund                                                               1   \n",
       "miscamble51                                                        1   \n",
       "Viktor Suvorov                                                     1   \n",
       "Korman                                                             1   \n",
       "Fruchter-Ronen                                                     1   \n",
       "Pitcher                                                            1   \n",
       "Sonne Isaiah (1962)                                                1   \n",
       "Spelling given in Morris                                           1   \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor     1   \n",
       "Mati Alon                                                          1   \n",
       "bandy.or.jp                                                        1   \n",
       " The Weekly Standard                                               1   \n",
       "izrus.co.il                                                        1   \n",
       "Heng                                                               1   \n",
       "Megillah (Talmud)                                                  1   \n",
       "Keating (2005)                                                     1   \n",
       "Gail D. Hershenzon                                                 1   \n",
       "Colour code                                                        1   \n",
       "Yusuf Mazhar                                                       1   \n",
       "Theological dictionary of the Old Testament                        1   \n",
       "[[Thant Myint-U]]                                                  1   \n",
       "Bardin                                                             1   \n",
       "Gobarev pages 1-29                                                 1   \n",
       "\n",
       "                                                               additions  \\\n",
       "BBC                                                                 5075   \n",
       "Haaretz                                                             4948   \n",
       "Jerusalem Post                                                      4038   \n",
       "New York Times                                                      2837   \n",
       "Ynet News                                                           2924   \n",
       "The Guardian                                                        2331   \n",
       "UN                                                                  2167   \n",
       "books.google.com                                                    1764   \n",
       "Israel Ministry of Foreign Affairs                                  1613   \n",
       "CNN                                                                 1481   \n",
       "Benny Morris                                                        1663   \n",
       "Jewish Virtual Library                                              1367   \n",
       "Al Jazeera                                                          1378   \n",
       "Washington Post                                                     1020   \n",
       "Reuters                                                             1064   \n",
       "The Telegraph                                                        830   \n",
       "Human Rights Watch                                                   753   \n",
       "Yahoo News                                                           637   \n",
       "google.com                                                           674   \n",
       "youtube.com                                                          638   \n",
       "B'Tselem                                                             665   \n",
       "Time                                                                 651   \n",
       "Times of London                                                      609   \n",
       "web.archive.org                                                      632   \n",
       "Jerusalem Center for Public Affairs                                  595   \n",
       "Arutz Sheva (Israel News)                                            627   \n",
       "US State Dept                                                        549   \n",
       "unispal.un.org                                                       610   \n",
       "Palestine Remembered                                                 587   \n",
       "Rashid Khalidi                                                       672   \n",
       "...                                                                  ...   \n",
       "Opposition claim                                                       1   \n",
       "Presence                                                               1   \n",
       "shmtemple.org                                                          1   \n",
       "Govrin                                                                 1   \n",
       "Tamar Levy                                                             1   \n",
       "khalidi128                                                             1   \n",
       "Al-Taee                                                                1   \n",
       "fund                                                                   1   \n",
       "miscamble51                                                            1   \n",
       "Viktor Suvorov                                                         1   \n",
       "Korman                                                                 1   \n",
       "Fruchter-Ronen                                                         1   \n",
       "Pitcher                                                                1   \n",
       "Sonne Isaiah (1962)                                                    1   \n",
       "Spelling given in Morris                                               1   \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor         1   \n",
       "Mati Alon                                                              1   \n",
       "bandy.or.jp                                                            1   \n",
       " The Weekly Standard                                                   1   \n",
       "izrus.co.il                                                            1   \n",
       "Heng                                                                   1   \n",
       "Megillah (Talmud)                                                      1   \n",
       "Keating (2005)                                                         1   \n",
       "Gail D. Hershenzon                                                     1   \n",
       "Colour code                                                            1   \n",
       "Yusuf Mazhar                                                           1   \n",
       "Theological dictionary of the Old Testament                            1   \n",
       "[[Thant Myint-U]]                                                      1   \n",
       "Bardin                                                                 1   \n",
       "Gobarev pages 1-29                                                     1   \n",
       "\n",
       "                                                               deletions  \\\n",
       "BBC                                                                 3741   \n",
       "Haaretz                                                             3011   \n",
       "Jerusalem Post                                                      2598   \n",
       "New York Times                                                      1903   \n",
       "Ynet News                                                           1786   \n",
       "The Guardian                                                        1706   \n",
       "UN                                                                  1566   \n",
       "books.google.com                                                    1299   \n",
       "Israel Ministry of Foreign Affairs                                  1113   \n",
       "CNN                                                                 1137   \n",
       "Benny Morris                                                         750   \n",
       "Jewish Virtual Library                                              1016   \n",
       "Al Jazeera                                                           902   \n",
       "Washington Post                                                      760   \n",
       "Reuters                                                              711   \n",
       "The Telegraph                                                        597   \n",
       "Human Rights Watch                                                   594   \n",
       "Yahoo News                                                           534   \n",
       "google.com                                                           489   \n",
       "youtube.com                                                          519   \n",
       "B'Tselem                                                             475   \n",
       "Time                                                                 481   \n",
       "Times of London                                                      484   \n",
       "web.archive.org                                                      450   \n",
       "Jerusalem Center for Public Affairs                                  478   \n",
       "Arutz Sheva (Israel News)                                            420   \n",
       "US State Dept                                                        456   \n",
       "unispal.un.org                                                       320   \n",
       "Palestine Remembered                                                 327   \n",
       "Rashid Khalidi                                                       235   \n",
       "...                                                                  ...   \n",
       "Opposition claim                                                       0   \n",
       "Presence                                                               0   \n",
       "shmtemple.org                                                          0   \n",
       "Govrin                                                                 0   \n",
       "Tamar Levy                                                             0   \n",
       "khalidi128                                                             0   \n",
       "Al-Taee                                                                0   \n",
       "fund                                                                   0   \n",
       "miscamble51                                                            0   \n",
       "Viktor Suvorov                                                         0   \n",
       "Korman                                                                 0   \n",
       "Fruchter-Ronen                                                         0   \n",
       "Pitcher                                                                0   \n",
       "Sonne Isaiah (1962)                                                    0   \n",
       "Spelling given in Morris                                               0   \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor         0   \n",
       "Mati Alon                                                              0   \n",
       "bandy.or.jp                                                            0   \n",
       " The Weekly Standard                                                   0   \n",
       "izrus.co.il                                                            0   \n",
       "Heng                                                                   0   \n",
       "Megillah (Talmud)                                                      0   \n",
       "Keating (2005)                                                         0   \n",
       "Gail D. Hershenzon                                                     0   \n",
       "Colour code                                                            0   \n",
       "Yusuf Mazhar                                                           0   \n",
       "Theological dictionary of the Old Testament                            0   \n",
       "[[Thant Myint-U]]                                                      0   \n",
       "Bardin                                                                 0   \n",
       "Gobarev pages 1-29                                                     0   \n",
       "\n",
       "                                                                                                         citation  \\\n",
       "BBC                                                                                                news.bbc.co.uk   \n",
       "Haaretz                                                                                               haaretz.com   \n",
       "Jerusalem Post                                                                                          jpost.com   \n",
       "New York Times                                                                                        nytimes.com   \n",
       "Ynet News                                                                                            ynetnews.com   \n",
       "The Guardian                                                                                       guardian.co.uk   \n",
       "UN                                                                                                  domino.un.org   \n",
       "books.google.com                                                                                 books.google.com   \n",
       "Israel Ministry of Foreign Affairs                                                                     mfa.gov.il   \n",
       "CNN                                                                                                       cnn.com   \n",
       "Benny Morris                                                                                               Morris   \n",
       "Jewish Virtual Library                                                                   jewishvirtuallibrary.org   \n",
       "Al Jazeera                                                                                  english.aljazeera.net   \n",
       "Washington Post                                                                                washingtonpost.com   \n",
       "Reuters                                                                                               reuters.com   \n",
       "The Telegraph                                                                                     telegraph.co.uk   \n",
       "Human Rights Watch                                                                                        hrw.org   \n",
       "Yahoo News                                                                                         news.yahoo.com   \n",
       "google.com                                                                                             google.com   \n",
       "youtube.com                                                                                           youtube.com   \n",
       "B'Tselem                                                                                              btselem.org   \n",
       "Time                                                                                                     time.com   \n",
       "Times of London                                                                                 timesonline.co.uk   \n",
       "web.archive.org                                                                                   web.archive.org   \n",
       "Jerusalem Center for Public Affairs                                                                      jcpa.org   \n",
       "Arutz Sheva (Israel News)                                                                  israelnationalnews.com   \n",
       "US State Dept                                                                                           state.gov   \n",
       "unispal.un.org                                                                                     unispal.un.org   \n",
       "Palestine Remembered                                                                      palestineremembered.com   \n",
       "Rashid Khalidi                                                                                            Khalidi   \n",
       "...                                                                                                           ...   \n",
       "Opposition claim                                                                                 Opposition claim   \n",
       "Presence                                                                                                 Presence   \n",
       "shmtemple.org                                                                                       shmtemple.org   \n",
       "Govrin                                                                                                     Govrin   \n",
       "Tamar Levy                                                                                             Tamar Levy   \n",
       "khalidi128                                                                                             khalidi128   \n",
       "Al-Taee                                                                                                   Al-Taee   \n",
       "fund                                                                                                         fund   \n",
       "miscamble51                                                                                           miscamble51   \n",
       "Viktor Suvorov                                                                                     Viktor Suvorov   \n",
       "Korman                                                                                                     Korman   \n",
       "Fruchter-Ronen                                                                                     Fruchter-Ronen   \n",
       "Pitcher                                                                                                   Pitcher   \n",
       "Sonne Isaiah (1962)                                                                          Sonne Isaiah (1962)    \n",
       "Spelling given in Morris                                                                 Spelling given in Morris   \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor  Out of the Ashes: Destruction and Reconstructi...   \n",
       "Mati Alon                                                                                               Mati Alon   \n",
       "bandy.or.jp                                                                                           bandy.or.jp   \n",
       " The Weekly Standard                                                                         The Weekly Standard    \n",
       "izrus.co.il                                                                                           izrus.co.il   \n",
       "Heng                                                                                                         Heng   \n",
       "Megillah (Talmud)                                                                               Megillah (Talmud)   \n",
       "Keating (2005)                                                                                     Keating (2005)   \n",
       "Gail D. Hershenzon                                                                             Gail D. Hershenzon   \n",
       "Colour code                                                                                           Colour code   \n",
       "Yusuf Mazhar                                                                                         Yusuf Mazhar   \n",
       "Theological dictionary of the Old Testament                           Theological dictionary of the Old Testament   \n",
       "[[Thant Myint-U]]                                                                               [[Thant Myint-U]]   \n",
       "Bardin                                                                                                     Bardin   \n",
       "Gobarev pages 1-29                                                                             Gobarev pages 1-29   \n",
       "\n",
       "                                                                                             variants  \n",
       "BBC                                                                                  [bbc.co.uk, BBC]  \n",
       "Haaretz                                                                      [haaretz.co.il, Haaretz]  \n",
       "Jerusalem Post                                                         [fr.jpost.com, Jerusalem Post]  \n",
       "New York Times                                                    [query.nytimes.com, New York Times]  \n",
       "Ynet News                                                                                [ynet.co.il]  \n",
       "The Guardian                                                                           [The Guardian]  \n",
       "UN                                                                                       [un.org, UN]  \n",
       "books.google.com                                                                                  NaN  \n",
       "Israel Ministry of Foreign Affairs                               [Israel Ministry of Foreign Affairs]  \n",
       "CNN                                                                            [edition.cnn.com, CNN]  \n",
       "Benny Morris                                                                           [Benny Morris]  \n",
       "Jewish Virtual Library                                                       [Jewish Virtual Library]  \n",
       "Al Jazeera                                                                [aljazeera.com, Al Jazeera]  \n",
       "Washington Post                                                                     [Washington Post]  \n",
       "Reuters                                                                     [uk.reuters.com, Reuters]  \n",
       "The Telegraph                                                                                     NaN  \n",
       "Human Rights Watch                                                               [Human Rights Watch]  \n",
       "Yahoo News                                                                                        NaN  \n",
       "google.com                                                                                        NaN  \n",
       "youtube.com                                                                                       NaN  \n",
       "B'Tselem                                                                                   [B'Tselem]  \n",
       "Time                                                                                           [Time]  \n",
       "Times of London                                                                                   NaN  \n",
       "web.archive.org                                                                                   NaN  \n",
       "Jerusalem Center for Public Affairs                             [Jerusalem Center for Public Affairs]  \n",
       "Arutz Sheva (Israel News)                                                                         NaN  \n",
       "US State Dept                                                                                     NaN  \n",
       "unispal.un.org                                                                                    NaN  \n",
       "Palestine Remembered                                                                              NaN  \n",
       "Rashid Khalidi                                                                       [Rashid Khalidi]  \n",
       "...                                                                                               ...  \n",
       "Opposition claim                                                                                  NaN  \n",
       "Presence                                                                                          NaN  \n",
       "shmtemple.org                                                                                     NaN  \n",
       "Govrin                                                                                            NaN  \n",
       "Tamar Levy                                                                                        NaN  \n",
       "khalidi128                                                                                        NaN  \n",
       "Al-Taee                                                                                           NaN  \n",
       "fund                                                                                              NaN  \n",
       "miscamble51                                                                                       NaN  \n",
       "Viktor Suvorov                                                                                    NaN  \n",
       "Korman                                                                                            NaN  \n",
       "Fruchter-Ronen                                                                                    NaN  \n",
       "Pitcher                                                                                           NaN  \n",
       "Sonne Isaiah (1962)                                                                               NaN  \n",
       "Spelling given in Morris                                                                          NaN  \n",
       "Out of the Ashes: Destruction and Reconstruction of East Timor                                    NaN  \n",
       "Mati Alon                                                                                         NaN  \n",
       "bandy.or.jp                                                                                       NaN  \n",
       " The Weekly Standard                                                                              NaN  \n",
       "izrus.co.il                                                                                       NaN  \n",
       "Heng                                                                                              NaN  \n",
       "Megillah (Talmud)                                                                                 NaN  \n",
       "Keating (2005)                                                                                    NaN  \n",
       "Gail D. Hershenzon                                                                                NaN  \n",
       "Colour code                                                                                       NaN  \n",
       "Yusuf Mazhar                                                                                      NaN  \n",
       "Theological dictionary of the Old Testament                                                       NaN  \n",
       "[[Thant Myint-U]]                                                                                 NaN  \n",
       "Bardin                                                                                            NaN  \n",
       "Gobarev pages 1-29                                                                                NaN  \n",
       "\n",
       "[44326 rows x 6 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotated = annotated[['name', 'total', 'additions', 'deletions', 'citation', 'variants']]\n",
    "annotated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "annotated.to_csv('/home/michael/school/research/wp/wp_articles/citations_annotated.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>additions</th>\n",
       "      <th>citation</th>\n",
       "      <th>deletions</th>\n",
       "      <th>total</th>\n",
       "      <th>variants</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Haaretz</th>\n",
       "      <td> 55</td>\n",
       "      <td> Haaretz</td>\n",
       "      <td> 39</td>\n",
       "      <td> 94</td>\n",
       "      <td> NaN</td>\n",
       "      <td> Haaretz</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        additions citation deletions total variants     name\n",
       "Haaretz        55  Haaretz        39    94      NaN  Haaretz"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotated[annotated['name'] == 'Haaretz']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>additions</th>\n",
       "      <th>citation</th>\n",
       "      <th>deletions</th>\n",
       "      <th>total</th>\n",
       "      <th>variants</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UN</th>\n",
       "      <td> 35</td>\n",
       "      <td> UN</td>\n",
       "      <td> 25</td>\n",
       "      <td> 60</td>\n",
       "      <td> NaN</td>\n",
       "      <td> UN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   additions citation deletions total variants name\n",
       "UN        35       UN        25    60      NaN   UN"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotated[annotated['name'] == 'UN']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Print out top citations: adds, dels, citations\n",
    "with open(outpath, 'w') as f:\n",
    "    w = csv.writer(f)\n",
    "    w.writerow(['citation', 'additions', 'deletions', 'total'])\n",
    "    for c in citations.most_common():\n",
    "        name = c[0]\n",
    "        w.writerow([name, adds[name], dels.get(name, 0), citations[name]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "float"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(freq.iloc[100,:]['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing citations in /home/michael/school/research/wp/wp_articles/citations/101_squadron_(israel)_citations.csv.finished.\n",
      "Parsing citations in /home/michael/school/research/wp/wp_articles/citations/1920_nebi_musa_riots_citations.csv.finished.\n",
      "Parsing citations in /home/michael/school/research/wp/wp_articles/citations/edward_said_citations.csv.....finished.\n",
      "Parsing citations in /home/michael/school/research/wp/wp_articles/citations/second_intifada_citations.csv..........finished.\n",
      "Parsing citations in /home/michael/school/research/wp/wp_articles/citations/six-day_war_citations.csv...................finished.\n"
     ]
    }
   ],
   "source": [
    "# Aggregate added, deleted citations for each revision\n",
    "citations = Counter()\n",
    "adds = Counter()\n",
    "dels = Counter()\n",
    "citedir = '/home/michael/school/research/wp/wp_articles/citations/'\n",
    "cite_diffdir = '/home/michael/school/research/wp/wp_articles/citediffs/'\n",
    "infopath = '/home/michael/school/research/wp/wikipedia/data/citations_reverts.csv'\n",
    "fps = []\n",
    "for p in sorted(os.listdir(citedir)):\n",
    "    if not p.startswith('.'): fps.append(os.path.join(citedir, p))\n",
    "\n",
    "# Load article revision metadata\n",
    "with open(infopath) as f:\n",
    "    r = csv.DictReader(f)\n",
    "    info = {(row['article_name'], row['timestamp']): row for row in r}\n",
    "\n",
    "# Parse added, deleted citations for each revision\n",
    "#for fp in fps[1:2]:\n",
    "for fp in fps:\n",
    "    print('Parsing citations in {:s}'.format(fp), end='')\n",
    "    sys.stdout.flush()\n",
    "    with open(fp) as f:\n",
    "        rows = [row for row in csv.DictReader(f)]\n",
    "        \n",
    "    #for i, row in enumerate(rows[beg:end]):\n",
    "    prev_ts = rows[0]['timestamp']\n",
    "    prev_cites = Counter()\n",
    "    cites = Counter()\n",
    "    outrows = [['article_name', 'timestamp', 'editor', 'add/delete', 'citation', 'edit_comment', \n",
    "               'edit_length', 'revert_md5', 'revert_comment', 'vandalism_comment']]\n",
    "    for i, row in enumerate(rows):\n",
    "        if i % 50000 == 0:\n",
    "            #print(i/1000, 'k')\n",
    "            print('.', end='')\n",
    "            sys.stdout.flush()\n",
    "            \n",
    "        ts = row['timestamp']\n",
    "        ri = info[(row['article_name'], ts)]\n",
    "        \n",
    "        # Update\n",
    "        if ts != prev_ts: \n",
    "            d = prev_cites-cites\n",
    "            a = cites-prev_cites\n",
    "            for c in a:\n",
    "                outrows.append([row['article_name'], ts, ri['editor'], 'add', c, ri['edit_comment'], ri['edit_length'],\n",
    "                               ri['revert_md5'], ri['revert_comment'], ri['vandalism_comment']])\n",
    "            for c in d:\n",
    "                outrows.append([row['article_name'], ts, ri['editor'], 'delete', c, ri['edit_comment'], ri['edit_length'],\n",
    "                               ri['revert_md5'], ri['revert_comment'], ri['vandalism_comment']])\n",
    "            \n",
    "            prev_cites = cites\n",
    "            cites = Counter()\n",
    "            prev_ts = ts\n",
    "\n",
    "        # Extract cite book\n",
    "        if re.search(r'(?:c|C)ite book', row['citation']):\n",
    "            m_first = re.search(r'(?:c|C)ite book.*first=(?P<first>.*?)\\ *?\\|', row['citation'])\n",
    "            m_last = re.search(r'(?:c|C)ite book.*last=(?P<last>.*?)\\ *?\\|', row['citation'])\n",
    "            m_author = re.search(r'(?:c|C)ite book.*author=(?P<author>.*?)\\ *?\\|', row['citation'])\n",
    "            m_title = re.search(r'(?:c|C)ite book.*title=(?P<title>.*?)\\ *?\\|', row['citation'])\n",
    "            if m_first and m_last:\n",
    "                cites[' '.join([m_first.group('first'), m_last.group('last')])] += 1\n",
    "            elif m_last:\n",
    "                cites[m_last.group('last')] += 1\n",
    "            elif m_author:\n",
    "                cites[m_author.group('author')] += 1\n",
    "            elif m_title:\n",
    "                cites[m_title.group('title')] += 1\n",
    "\n",
    "        # Extract cite web\n",
    "        elif re.search(r'(?:c|C)ite ?web', row['citation']):\n",
    "            m_auth = re.search(r'(?:c|C)ite ?web.*?(?:author=(?P<author>.*?))(?:\\||$)', row['citation'])\n",
    "            m_url = re.search(r'(?:c|C)ite ?web.*?url=http:\\/\\/(?:www\\.)?(?P<url>.*?)\\/', row['citation'])\n",
    "            # r'(?:c|C)ite web.*(?:author=(?P<author>.*?)\\|.*)?.*url=http:\\/\\/(?:www\\.)?(?P<url>.*?)\\/'\n",
    "            if m_auth: \n",
    "                #cites[('web', m_auth.group('author'))] += 1\n",
    "                cites[m_auth.group('author')] += 1\n",
    "            if m_url: \n",
    "                cites[m_url.group('url')] += 1\n",
    "                #cites[('web', m_url.group('url'))] += 1\n",
    "\n",
    "        # Extract cite journal\n",
    "        elif re.search(r'(?:c|C)ite ?journal', row['citation']):\n",
    "            m_auth = re.search(r'(?:c|C)ite ?journal.*?(?:author=(?P<author>.*?))(?:\\||$)', row['citation'])\n",
    "            m_journal = re.search(r'(?:c|C)ite ?journal.*?(?:journal=(?P<journal>.*?))(?:\\||$)', row['citation'])\n",
    "            #if m_auth: cites[('journal', m_auth.group('author'))] += 1\n",
    "            if m_auth: cites[m_auth.group('author')] += 1\n",
    "            #if m_journal: cites[('journal', m_journal.group('journal'))] += 1\n",
    "            if m_journal: cites[(m_journal.group('journal'))] += 1\n",
    "\n",
    "        # Extract cite news\n",
    "        elif re.search(r'(?:c|C)ite ?news', row['citation']):\n",
    "            m_auth = re.search(r'(?:c|C)ite ?news\\*?(?:author=(?P<author>.*?)\\|.*)(?:\\||$)', row['citation'])\n",
    "            m_url = re.search(r'(?:c|C)ite ?news.*?url=http:\\/\\/(?:www\\.)?(?P<url>.*?)\\/', row['citation'])\n",
    "            #if m_auth: cites[('news', m_auth.group('author'))] += 1\n",
    "            #if m_url: cites[('news', m_url.group('url'))] += 1\n",
    "            #if m_url: citations.append(('web', m_url.group('url')))\n",
    "            if m_auth: cites[m_auth.group('author')] += 1\n",
    "            if m_url: cites[m_url.group('url')] += 1\n",
    "\n",
    "        # Extract just url\n",
    "        elif re.search(r'http:\\/\\/(?:www\\.)?(?P<url>.*?)[ \\/]', row['citation']):\n",
    "            c = re.search(r'http:\\/\\/(?:www\\.)?(?P<url>.*?)[ \\/]', row['citation']).group('url')\n",
    "            #citations.append((\"url\", c))\n",
    "            cites[c] += 1\n",
    "            \n",
    "        # Extract harvnb\n",
    "        elif re.search(r'[Hh]arvnb\\|(.*?)\\|', row['citation']):\n",
    "            c = re.search(r'[Hh]arvnb\\|(.*?)\\|', row['citation']).group(1)\n",
    "            cites[c] += 1\n",
    "\n",
    "        # Extract harvtxt\n",
    "        elif re.search(r'[Hh]arvtxt\\|(.*?)\\|', row['citation']):\n",
    "            c = re.search(r'[Hh]arvtxt\\|(.*?)\\|', row['citation']).group(1)\n",
    "            cites[c] += 1\n",
    "\n",
    "        else: # Default\n",
    "            m = re.search(r'\\ *(?:\"|\\'\\'|\\[\\[)?(.+?)(?:\\.|,|\"|“|\\'|\\||\\])', row['citation'])\n",
    "            if m:\n",
    "                c = m.group(1)\n",
    "                if 'http' in c:\n",
    "                    debug_here()\n",
    "                #print(beg+i+2, 'default', c)\n",
    "                #citations.append(('unknown', c))\n",
    "                cites[c] += 1\n",
    "            #else:\n",
    "            #    print(\"no parse\")\n",
    "            \n",
    "    # Write citation diffs\n",
    "    with open(os.path.join(cite_diffdir, os.path.split(fp)[1][:-4]) + '_citediff.csv', 'w') as f:\n",
    "        w = csv.writer(f)\n",
    "        w.writerows(outrows)\n",
    "            \n",
    "    print(\"finished\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Print out citations\n",
    "with open(outpath, 'w') as f:\n",
    "    w = csv.writer(f)\n",
    "    w.writerow(['citation', 'additions', 'deletions', 'total'])\n",
    "    for c in citations.most_common():\n",
    "        name = c[0]\n",
    "        w.writerow([name, adds[name], dels.get(name, 0), citations[name]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing citations in /home/michael/school/research/wp/wp_articles/citations/101_squadron_(israel)_citations.csv\n",
      ".finished.\n",
      "Parsing citations in /home/michael/school/research/wp/wp_articles/citations/1920_nebi_musa_riots_citations.csv\n",
      ".finished.\n",
      "Parsing citations in /home/michael/school/research/wp/wp_articles/citations/edward_said_citations.csv\n",
      ".....finished.\n",
      "Parsing citations in /home/michael/school/research/wp/wp_articles/citations/second_intifada_citations.csv\n",
      "..........finished.\n",
      "Parsing citations in /home/michael/school/research/wp/wp_articles/citations/six-day_war_citations.csv\n",
      "...................finished.\n",
      "979\n",
      "789\n",
      "979\n"
     ]
    }
   ],
   "source": [
    "# Aggregate citations for top add/del\n",
    "#for fp in fps[1:2]:\n",
    "for fp in fps:\n",
    "    print('Parsing citations in {:s}'.format(fp), end='')\n",
    "    sys.stdout.flush()\n",
    "    with open(fp) as f:\n",
    "        rows = [row for row in csv.DictReader(f)]\n",
    "        \n",
    "    #for i, row in enumerate(rows[beg:end]):\n",
    "    prev_ts = rows[0]['timestamp']\n",
    "    prev_cites = Counter()\n",
    "    cites = Counter()\n",
    "    for i, row in enumerate(rows):\n",
    "        if i % 50000 == 0:\n",
    "            #print(i/1000, 'k')\n",
    "            print('.', end='')\n",
    "            sys.stdout.flush()\n",
    "            \n",
    "        ts = row['timestamp']\n",
    "        \n",
    "        # Update\n",
    "        if ts != prev_ts: \n",
    "            d = prev_cites-cites\n",
    "            a = cites-prev_cites\n",
    "            for c in a:\n",
    "                adds[c] += a[c]\n",
    "                citations[c] += a[c]\n",
    "            for c in d:\n",
    "                dels[c] += d[c]\n",
    "                citations[c] += d[c]\n",
    "            \n",
    "            prev_cites = cites\n",
    "            cites = Counter()\n",
    "            prev_ts = ts\n",
    "\n",
    "        # Extract cite book\n",
    "        if re.search(r'(?:c|C)ite book', row['citation']):\n",
    "            m_first = re.search(r'(?:c|C)ite book.*first=(?P<first>.*?)\\ *?\\|', row['citation'])\n",
    "            m_last = re.search(r'(?:c|C)ite book.*last=(?P<last>.*?)\\ *?\\|', row['citation'])\n",
    "            m_author = re.search(r'(?:c|C)ite book.*author=(?P<author>.*?)\\ *?\\|', row['citation'])\n",
    "            m_title = re.search(r'(?:c|C)ite book.*title=(?P<title>.*?)\\ *?\\|', row['citation'])\n",
    "            if m_first and m_last:\n",
    "                cites[' '.join([m_first.group('first'), m_last.group('last')])] += 1\n",
    "            elif m_last:\n",
    "                cites[m_last.group('last')] += 1\n",
    "            elif m_author:\n",
    "                cites[m_author.group('author')] += 1\n",
    "            elif m_title:\n",
    "                cites[m_title.group('title')] += 1\n",
    "\n",
    "        # Extract cite web\n",
    "        elif re.search(r'(?:c|C)ite ?web', row['citation']):\n",
    "            m_auth = re.search(r'(?:c|C)ite ?web.*?(?:author=(?P<author>.*?))(?:\\||$)', row['citation'])\n",
    "            m_url = re.search(r'(?:c|C)ite ?web.*?url=http:\\/\\/(?:www\\.)?(?P<url>.*?)\\/', row['citation'])\n",
    "            # r'(?:c|C)ite web.*(?:author=(?P<author>.*?)\\|.*)?.*url=http:\\/\\/(?:www\\.)?(?P<url>.*?)\\/'\n",
    "            if m_auth: \n",
    "                #cites[('web', m_auth.group('author'))] += 1\n",
    "                cites[m_auth.group('author')] += 1\n",
    "            if m_url: \n",
    "                cites[m_url.group('url')] += 1\n",
    "                #cites[('web', m_url.group('url'))] += 1\n",
    "\n",
    "        # Extract cite journal\n",
    "        elif re.search(r'(?:c|C)ite ?journal', row['citation']):\n",
    "            m_auth = re.search(r'(?:c|C)ite ?journal.*?(?:author=(?P<author>.*?))(?:\\||$)', row['citation'])\n",
    "            m_journal = re.search(r'(?:c|C)ite ?journal.*?(?:journal=(?P<journal>.*?))(?:\\||$)', row['citation'])\n",
    "            #if m_auth: cites[('journal', m_auth.group('author'))] += 1\n",
    "            if m_auth: cites[m_auth.group('author')] += 1\n",
    "            #if m_journal: cites[('journal', m_journal.group('journal'))] += 1\n",
    "            if m_journal: cites[(m_journal.group('journal'))] += 1\n",
    "\n",
    "        # Extract cite news\n",
    "        elif re.search(r'(?:c|C)ite ?news', row['citation']):\n",
    "            m_auth = re.search(r'(?:c|C)ite ?news\\*?(?:author=(?P<author>.*?)\\|.*)(?:\\||$)', row['citation'])\n",
    "            m_url = re.search(r'(?:c|C)ite ?news.*?url=http:\\/\\/(?:www\\.)?(?P<url>.*?)\\/', row['citation'])\n",
    "            #if m_auth: cites[('news', m_auth.group('author'))] += 1\n",
    "            #if m_url: cites[('news', m_url.group('url'))] += 1\n",
    "            #if m_url: citations.append(('web', m_url.group('url')))\n",
    "            if m_auth: cites[m_auth.group('author')] += 1\n",
    "            if m_url: cites[m_url.group('url')] += 1\n",
    "\n",
    "        # Extract just url\n",
    "        elif re.search(r'http:\\/\\/(?:www\\.)?(?P<url>.*?)[ \\/]', row['citation']):\n",
    "            c = re.search(r'http:\\/\\/(?:www\\.)?(?P<url>.*?)[ \\/]', row['citation']).group('url')\n",
    "            #citations.append((\"url\", c))\n",
    "            cites[c] += 1\n",
    "            \n",
    "        # Extract harvnb\n",
    "        elif re.search(r'[Hh]arvnb\\|(.*?)\\|', row['citation']):\n",
    "            c = re.search(r'[Hh]arvnb\\|(.*?)\\|', row['citation']).group(1)\n",
    "            cites[c] += 1\n",
    "\n",
    "        # Extract harvtxt\n",
    "        elif re.search(r'[Hh]arvtxt\\|(.*?)\\|', row['citation']):\n",
    "            c = re.search(r'[Hh]arvtxt\\|(.*?)\\|', row['citation']).group(1)\n",
    "            cites[c] += 1\n",
    "\n",
    "        else: # Default\n",
    "            m = re.search(r'\\ *(?:\"|\\'\\'|\\[\\[)?(.+?)(?:\\.|,|\"|“|\\'|\\||\\])', row['citation'])\n",
    "            if m:\n",
    "                c = m.group(1)\n",
    "                if 'http' in c:\n",
    "                    debug_here()\n",
    "                #print(beg+i+2, 'default', c)\n",
    "                #citations.append(('unknown', c))\n",
    "                cites[c] += 1\n",
    "            #else:\n",
    "            #    print(\"no parse\")\n",
    "    print(\"finished.\")\n",
    "\n",
    "print(len(adds))\n",
    "print(len(dels))\n",
    "print(len(citations))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('web', 'news.bbc.co.uk'), 1129311),\n",
       " (('unknown', '{{harvnb'), 1060527),\n",
       " (('url', 'news.bbc.co.uk'), 943174),\n",
       " (('web', 'haaretz.com'), 915186),\n",
       " (('url', 'haaretz.com'), 873409),\n",
       " (('web', 'nytimes.com'), 801670),\n",
       " (('web', 'ynetnews.com'), 748725),\n",
       " (('web', 'guardian.co.uk'), 631914),\n",
       " (('url', 'jpost.com'), 618700),\n",
       " (('web', 'jpost.com'), 614003),\n",
       " (('unknown', '{{Harvnb'), 576263),\n",
       " (('web', 'english.aljazeera.net'), 506626),\n",
       " (('url', 'nytimes.com'), 461524),\n",
       " (('url', 'ynetnews.com'), 438354),\n",
       " (('url', 'books.google.com'), 433677),\n",
       " (('web', 'washingtonpost.com'), 389789),\n",
       " (('url', 'guardian.co.uk'), 388590),\n",
       " (('url', 'mfa.gov.il'), 377654),\n",
       " (('web', 'reuters.com'), 374264),\n",
       " (('web', 'bbc.co.uk'), 372687),\n",
       " (('web', 'cnn.com'), 331145),\n",
       " (('unknown', 'Varble'), 314212),\n",
       " (('unknown', 'Morris'), 252079),\n",
       " (('url', 'jewishvirtuallibrary.org'), 250487),\n",
       " (('web', 'edition.cnn.com'), 218842),\n",
       " (('url', 'un.org'), 218764),\n",
       " (('web', 'telegraph.co.uk'), 214839),\n",
       " (('web', 'time.com'), 211390),\n",
       " (('url', 'domino.un.org'), 210401),\n",
       " (('web', 'un.org'), 207118),\n",
       " (('unknown', 'Rabinovich'), 197575),\n",
       " (('url', 'cnn.com'), 196417),\n",
       " (('url', 'hrw.org'), 195573),\n",
       " (('web', 'google.com'), 195484),\n",
       " (('url', 'washingtonpost.com'), 184003),\n",
       " (('url', 'english.aljazeera.net'), 172831),\n",
       " (('web', 'mfa.gov.il'), 167127),\n",
       " (('url', 'web.archive.org'), 161901),\n",
       " (('web', 'timesonline.co.uk'), 155653),\n",
       " (('url', 'time.com'), 152547),\n",
       " (('web', 'msnbc.msn.com'), 148556),\n",
       " (('web', 'jewishvirtuallibrary.org'), 145946),\n",
       " (('url', 'reuters.com'), 143900),\n",
       " (('url', 'news.yahoo.com'), 143331),\n",
       " (('unknown', 'Economist'), 138509),\n",
       " (('url', 'telegraph.co.uk'), 138456),\n",
       " (('url', 'state.gov'), 138425),\n",
       " (('web', 'news.yahoo.com'), 138037),\n",
       " (('web', 'iht.com'), 137200),\n",
       " (('url', 'timesonline.co.uk'), 136228),\n",
       " (('web', 'bloomberg.com'), 132858),\n",
       " (('web', 'hrw.org'), 132446),\n",
       " (('unknown', 'autogenerated1'), 131399),\n",
       " (('web', 'foxnews.com'), 131060),\n",
       " (('url', 'bbc.co.uk'), 125477),\n",
       " (('web', 'obama.senate.gov'), 124727),\n",
       " (('web', 'dailystar.com.lb'), 123861),\n",
       " (('unknown', 'unesco'), 121944),\n",
       " (('url', 'google.com'), 121429),\n",
       " (('web', ''), 121185),\n",
       " (('url', 'edition.cnn.com'), 119896),\n",
       " (('web', 'independent.co.uk'), 113509),\n",
       " (('unknown', 'Khalidi'), 113443),\n",
       " (('web', 'chicagotribune.com'), 113227),\n",
       " (('url', 'www1.yadvashem.org'), 111240),\n",
       " (('url', 'youtube.com'), 110294),\n",
       " (('url', 'btselem.org'), 109791),\n",
       " (('web', 'online.wsj.com'), 105398),\n",
       " (('url', 'fr.jpost.com'), 103770),\n",
       " (('web', 'query.nytimes.com'), 103421),\n",
       " (('url', 'books.google.co.uk'), 102331),\n",
       " (('url', 'jcpa.org'), 100099),\n",
       " (('url', 'msnbc.msn.com'), 99764),\n",
       " (('web', 'state.gov'), 99630),\n",
       " (('web', 'cbsnews.com'), 98110),\n",
       " (('url', 'iht.com'), 97617),\n",
       " (('web', 'abcnews.go.com'), 97244),\n",
       " (('unknown', 'Benny Morris'), 96346),\n",
       " (('unknown', 'ocha_report_1_2'), 95993),\n",
       " (('web', 'voanews.com'), 95305),\n",
       " (('url', 'books.google.ca'), 95113),\n",
       " (('web', 'news.xinhuanet.com'), 94516),\n",
       " (('web', 'usatoday.com'), 93910),\n",
       " (('web', 'af.reuters.com'), 93190),\n",
       " (('book', ' Spencer C.  Tucker'), 92918),\n",
       " (('web', 'fr.jpost.com'), 92636),\n",
       " (('web', 'pqasb.pqarchiver.com'), 92573),\n",
       " (('unknown', 'Pollack'), 89067),\n",
       " (('web', 'youtube.com'), 88667),\n",
       " (('url', 'britannica.com'), 88665),\n",
       " (('unknown', 'cia'), 87203),\n",
       " (('web', 'latimes.com'), 86836),\n",
       " (('web', 'maannews.net'), 85855),\n",
       " (('web', 'huffingtonpost.com'), 85510),\n",
       " (('unknown', 'Gordon 2008'), 84747),\n",
       " (('web', 'aljazeera.com'), 84191),\n",
       " (('web', 'webcitation.org'), 83837),\n",
       " (('url', 'globalsecurity.org'), 82919),\n",
       " (('web', 'csmonitor.com'), 82018),\n",
       " (('url', 'ynet.co.il'), 80608)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scites = sorted(cites.items(), key=itemgetter(1), reverse=True)\n",
    "scites[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 M\n",
      "1000000 M\n",
      "2000000 M\n",
      "3000000 M\n",
      "4000000 M\n",
      "5000000 M\n",
      "6000000 M\n",
      "7000000 M\n",
      "8000000 M\n",
      "9000000 M\n",
      "10000000 M\n",
      "11000000 M\n",
      "12000000 M\n",
      "13000000 M\n",
      "14000000 M\n",
      "15000000 M\n",
      "16000000 M\n",
      "17000000 M\n",
      "18000000 M\n",
      "19000000 M\n",
      "20000000 M\n",
      "21000000 M\n",
      "22000000 M\n",
      "23000000 M\n",
      "24000000 M\n",
      "25000000 M\n",
      "26000000 M\n",
      "27000000 M\n",
      "28000000 M\n",
      "29000000 M\n",
      "30000000 M\n",
      "31000000 M\n",
      "32000000 M\n",
      "33000000 M\n",
      "34000000 M\n",
      "35000000 M\n",
      "36000000 M\n",
      "37000000 M\n",
      "38000000 M\n",
      "39000000 M\n",
      "40000000 M\n",
      "41000000 M\n",
      "42000000 M\n",
      "43000000 M\n",
      "44000000 M\n",
      "45000000 M\n",
      "46000000 M\n",
      "47000000 M\n",
      "48000000 M\n",
      "49000000 M\n",
      "50000000 M\n",
      "51000000 M\n",
      "52000000 M\n",
      "53000000 M\n",
      "54000000 M\n",
      "55000000 M\n",
      "56000000 M\n",
      "57000000 M\n",
      "58000000 M\n",
      "59000000 M\n",
      "60000000 M\n",
      "61000000 M\n",
      "62000000 M\n",
      "63000000 M\n",
      "64000000 M\n",
      "65000000 M\n",
      "66000000 M\n",
      "67000000 M\n",
      "68000000 M\n",
      "69000000 M\n",
      "70000000 M\n",
      "71000000 M\n",
      "72000000 M\n",
      "73000000 M\n",
      "74000000 M\n",
      "75000000 M\n",
      "76000000 M\n",
      "77000000 M\n",
      "78000000 M\n",
      "79000000 M\n",
      "80000000 M\n",
      "81000000 M\n",
      "82000000 M\n",
      "83000000 M\n",
      "84000000 M\n",
      "85000000 M\n",
      "86000000 M\n",
      "87000000 M\n",
      "88000000 M\n",
      "89000000 M\n",
      "90000000 M\n",
      "91000000 M\n",
      "92000000 M\n",
      "93000000 M\n",
      "94000000 M\n",
      "95000000 M\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "54545"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get citation counts from massive csv\n",
    "cites = defaultdict(int)\n",
    "cf = '/home/michael/school/research/wp/wp_articles/citations.csv'\n",
    "with open(cf) as f:\n",
    "    r = csv.reader(f)\n",
    "    for i, row in enumerate(r):\n",
    "        if i % 1e6 == 0:\n",
    "            print(i/1e6, 'M')\n",
    "        cites[(row[0], row[1])] += 1\n",
    "        \n",
    "len(cites)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Print out sorted list of sources\n",
    "scites_path = '/home/michael/school/research/wp/wp_articles/citation_counts.csv'\n",
    "with open(scites_path, 'w') as f:\n",
    "    w = csv.writer(f)\n",
    "    w.writerow(['type', 'source', 'count'])\n",
    "    for typ, count in scites:\n",
    "        w.writerow([typ[0], typ[1], count])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting to parse citations for /home/michael/school/research/wp/wp_articles/citations/101_squadron_(israel)_citations.csv\n",
      "0.0 k\n",
      "Finished parsing\n",
      "\n",
      "Starting to parse citations for /home/michael/school/research/wp/wp_articles/citations/edward_said_citations.csv\n",
      "0.0 k\n",
      "50.0 k\n",
      "100.0 k\n",
      "150.0 k\n",
      "200.0 k\n",
      "Finished parsing\n",
      "\n",
      "Starting to parse citations for /home/michael/school/research/wp/wp_articles/citations/second_intifada_citations.csv\n",
      "0.0 k\n",
      "50.0 k\n",
      "100.0 k\n",
      "150.0 k\n",
      "200.0 k\n",
      "250.0 k\n",
      "300.0 k\n",
      "350.0 k\n",
      "400.0 k\n",
      "450.0 k\n",
      "Finished parsing\n",
      "\n",
      "Starting to parse citations for /home/michael/school/research/wp/wp_articles/citations/six-day_war_citations.csv\n",
      "0.0 k\n",
      "50.0 k\n",
      "100.0 k\n",
      "150.0 k\n",
      "200.0 k\n",
      "250.0 k\n",
      "300.0 k\n",
      "350.0 k\n",
      "400.0 k\n",
      "450.0 k\n",
      "500.0 k\n",
      "550.0 k\n",
      "600.0 k\n",
      "650.0 k\n",
      "700.0 k\n",
      "750.0 k\n",
      "800.0 k\n",
      "850.0 k\n",
      "900.0 k\n",
      "Wrote rows to csv\n",
      "Finished parsing\n",
      "\n"
     ]
    }
   ],
   "source": [
    "    if len(citations) > 1e6:\n",
    "        with open(outpath, 'a') as f:\n",
    "            w = csv.writer(f)\n",
    "            w.writerows(citations)\n",
    "        print(\"Wrote rows to csv\")\n",
    "        del(citations)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}