elastic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'version': {'build_timestamp': '2015-04-27T09:21:06Z', 'number': '1.5.2', 'build_hash': '62ff9868b4c8a0c45860bebb259e21980778ab1c', 'lucene_version': '4.10.4', 'build_snapshot': False}, 'tagline': 'You Know, for Search', 'name': 'Crimson and the Raven', 'status': 200, 'cluster_name': '275592502545:trial'}\n"
     ]
    }
   ],
   "source": [
    "from elasticsearch import Elasticsearch, RequestsHttpConnection\n",
    "from requests_aws4auth import AWS4Auth\n",
    "\n",
    "host = 'search-trial-cc4abhfofwbjogy5nla5sndexq.us-west-2.es.amazonaws.com'\n",
    "awsauth = AWS4Auth('AKIAJQ5JJZM5HDDOMTNQ', 'uVqOuxD+e/iaLkfusHi4TgO1wqrSdUsz2I+VLoAS', 'us-west-2', 'es')\n",
    "\n",
    "es = Elasticsearch(\n",
    "    hosts=[{'host': host, 'port': 443}],\n",
    "    http_auth=awsauth,\n",
    "    use_ssl=True,\n",
    "    verify_certs=True,\n",
    "    connection_class=RequestsHttpConnection\n",
    ")\n",
    "print(es.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PS DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "##PS_DF CSV\n",
    "import csv\n",
    "import urllib3\n",
    "f = open(raw_data_path+csv_filename)\n",
    "csv_file_object = csv.reader(f)\n",
    "    \n",
    "header = list(df.columns)\n",
    "header = [item.lower() for item in header]\n",
    "INDEX_NAME = 'weiboscope'\n",
    "TYPE_NAME = 'tweet'\n",
    "\n",
    "bulk_data = [] \n",
    "\n",
    "for row in df.iterrows():\n",
    "    data_dict = {}\n",
    "    for i in range(len(row[1])):\n",
    "        data_dict[header[i]] = row[1][i]\n",
    "    op_dict = {\n",
    "        \"index\": {\n",
    "            \"_index\": INDEX_NAME, \n",
    "            \"_type\": TYPE_NAME, \n",
    "#             \"_id\": data_dict[ID_FIELD]\n",
    "        }\n",
    "    }\n",
    "    bulk_data.append(op_dict)\n",
    "    bulk_data.append(data_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from elasticsearch import helpers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "ConnectionError",
     "evalue": "ConnectionError(('Connection aborted.', timeout('The write operation timed out',))) caused by: ConnectionError(('Connection aborted.', timeout('The write operation timed out',)))",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mtimeout\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, **response_kw)\u001b[0m\n\u001b[0;32m    577\u001b[0m                                                   \u001b[0mbody\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 578\u001b[1;33m                                                   chunked=chunked)\n\u001b[0m\u001b[0;32m    579\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[1;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[0;32m    361\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 362\u001b[1;33m             \u001b[0mconn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mhttplib_request_kw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    363\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\http\\client.py\u001b[0m in \u001b[0;36mrequest\u001b[1;34m(self, method, url, body, headers)\u001b[0m\n\u001b[0;32m   1082\u001b[0m         \u001b[1;34m\"\"\"Send a complete request to the server.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1083\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_send_request\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1084\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\http\\client.py\u001b[0m in \u001b[0;36m_send_request\u001b[1;34m(self, method, url, body, headers)\u001b[0m\n\u001b[0;32m   1127\u001b[0m             \u001b[0mbody\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'iso-8859-1'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1128\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mendheaders\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbody\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\http\\client.py\u001b[0m in \u001b[0;36mendheaders\u001b[1;34m(self, message_body)\u001b[0m\n\u001b[0;32m   1078\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mCannotSendHeader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1079\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_send_output\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage_body\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1080\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\http\\client.py\u001b[0m in \u001b[0;36m_send_output\u001b[1;34m(self, message_body)\u001b[0m\n\u001b[0;32m    912\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mmessage_body\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 913\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage_body\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    914\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\http\\client.py\u001b[0m in \u001b[0;36msend\u001b[1;34m(self, data)\u001b[0m\n\u001b[0;32m    884\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 885\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msock\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msendall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    886\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\ssl.py\u001b[0m in \u001b[0;36msendall\u001b[1;34m(self, data, flags)\u001b[0m\n\u001b[0;32m    885\u001b[0m             \u001b[1;32mwhile\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mcount\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0mamount\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 886\u001b[1;33m                 \u001b[0mv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcount\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    887\u001b[0m                 \u001b[0mcount\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mv\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\ssl.py\u001b[0m in \u001b[0;36msend\u001b[1;34m(self, data, flags)\u001b[0m\n\u001b[0;32m    855\u001b[0m                     self.__class__)\n\u001b[1;32m--> 856\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    857\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\ssl.py\u001b[0m in \u001b[0;36mwrite\u001b[1;34m(self, data)\u001b[0m\n\u001b[0;32m    580\u001b[0m         \"\"\"\n\u001b[1;32m--> 581\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    582\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mtimeout\u001b[0m: The write operation timed out",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mProtocolError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\requests\\adapters.py\u001b[0m in \u001b[0;36msend\u001b[1;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[0;32m    402\u001b[0m                     \u001b[0mretries\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 403\u001b[1;33m                     \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    404\u001b[0m                 )\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, **response_kw)\u001b[0m\n\u001b[0;32m    622\u001b[0m             retries = retries.increment(method, url, error=e, _pool=self,\n\u001b[1;32m--> 623\u001b[1;33m                                         _stacktrace=sys.exc_info()[2])\n\u001b[0m\u001b[0;32m    624\u001b[0m             \u001b[0mretries\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\util\\retry.py\u001b[0m in \u001b[0;36mincrement\u001b[1;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[0;32m    254\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mread\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 255\u001b[1;33m                 \u001b[1;32mraise\u001b[0m \u001b[0msix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merror\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_stacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    256\u001b[0m             \u001b[1;32melif\u001b[0m \u001b[0mread\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\packages\\six.py\u001b[0m in \u001b[0;36mreraise\u001b[1;34m(tp, value, tb)\u001b[0m\n\u001b[0;32m    308\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 309\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    310\u001b[0m         \u001b[1;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, **response_kw)\u001b[0m\n\u001b[0;32m    577\u001b[0m                                                   \u001b[0mbody\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 578\u001b[1;33m                                                   chunked=chunked)\n\u001b[0m\u001b[0;32m    579\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[1;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[0;32m    361\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 362\u001b[1;33m             \u001b[0mconn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mhttplib_request_kw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    363\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\http\\client.py\u001b[0m in \u001b[0;36mrequest\u001b[1;34m(self, method, url, body, headers)\u001b[0m\n\u001b[0;32m   1082\u001b[0m         \u001b[1;34m\"\"\"Send a complete request to the server.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1083\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_send_request\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1084\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\http\\client.py\u001b[0m in \u001b[0;36m_send_request\u001b[1;34m(self, method, url, body, headers)\u001b[0m\n\u001b[0;32m   1127\u001b[0m             \u001b[0mbody\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'iso-8859-1'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1128\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mendheaders\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbody\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\http\\client.py\u001b[0m in \u001b[0;36mendheaders\u001b[1;34m(self, message_body)\u001b[0m\n\u001b[0;32m   1078\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mCannotSendHeader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1079\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_send_output\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage_body\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1080\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\http\\client.py\u001b[0m in \u001b[0;36m_send_output\u001b[1;34m(self, message_body)\u001b[0m\n\u001b[0;32m    912\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mmessage_body\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 913\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage_body\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    914\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\http\\client.py\u001b[0m in \u001b[0;36msend\u001b[1;34m(self, data)\u001b[0m\n\u001b[0;32m    884\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 885\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msock\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msendall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    886\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\ssl.py\u001b[0m in \u001b[0;36msendall\u001b[1;34m(self, data, flags)\u001b[0m\n\u001b[0;32m    885\u001b[0m             \u001b[1;32mwhile\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mcount\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0mamount\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 886\u001b[1;33m                 \u001b[0mv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcount\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    887\u001b[0m                 \u001b[0mcount\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mv\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\ssl.py\u001b[0m in \u001b[0;36msend\u001b[1;34m(self, data, flags)\u001b[0m\n\u001b[0;32m    855\u001b[0m                     self.__class__)\n\u001b[1;32m--> 856\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    857\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\ssl.py\u001b[0m in \u001b[0;36mwrite\u001b[1;34m(self, data)\u001b[0m\n\u001b[0;32m    580\u001b[0m         \"\"\"\n\u001b[1;32m--> 581\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    582\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mProtocolError\u001b[0m: ('Connection aborted.', timeout('The write operation timed out',))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mConnectionError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\elasticsearch\\connection\\http_requests.py\u001b[0m in \u001b[0;36mperform_request\u001b[1;34m(self, method, url, params, body, timeout, ignore)\u001b[0m\n\u001b[0;32m     67\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mrequests\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTimeout\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 68\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog_request_fail\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mstart\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexception\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     69\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mConnectionTimeout\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'TIMEOUT'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\requests\\sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[1;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[0;32m    474\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 475\u001b[1;33m         \u001b[0mresp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    476\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\requests\\sessions.py\u001b[0m in \u001b[0;36msend\u001b[1;34m(self, request, **kwargs)\u001b[0m\n\u001b[0;32m    584\u001b[0m         \u001b[1;31m# Send the request\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 585\u001b[1;33m         \u001b[0mr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    586\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\requests\\adapters.py\u001b[0m in \u001b[0;36msend\u001b[1;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[0;32m    452\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mProtocolError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msocket\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 453\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    454\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mConnectionError\u001b[0m: ('Connection aborted.', timeout('The write operation timed out',))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mConnectionError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-190-c8ce606b5784>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindices\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mINDEX_NAME\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[0mes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbulk\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mINDEX_NAME\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbody\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbulk_data\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1000\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\elasticsearch\\client\\utils.py\u001b[0m in \u001b[0;36m_wrapped\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     67\u001b[0m                     \u001b[0mparams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     68\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 69\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_wrapped\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     70\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0m_wrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     71\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\elasticsearch\\client\\__init__.py\u001b[0m in \u001b[0;36mbulk\u001b[1;34m(self, body, index, doc_type, params)\u001b[0m\n\u001b[0;32m    793\u001b[0m         \u001b[1;33m:\u001b[0m\u001b[0marg\u001b[0m \u001b[0msource\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mThe\u001b[0m \u001b[0mURL\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mencoded\u001b[0m \u001b[0mrequest\u001b[0m \u001b[0mdefinition\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0minstead\u001b[0m \u001b[0mof\u001b[0m \u001b[0musing\u001b[0m \u001b[0mrequest\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    794\u001b[0m         \"\"\"\n\u001b[1;32m--> 795\u001b[1;33m         \u001b[1;32mif\u001b[0m \u001b[0mbody\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mSKIP_IN_PATH\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    796\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Empty value passed for a required argument 'body'.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    797\u001b[0m         _, data = self.transport.perform_request('POST', _make_path(index, '_suggest'),\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\elasticsearch\\transport.py\u001b[0m in \u001b[0;36mperform_request\u001b[1;34m(self, method, url, params, body)\u001b[0m\n\u001b[0;32m    327\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    328\u001b[0m                 \u001b[1;31m# connection didn't fail, confirm it's live status\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 329\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconnection_pool\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmark_live\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconnection\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    330\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    331\u001b[0m                     \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdeserializer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'content-type'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\kennd\\Anaconda3\\lib\\site-packages\\elasticsearch\\connection\\http_requests.py\u001b[0m in \u001b[0;36mperform_request\u001b[1;34m(self, method, url, params, body, timeout, ignore)\u001b[0m\n\u001b[0;32m     77\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_raise_error\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus_code\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mraw_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     78\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 79\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog_request_success\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath_url\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus_code\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mraw_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mduration\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     80\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     81\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus_code\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mheaders\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mraw_data\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mConnectionError\u001b[0m: ConnectionError(('Connection aborted.', timeout('The write operation timed out',))) caused by: ConnectionError(('Connection aborted.', timeout('The write operation timed out',)))"
     ]
    }
   ],
   "source": [
    "try :\n",
    "    es.indices.delete(index=INDEX_NAME)\n",
    "except :\n",
    "    pass\n",
    "\n",
    "es.indices.create(index=INDEX_NAME)\n",
    "\n",
    "es.bulk(index = INDEX_NAME, body = bulk_data,timeout=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bulk = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bulk.append((op_dict,data_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_json('C:/Users/kennd/Documents/GitHub/kcs_app/ps_df.json').reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PS DF Bulk Helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_json('C:/Users/kennd/Documents/GitHub/kcs_app/processed_df.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df2 = df.iloc[0:1000,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "##PS_DF CSV\n",
    "    \n",
    "header = list(df.columns)\n",
    "header = [item.lower() for item in header]\n",
    "INDEX_NAME = 'weiboscope'\n",
    "TYPE_NAME = 'tweet'\n",
    "\n",
    "bulk_data = [] \n",
    "index = 0\n",
    "for row in df.iterrows():\n",
    "    index += 1\n",
    "    data_dict = {}\n",
    "    for i in range(len(row[1])):\n",
    "        data_dict[header[i]] = row[1][i]\n",
    "    op_dict = {\n",
    "            \"_index\": INDEX_NAME, \n",
    "            \"_type\": TYPE_NAME, \n",
    "            \"_id\": index,\n",
    "            \"_source\": data_dict\n",
    "        }\n",
    "    bulk_data.append(op_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10411, [])"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "try :\n",
    "    es.indices.delete(index=INDEX_NAME)\n",
    "except :\n",
    "    pass\n",
    "\n",
    "es.indices.create(index=INDEX_NAME)\n",
    "\n",
    "helpers.bulk(es,bulk_data,timeout=30,chunk_size=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "matches = es.search(index='weiboscope', q='food', size=100)\n",
    "hits = matches['hits']['hits']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>author</th>\n",
       "      <th>date</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>New York – Lack of food is rarely the reason p...</td>\n",
       "      <td>Jomo Kwame Sundaram</td>\n",
       "      <td>2008-06-01T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/h...</td>\n",
       "      <td>Harvests of Hunger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROME – Lack of food is rarely the reason that ...</td>\n",
       "      <td>Jomo Kwame Sundaram</td>\n",
       "      <td>2011-05-09T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/f...</td>\n",
       "      <td>Food Fears Return</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NEW YORK – Had you told me a year ago that I w...</td>\n",
       "      <td>Esther Dyson</td>\n",
       "      <td>2014-02-25T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/e...</td>\n",
       "      <td>Lesson One in School Food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A fundamental global trend nowadays is growing...</td>\n",
       "      <td>Jeffrey D. Sachs</td>\n",
       "      <td>2007-09-20T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/t...</td>\n",
       "      <td>The Rising Cost of Nature</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LONDON – With food prices having doubled in th...</td>\n",
       "      <td>Jeremy Oppenheim</td>\n",
       "      <td>2013-11-13T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/j...</td>\n",
       "      <td>Food for All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>COPENHAGEN\\r  – \\r The global food system is i...</td>\n",
       "      <td>Per Pinstrup-Andersen</td>\n",
       "      <td>2008-09-05T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/f...</td>\n",
       "      <td>Fixing the Food Chain</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NEW YORK – Food systems have gone global. The ...</td>\n",
       "      <td>Olivier De Schutter</td>\n",
       "      <td>2013-11-01T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/o...</td>\n",
       "      <td>The Right to Eat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>GENEVA – The recent sharp spike in food prices...</td>\n",
       "      <td>Pascal Lamy</td>\n",
       "      <td>2011-02-21T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/m...</td>\n",
       "      <td>More Trade, Less Hunger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>PARIS – The world has been shaken by unprecede...</td>\n",
       "      <td>Michel Barnier</td>\n",
       "      <td>2008-11-22T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/e...</td>\n",
       "      <td>Europe and the Global Food Crisis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ROME – Every year, we waste or lose 1.3 billio...</td>\n",
       "      <td>José Graziano da Silva</td>\n",
       "      <td>2013-10-15T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/j...</td>\n",
       "      <td>Waste Not, Want Not</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>SINGAPORE – According to current estimates, In...</td>\n",
       "      <td>Asit K. Biswas</td>\n",
       "      <td>2014-08-08T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/a...</td>\n",
       "      <td>India’s Homemade Food Crisis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>STANFORD – During the late 1990’s, a singular ...</td>\n",
       "      <td>Henry I. Miller</td>\n",
       "      <td>2012-02-29T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/w...</td>\n",
       "      <td>When Technophobia Becomes Toxic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>BRUSSELS – What should the G-20 do to prepare ...</td>\n",
       "      <td>Olivier De Schutter</td>\n",
       "      <td>2011-01-28T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/f...</td>\n",
       "      <td>Food for All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>BRUSSELS – There are plenty of summits to choo...</td>\n",
       "      <td>José Manuel Barroso</td>\n",
       "      <td>2009-11-11T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/t...</td>\n",
       "      <td>The Anti-Hunger Imperative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Niger’s food emergency has reached the world’s...</td>\n",
       "      <td>Jeffrey D. Sachs</td>\n",
       "      <td>2005-08-19T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/t...</td>\n",
       "      <td>The Case Against Emergency Food Aid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>OTTAWA – One-quarter of all the food in the wo...</td>\n",
       "      <td>Bjørn Lomborg</td>\n",
       "      <td>2015-06-18T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/w...</td>\n",
       "      <td>Wasting Food in a Hungry World</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>STANFORD/BERKELEY – Virtually everyone in the ...</td>\n",
       "      <td>Paul R. Ehrlich</td>\n",
       "      <td>2015-12-30T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/f...</td>\n",
       "      <td>Feeding a Flawed Society</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>VIENNA – Of the world’s almost seven billion p...</td>\n",
       "      <td>Franz Fischler</td>\n",
       "      <td>2011-07-15T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/b...</td>\n",
       "      <td>Banishing Starvation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ROME – The world has a nutrition problem. Thou...</td>\n",
       "      <td>Jomo Kwame Sundaram</td>\n",
       "      <td>2014-10-30T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/r...</td>\n",
       "      <td>Rethinking Hunger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>BUENOS AIRES – Ever since Mexico’s so-called “...</td>\n",
       "      <td>Khatchik Der Ghougassian</td>\n",
       "      <td>2008-08-08T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/t...</td>\n",
       "      <td>Tariffs and Tortillas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>PRINCETON – Summits are defined by their locat...</td>\n",
       "      <td>Harold James</td>\n",
       "      <td>2011-06-02T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/f...</td>\n",
       "      <td>Food for Revolution</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>NEW YORK – Many poor, food-importing countries...</td>\n",
       "      <td>Jeffrey D. Sachs</td>\n",
       "      <td>2008-05-20T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/a...</td>\n",
       "      <td>A New Deal for Poor Farmers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>SINGAPORE – India is expected to overtake Chin...</td>\n",
       "      <td>Asit K. Biswas</td>\n",
       "      <td>2012-10-16T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/f...</td>\n",
       "      <td>India’s Needless Hunger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>ROME  – When the captains of business and indu...</td>\n",
       "      <td>Kanayo F. Nwanze</td>\n",
       "      <td>2010-01-25T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/r...</td>\n",
       "      <td>Rebuilding Haiti From Davos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Last year was a bad one for free trade. The Do...</td>\n",
       "      <td>Malcolm Cook</td>\n",
       "      <td>2007-02-19T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/d...</td>\n",
       "      <td>Dying for Free Trade</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ROME – Over the past several years, biofuels h...</td>\n",
       "      <td>José Graziano da Silva</td>\n",
       "      <td>2015-06-25T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/b...</td>\n",
       "      <td>Food in the Age of Biofuels</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>London – I feel a little sorry for President B...</td>\n",
       "      <td>Chris Patten</td>\n",
       "      <td>2008-05-27T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/b...</td>\n",
       "      <td>Bread and Bush-Bashing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>NEW YORK – There is no denying the fast-food i...</td>\n",
       "      <td>Andrew Billo</td>\n",
       "      <td>2012-06-26T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/f...</td>\n",
       "      <td>Fast Food’s “Ethnic Insights”</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>AMSTERDAM – In the Mekong Delta, farmers obtai...</td>\n",
       "      <td>Louise O. Fresco</td>\n",
       "      <td>2013-07-17T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/h...</td>\n",
       "      <td>Starved for Science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>FRANKFURT – A systematic and broad failure of ...</td>\n",
       "      <td>Kenneth Rogoff</td>\n",
       "      <td>2012-02-01T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/c...</td>\n",
       "      <td>Coronary Capitalism</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>Trans fats are unsaturated fatty acids with at...</td>\n",
       "      <td>Dariush Mozaffarian</td>\n",
       "      <td>2006-08-31T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/t...</td>\n",
       "      <td>The Danger of Trans Fats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>Global meat consumption is predicted to double...</td>\n",
       "      <td>Peter Singer</td>\n",
       "      <td>2006-06-14T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/t...</td>\n",
       "      <td>The Ethics of Eating</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>COPENHAGEN – The European Union is highly depe...</td>\n",
       "      <td>Anders Fogh Rasmussen</td>\n",
       "      <td>2016-01-26T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/b...</td>\n",
       "      <td>Homegrown Energy Security for Europe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>NEW YORK – Over the last two years, the majori...</td>\n",
       "      <td>Isabel Ortiz</td>\n",
       "      <td>2010-11-09T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/a...</td>\n",
       "      <td>A Recovery for All</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>NEW YORK – Around the world, protests against ...</td>\n",
       "      <td>Joseph E. Stiglitz</td>\n",
       "      <td>2008-06-06T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/s...</td>\n",
       "      <td>Scarcity in an Age of Plenty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>SINGAPORE – According to the United Nations Po...</td>\n",
       "      <td>Peter Brabeck-Letmathe</td>\n",
       "      <td>2012-05-01T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/h...</td>\n",
       "      <td>Hunger in a Complex World</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>GENEVA – Every year, 3.5 million mothers and c...</td>\n",
       "      <td>Marc van Ameringen</td>\n",
       "      <td>2008-06-24T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/t...</td>\n",
       "      <td>The Fight for Food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>NAIROBI – Kenya’s ban on imports of geneticall...</td>\n",
       "      <td>Francis Nang'ayo</td>\n",
       "      <td>2014-08-20T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/f...</td>\n",
       "      <td>Good Crop, Bad Crop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>DRESDEN – To what extent should governments re...</td>\n",
       "      <td>Kenneth Rogoff</td>\n",
       "      <td>2015-06-12T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/c...</td>\n",
       "      <td>The Right Food Fight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>PRINCETON – Can central banks contain inflatio...</td>\n",
       "      <td>Harold James</td>\n",
       "      <td>2011-04-05T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/i...</td>\n",
       "      <td>Inflationary Angst</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>UN Secretary-General Kofi Annan recently decla...</td>\n",
       "      <td>Gregory Conko</td>\n",
       "      <td>2003-10-03T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/t...</td>\n",
       "      <td>The UN's War on Science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>WASHINGTON, DC – Last year, the international ...</td>\n",
       "      <td>Thierry Tanoh</td>\n",
       "      <td>2012-08-13T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/a...</td>\n",
       "      <td>Africa’s Last Famine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>ATLANTA – We are, supposedly, in the midst of ...</td>\n",
       "      <td>Sander L. Gilman</td>\n",
       "      <td>2009-09-30T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/f...</td>\n",
       "      <td>Fat Chance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>GENEVA – Parents, take note: there is a seriou...</td>\n",
       "      <td>Oleg Chestnov</td>\n",
       "      <td>2014-09-30T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/o...</td>\n",
       "      <td>Hold the Salt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>BERKELEY – Until very recently, one of the big...</td>\n",
       "      <td>J. Bradford DeLong</td>\n",
       "      <td>2016-01-28T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/e...</td>\n",
       "      <td>Economics in the Age of Abundance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>New York – The World’s central bankers are a c...</td>\n",
       "      <td>Joseph E. Stiglitz</td>\n",
       "      <td>2008-05-06T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/t...</td>\n",
       "      <td>The Failure of Inflation Targeting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>STANFORD – Obtaining medicines from plants is ...</td>\n",
       "      <td>Henry I. Miller</td>\n",
       "      <td>2015-06-19T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/b...</td>\n",
       "      <td>Revolution and Reaction in Biopharming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>OUAGADOUGOU – Burkina Faso is located in the h...</td>\n",
       "      <td>Issa Martin Bikienga</td>\n",
       "      <td>2014-09-26T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/i...</td>\n",
       "      <td>Climate Change at Ground Zero</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>NEW YORK – At this year’s International Moneta...</td>\n",
       "      <td>Isabel Ortiz</td>\n",
       "      <td>2013-06-14T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/a...</td>\n",
       "      <td>The Austerity Pandemic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>BRUSSELS – The World Bank, the United Nations ...</td>\n",
       "      <td>Olivier De Schutter</td>\n",
       "      <td>2010-06-04T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/r...</td>\n",
       "      <td>Responsibly Destroying the World’s Peasantry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>CAMBRIDGE – Few policies place good economics ...</td>\n",
       "      <td>Jeffrey Frankel</td>\n",
       "      <td>2014-08-14T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/j...</td>\n",
       "      <td>The Subsidy Trap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>OXFORD – From 2000 to 2015, the Millennium Dev...</td>\n",
       "      <td>Ngaire Woods</td>\n",
       "      <td>2016-04-14T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/u...</td>\n",
       "      <td>Unsustainable Development Goals?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>DAKAR – As leaders of the world’s most powerfu...</td>\n",
       "      <td>Abdou Diouf</td>\n",
       "      <td>2012-06-20T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/t...</td>\n",
       "      <td>The Right Moves in Los Cabos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>The G-8 Summit in Japan earlier this month was...</td>\n",
       "      <td>Jeffrey D. Sachs</td>\n",
       "      <td>2008-07-21T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/w...</td>\n",
       "      <td>Where Are the Global Leaders?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>You know how you are told to give your kids or...</td>\n",
       "      <td>Bjørn Lomborg</td>\n",
       "      <td>2007-09-12T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/g...</td>\n",
       "      <td>Global Warnings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>NEW YORK  – The horrors of Haiti’s earthquake ...</td>\n",
       "      <td>Jeffrey D. Sachs</td>\n",
       "      <td>2010-01-25T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/r...</td>\n",
       "      <td>Reconstructing Haiti</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>VIRGINIA BEACH – In October, a group of storyt...</td>\n",
       "      <td>Beth Cardier</td>\n",
       "      <td>2014-09-25T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/b...</td>\n",
       "      <td>The Future of Storytelling</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>KIGALI – There is no greater hindrance to a co...</td>\n",
       "      <td>Agnes Binagwaho</td>\n",
       "      <td>2013-05-31T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/s...</td>\n",
       "      <td>Diet and Growth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>WASHINGTON, DC – The World Bank has set two ne...</td>\n",
       "      <td>Kaushik Basu</td>\n",
       "      <td>2013-10-07T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/k...</td>\n",
       "      <td>Reason and the End of Poverty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>TOKYO – Brinkmanship seems to be congenital in...</td>\n",
       "      <td>Yuriko Koike</td>\n",
       "      <td>2012-03-28T00:00:00</td>\n",
       "      <td>https://www.project-syndicate.org/commentary/t...</td>\n",
       "      <td>The Young General’s Old Tricks</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              article  \\\n",
       "0   New York – Lack of food is rarely the reason p...   \n",
       "1   ROME – Lack of food is rarely the reason that ...   \n",
       "2   NEW YORK – Had you told me a year ago that I w...   \n",
       "3   A fundamental global trend nowadays is growing...   \n",
       "4   LONDON – With food prices having doubled in th...   \n",
       "5   COPENHAGEN\\r  – \\r The global food system is i...   \n",
       "6   NEW YORK – Food systems have gone global. The ...   \n",
       "7   GENEVA – The recent sharp spike in food prices...   \n",
       "8   PARIS – The world has been shaken by unprecede...   \n",
       "9   ROME – Every year, we waste or lose 1.3 billio...   \n",
       "10  SINGAPORE – According to current estimates, In...   \n",
       "11  STANFORD – During the late 1990’s, a singular ...   \n",
       "12  BRUSSELS – What should the G-20 do to prepare ...   \n",
       "13  BRUSSELS – There are plenty of summits to choo...   \n",
       "14  Niger’s food emergency has reached the world’s...   \n",
       "15  OTTAWA – One-quarter of all the food in the wo...   \n",
       "16  STANFORD/BERKELEY – Virtually everyone in the ...   \n",
       "17  VIENNA – Of the world’s almost seven billion p...   \n",
       "18  ROME – The world has a nutrition problem. Thou...   \n",
       "19  BUENOS AIRES – Ever since Mexico’s so-called “...   \n",
       "20  PRINCETON – Summits are defined by their locat...   \n",
       "21  NEW YORK – Many poor, food-importing countries...   \n",
       "22  SINGAPORE – India is expected to overtake Chin...   \n",
       "23  ROME  – When the captains of business and indu...   \n",
       "24  Last year was a bad one for free trade. The Do...   \n",
       "25  ROME – Over the past several years, biofuels h...   \n",
       "26  London – I feel a little sorry for President B...   \n",
       "27  NEW YORK – There is no denying the fast-food i...   \n",
       "28  AMSTERDAM – In the Mekong Delta, farmers obtai...   \n",
       "29  FRANKFURT – A systematic and broad failure of ...   \n",
       "..                                                ...   \n",
       "70  Trans fats are unsaturated fatty acids with at...   \n",
       "71  Global meat consumption is predicted to double...   \n",
       "72  COPENHAGEN – The European Union is highly depe...   \n",
       "73  NEW YORK – Over the last two years, the majori...   \n",
       "74  NEW YORK – Around the world, protests against ...   \n",
       "75  SINGAPORE – According to the United Nations Po...   \n",
       "76  GENEVA – Every year, 3.5 million mothers and c...   \n",
       "77  NAIROBI – Kenya’s ban on imports of geneticall...   \n",
       "78  DRESDEN – To what extent should governments re...   \n",
       "79  PRINCETON – Can central banks contain inflatio...   \n",
       "80  UN Secretary-General Kofi Annan recently decla...   \n",
       "81  WASHINGTON, DC – Last year, the international ...   \n",
       "82  ATLANTA – We are, supposedly, in the midst of ...   \n",
       "83  GENEVA – Parents, take note: there is a seriou...   \n",
       "84  BERKELEY – Until very recently, one of the big...   \n",
       "85  New York – The World’s central bankers are a c...   \n",
       "86  STANFORD – Obtaining medicines from plants is ...   \n",
       "87  OUAGADOUGOU – Burkina Faso is located in the h...   \n",
       "88  NEW YORK – At this year’s International Moneta...   \n",
       "89  BRUSSELS – The World Bank, the United Nations ...   \n",
       "90  CAMBRIDGE – Few policies place good economics ...   \n",
       "91  OXFORD – From 2000 to 2015, the Millennium Dev...   \n",
       "92  DAKAR – As leaders of the world’s most powerfu...   \n",
       "93  The G-8 Summit in Japan earlier this month was...   \n",
       "94  You know how you are told to give your kids or...   \n",
       "95  NEW YORK  – The horrors of Haiti’s earthquake ...   \n",
       "96  VIRGINIA BEACH – In October, a group of storyt...   \n",
       "97  KIGALI – There is no greater hindrance to a co...   \n",
       "98  WASHINGTON, DC – The World Bank has set two ne...   \n",
       "99  TOKYO – Brinkmanship seems to be congenital in...   \n",
       "\n",
       "                      author                 date  \\\n",
       "0        Jomo Kwame Sundaram  2008-06-01T00:00:00   \n",
       "1        Jomo Kwame Sundaram  2011-05-09T00:00:00   \n",
       "2               Esther Dyson  2014-02-25T00:00:00   \n",
       "3           Jeffrey D. Sachs  2007-09-20T00:00:00   \n",
       "4           Jeremy Oppenheim  2013-11-13T00:00:00   \n",
       "5      Per Pinstrup-Andersen  2008-09-05T00:00:00   \n",
       "6        Olivier De Schutter  2013-11-01T00:00:00   \n",
       "7                Pascal Lamy  2011-02-21T00:00:00   \n",
       "8             Michel Barnier  2008-11-22T00:00:00   \n",
       "9     José Graziano da Silva  2013-10-15T00:00:00   \n",
       "10            Asit K. Biswas  2014-08-08T00:00:00   \n",
       "11           Henry I. Miller  2012-02-29T00:00:00   \n",
       "12       Olivier De Schutter  2011-01-28T00:00:00   \n",
       "13       José Manuel Barroso  2009-11-11T00:00:00   \n",
       "14          Jeffrey D. Sachs  2005-08-19T00:00:00   \n",
       "15             Bjørn Lomborg  2015-06-18T00:00:00   \n",
       "16           Paul R. Ehrlich  2015-12-30T00:00:00   \n",
       "17            Franz Fischler  2011-07-15T00:00:00   \n",
       "18       Jomo Kwame Sundaram  2014-10-30T00:00:00   \n",
       "19  Khatchik Der Ghougassian  2008-08-08T00:00:00   \n",
       "20              Harold James  2011-06-02T00:00:00   \n",
       "21          Jeffrey D. Sachs  2008-05-20T00:00:00   \n",
       "22            Asit K. Biswas  2012-10-16T00:00:00   \n",
       "23          Kanayo F. Nwanze  2010-01-25T00:00:00   \n",
       "24              Malcolm Cook  2007-02-19T00:00:00   \n",
       "25    José Graziano da Silva  2015-06-25T00:00:00   \n",
       "26              Chris Patten  2008-05-27T00:00:00   \n",
       "27              Andrew Billo  2012-06-26T00:00:00   \n",
       "28          Louise O. Fresco  2013-07-17T00:00:00   \n",
       "29            Kenneth Rogoff  2012-02-01T00:00:00   \n",
       "..                       ...                  ...   \n",
       "70       Dariush Mozaffarian  2006-08-31T00:00:00   \n",
       "71              Peter Singer  2006-06-14T00:00:00   \n",
       "72     Anders Fogh Rasmussen  2016-01-26T00:00:00   \n",
       "73              Isabel Ortiz  2010-11-09T00:00:00   \n",
       "74        Joseph E. Stiglitz  2008-06-06T00:00:00   \n",
       "75    Peter Brabeck-Letmathe  2012-05-01T00:00:00   \n",
       "76        Marc van Ameringen  2008-06-24T00:00:00   \n",
       "77          Francis Nang'ayo  2014-08-20T00:00:00   \n",
       "78            Kenneth Rogoff  2015-06-12T00:00:00   \n",
       "79              Harold James  2011-04-05T00:00:00   \n",
       "80             Gregory Conko  2003-10-03T00:00:00   \n",
       "81             Thierry Tanoh  2012-08-13T00:00:00   \n",
       "82          Sander L. Gilman  2009-09-30T00:00:00   \n",
       "83             Oleg Chestnov  2014-09-30T00:00:00   \n",
       "84        J. Bradford DeLong  2016-01-28T00:00:00   \n",
       "85        Joseph E. Stiglitz  2008-05-06T00:00:00   \n",
       "86           Henry I. Miller  2015-06-19T00:00:00   \n",
       "87      Issa Martin Bikienga  2014-09-26T00:00:00   \n",
       "88              Isabel Ortiz  2013-06-14T00:00:00   \n",
       "89       Olivier De Schutter  2010-06-04T00:00:00   \n",
       "90           Jeffrey Frankel  2014-08-14T00:00:00   \n",
       "91              Ngaire Woods  2016-04-14T00:00:00   \n",
       "92               Abdou Diouf  2012-06-20T00:00:00   \n",
       "93          Jeffrey D. Sachs  2008-07-21T00:00:00   \n",
       "94             Bjørn Lomborg  2007-09-12T00:00:00   \n",
       "95          Jeffrey D. Sachs  2010-01-25T00:00:00   \n",
       "96              Beth Cardier  2014-09-25T00:00:00   \n",
       "97           Agnes Binagwaho  2013-05-31T00:00:00   \n",
       "98              Kaushik Basu  2013-10-07T00:00:00   \n",
       "99              Yuriko Koike  2012-03-28T00:00:00   \n",
       "\n",
       "                                               source  \\\n",
       "0   https://www.project-syndicate.org/commentary/h...   \n",
       "1   https://www.project-syndicate.org/commentary/f...   \n",
       "2   https://www.project-syndicate.org/commentary/e...   \n",
       "3   https://www.project-syndicate.org/commentary/t...   \n",
       "4   https://www.project-syndicate.org/commentary/j...   \n",
       "5   https://www.project-syndicate.org/commentary/f...   \n",
       "6   https://www.project-syndicate.org/commentary/o...   \n",
       "7   https://www.project-syndicate.org/commentary/m...   \n",
       "8   https://www.project-syndicate.org/commentary/e...   \n",
       "9   https://www.project-syndicate.org/commentary/j...   \n",
       "10  https://www.project-syndicate.org/commentary/a...   \n",
       "11  https://www.project-syndicate.org/commentary/w...   \n",
       "12  https://www.project-syndicate.org/commentary/f...   \n",
       "13  https://www.project-syndicate.org/commentary/t...   \n",
       "14  https://www.project-syndicate.org/commentary/t...   \n",
       "15  https://www.project-syndicate.org/commentary/w...   \n",
       "16  https://www.project-syndicate.org/commentary/f...   \n",
       "17  https://www.project-syndicate.org/commentary/b...   \n",
       "18  https://www.project-syndicate.org/commentary/r...   \n",
       "19  https://www.project-syndicate.org/commentary/t...   \n",
       "20  https://www.project-syndicate.org/commentary/f...   \n",
       "21  https://www.project-syndicate.org/commentary/a...   \n",
       "22  https://www.project-syndicate.org/commentary/f...   \n",
       "23  https://www.project-syndicate.org/commentary/r...   \n",
       "24  https://www.project-syndicate.org/commentary/d...   \n",
       "25  https://www.project-syndicate.org/commentary/b...   \n",
       "26  https://www.project-syndicate.org/commentary/b...   \n",
       "27  https://www.project-syndicate.org/commentary/f...   \n",
       "28  https://www.project-syndicate.org/commentary/h...   \n",
       "29  https://www.project-syndicate.org/commentary/c...   \n",
       "..                                                ...   \n",
       "70  https://www.project-syndicate.org/commentary/t...   \n",
       "71  https://www.project-syndicate.org/commentary/t...   \n",
       "72  https://www.project-syndicate.org/commentary/b...   \n",
       "73  https://www.project-syndicate.org/commentary/a...   \n",
       "74  https://www.project-syndicate.org/commentary/s...   \n",
       "75  https://www.project-syndicate.org/commentary/h...   \n",
       "76  https://www.project-syndicate.org/commentary/t...   \n",
       "77  https://www.project-syndicate.org/commentary/f...   \n",
       "78  https://www.project-syndicate.org/commentary/c...   \n",
       "79  https://www.project-syndicate.org/commentary/i...   \n",
       "80  https://www.project-syndicate.org/commentary/t...   \n",
       "81  https://www.project-syndicate.org/commentary/a...   \n",
       "82  https://www.project-syndicate.org/commentary/f...   \n",
       "83  https://www.project-syndicate.org/commentary/o...   \n",
       "84  https://www.project-syndicate.org/commentary/e...   \n",
       "85  https://www.project-syndicate.org/commentary/t...   \n",
       "86  https://www.project-syndicate.org/commentary/b...   \n",
       "87  https://www.project-syndicate.org/commentary/i...   \n",
       "88  https://www.project-syndicate.org/commentary/a...   \n",
       "89  https://www.project-syndicate.org/commentary/r...   \n",
       "90  https://www.project-syndicate.org/commentary/j...   \n",
       "91  https://www.project-syndicate.org/commentary/u...   \n",
       "92  https://www.project-syndicate.org/commentary/t...   \n",
       "93  https://www.project-syndicate.org/commentary/w...   \n",
       "94  https://www.project-syndicate.org/commentary/g...   \n",
       "95  https://www.project-syndicate.org/commentary/r...   \n",
       "96  https://www.project-syndicate.org/commentary/b...   \n",
       "97  https://www.project-syndicate.org/commentary/s...   \n",
       "98  https://www.project-syndicate.org/commentary/k...   \n",
       "99  https://www.project-syndicate.org/commentary/t...   \n",
       "\n",
       "                                           title  \n",
       "0                             Harvests of Hunger  \n",
       "1                              Food Fears Return  \n",
       "2                      Lesson One in School Food  \n",
       "3                      The Rising Cost of Nature  \n",
       "4                                   Food for All  \n",
       "5                          Fixing the Food Chain  \n",
       "6                               The Right to Eat  \n",
       "7                        More Trade, Less Hunger  \n",
       "8              Europe and the Global Food Crisis  \n",
       "9                            Waste Not, Want Not  \n",
       "10                  India’s Homemade Food Crisis  \n",
       "11               When Technophobia Becomes Toxic  \n",
       "12                                  Food for All  \n",
       "13                    The Anti-Hunger Imperative  \n",
       "14           The Case Against Emergency Food Aid  \n",
       "15                Wasting Food in a Hungry World  \n",
       "16                      Feeding a Flawed Society  \n",
       "17                          Banishing Starvation  \n",
       "18                             Rethinking Hunger  \n",
       "19                         Tariffs and Tortillas  \n",
       "20                           Food for Revolution  \n",
       "21                   A New Deal for Poor Farmers  \n",
       "22                       India’s Needless Hunger  \n",
       "23                   Rebuilding Haiti From Davos  \n",
       "24                          Dying for Free Trade  \n",
       "25                   Food in the Age of Biofuels  \n",
       "26                        Bread and Bush-Bashing  \n",
       "27                 Fast Food’s “Ethnic Insights”  \n",
       "28                           Starved for Science  \n",
       "29                           Coronary Capitalism  \n",
       "..                                           ...  \n",
       "70                      The Danger of Trans Fats  \n",
       "71                          The Ethics of Eating  \n",
       "72          Homegrown Energy Security for Europe  \n",
       "73                            A Recovery for All  \n",
       "74                  Scarcity in an Age of Plenty  \n",
       "75                     Hunger in a Complex World  \n",
       "76                            The Fight for Food  \n",
       "77                           Good Crop, Bad Crop  \n",
       "78                          The Right Food Fight  \n",
       "79                            Inflationary Angst  \n",
       "80                       The UN's War on Science  \n",
       "81                          Africa’s Last Famine  \n",
       "82                                    Fat Chance  \n",
       "83                                 Hold the Salt  \n",
       "84             Economics in the Age of Abundance  \n",
       "85            The Failure of Inflation Targeting  \n",
       "86        Revolution and Reaction in Biopharming  \n",
       "87                 Climate Change at Ground Zero  \n",
       "88                        The Austerity Pandemic  \n",
       "89  Responsibly Destroying the World’s Peasantry  \n",
       "90                              The Subsidy Trap  \n",
       "91              Unsustainable Development Goals?  \n",
       "92                  The Right Moves in Los Cabos  \n",
       "93                 Where Are the Global Leaders?  \n",
       "94                               Global Warnings  \n",
       "95                          Reconstructing Haiti  \n",
       "96                    The Future of Storytelling  \n",
       "97                               Diet and Growth  \n",
       "98                 Reason and the End of Poverty  \n",
       "99                The Young General’s Old Tricks  \n",
       "\n",
       "[100 rows x 5 columns]"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(list(pd.DataFrame(hits)['_source']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-215-bf7b7bbb961e>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-215-bf7b7bbb961e>\"\u001b[1;36m, line \u001b[1;32m1\u001b[0m\n\u001b[1;33m    es.\u001b[0m\n\u001b[1;37m       ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}