CleaningArticleData.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T20:17:51.384912Z",
     "start_time": "2019-04-19T20:17:51.381097Z"
    }
   },
   "outputs": [],
   "source": [
    "#removes stopwords, punctuation,numbers, does stemmming\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer \n",
    "from nltk.stem import WordNetLemmatizer \n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T02:44:46.146026Z",
     "start_time": "2019-04-19T02:44:46.090411Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lemmatizer = WordNetLemmatizer()\n",
    "ps = PorterStemmer() \n",
    "stop_words = set(stopwords.words(\"english\"))\n",
    "\n",
    "with open('article1.txt','r') as inFile, open('AoutputFile.txt','w') as outFile:\n",
    "    for line in inFile.readlines():\n",
    "        inpline=line.lower()\n",
    "        wordslist = word_tokenize(inpline)\n",
    "        #print(example_words)\n",
    "        wordslist = filter(lambda x: x not in string.punctuation, wordslist) #punctuation\n",
    "        alpha_words = filter(lambda x: x.isalpha(), wordslist) #no numbers\n",
    "        cleaned_text = filter(lambda x: x not in stop_words, alpha_words) #stop words\n",
    "        final_text = filter(lambda x: len(x)>3,cleaned_text) #styled punctuations and other 2 letter words\n",
    "        #op=\" \".join([ps.stem(w) for w in final_text]) #stemming\n",
    "        op=\" \".join([lemmatizer.lemmatize(w) for w in final_text]) #lemmetize\n",
    "        print(op)\n",
    "        outFile.write(op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T21:36:16.239662Z",
     "start_time": "2019-04-19T21:36:16.181687Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'twt' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-69bb6f72ff18>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclean_tweet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtwt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m17\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'twt' is not defined"
     ]
    }
   ],
   "source": [
    "def clean_tweet(tweet): \n",
    "    str = ' '.join(re.sub(\"(@[A-Za-z0-9]+)|([^0-9A-Za-z \\t])|(\\w+:\\/\\/\\S+)|(#[A-Za-z0-9]+)|(http[A-Za-z0-9]+)\", \" \", tweet).split()) \n",
    "    str = str.lower()\n",
    "    return str\n",
    "\n",
    "t = clean_tweet(twt[17])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T02:45:27.585321Z",
     "start_time": "2019-04-19T02:45:27.579276Z"
    }
   },
   "outputs": [],
   "source": [
    "twt[17]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T02:54:22.002129Z",
     "start_time": "2019-04-19T02:54:21.998958Z"
    }
   },
   "outputs": [],
   "source": [
    "ps = PorterStemmer()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-19T03:12:51.936427Z",
     "start_time": "2019-04-19T03:12:51.905572Z"
    }
   },
   "outputs": [],
   "source": [
    "tt = word_tokenize(twt[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}