Feature_Engineering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ca339e6a-acf3-4e83-9f66-db7ba2b0aea7",
   "metadata": {},
   "source": [
    "# Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1b4b9902-dee4-4bbe-bfbf-63fd2a8c01ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                avatar     fullname  \\\n",
      "0                                                  NaN          NaN   \n",
      "1                                                  NaN          NaN   \n",
      "2                                                  NaN          NaN   \n",
      "3                                                  NaN          NaN   \n",
      "4                                                  NaN          NaN   \n",
      "..                                                 ...          ...   \n",
      "298  https://pbs.twimg.com/profile_images/570612588...  Vimal Mohan   \n",
      "299                                                NaN          NaN   \n",
      "300                                                NaN          NaN   \n",
      "301                                                NaN          NaN   \n",
      "302                                                NaN          NaN   \n",
      "\n",
      "                      id                                         images/0  \\\n",
      "0    1608321408001781762  https://pbs.twimg.com/media/FlHnyq-aMAAM8lK.jpg   \n",
      "1    1608486384784015362                                              NaN   \n",
      "2    1608711406383435778                                              NaN   \n",
      "3    1609128810829565953                                              NaN   \n",
      "4    1609128542184574982  https://pbs.twimg.com/media/FlTF3wdakAEYsQT.jpg   \n",
      "..                   ...                                              ...   \n",
      "298   913057826083237888                                              NaN   \n",
      "299   970186732153987072                                              NaN   \n",
      "300  1594575991581863941                                              NaN   \n",
      "301  1594665034189193216  https://pbs.twimg.com/media/FiFjYoZVsAANYsX.jpg   \n",
      "302   926712549927612416                                              NaN   \n",
      "\n",
      "                                            images/1  \\\n",
      "0                                                NaN   \n",
      "1                                                NaN   \n",
      "2                                                NaN   \n",
      "3                                                NaN   \n",
      "4                                                NaN   \n",
      "..                                               ...   \n",
      "298                                              NaN   \n",
      "299                                              NaN   \n",
      "300                                              NaN   \n",
      "301  https://pbs.twimg.com/media/FiFjY3XVEAAZklv.jpg   \n",
      "302                                              NaN   \n",
      "\n",
      "                                            images/2  \\\n",
      "0                                                NaN   \n",
      "1                                                NaN   \n",
      "2                                                NaN   \n",
      "3                                                NaN   \n",
      "4                                                NaN   \n",
      "..                                               ...   \n",
      "298                                              NaN   \n",
      "299                                              NaN   \n",
      "300                                              NaN   \n",
      "301  https://pbs.twimg.com/media/FiFjZJkVIAAEgV2.jpg   \n",
      "302                                              NaN   \n",
      "\n",
      "                                            images/3  isQuote  isRetweet  \\\n",
      "0                                                NaN    False      False   \n",
      "1                                                NaN    False      False   \n",
      "2                                                NaN    False      False   \n",
      "3                                                NaN    False      False   \n",
      "4                                                NaN    False      False   \n",
      "..                                               ...      ...        ...   \n",
      "298                                              NaN    False      False   \n",
      "299                                              NaN    False      False   \n",
      "300                                              NaN    False      False   \n",
      "301  https://pbs.twimg.com/media/FiFjZYoVsAAgVlx.jpg    False      False   \n",
      "302                                              NaN    False      False   \n",
      "\n",
      "     likes  ...     user/username user/verified  \\\n",
      "0        0  ...       @Ballugreat         False   \n",
      "1        0  ...  @bigpeaksinsight         False   \n",
      "2       10  ...    @singhaman1904         False   \n",
      "3        0  ...        @themasumm         False   \n",
      "4       10  ...          @sdasila         False   \n",
      "..     ...  ...               ...           ...   \n",
      "298      3  ...     @vimalmohan59         False   \n",
      "299      5  ...  @Pallavi32818501         False   \n",
      "300      5  ...           @va6004         False   \n",
      "301     12  ...   @SfiUttarakhand         False   \n",
      "302      1  ...    @SatrangiBalak         False   \n",
      "\n",
      "                      user/website       username verified  \\\n",
      "0    instagram.com/invinciblebobb…            NaN    False   \n",
      "1        dreamcatcheradventure.com            NaN    False   \n",
      "2                              NaN            NaN    False   \n",
      "3    linkedin.com/in/masum-raza-5…            NaN    False   \n",
      "4                              NaN            NaN    False   \n",
      "..                             ...            ...      ...   \n",
      "298                            NaN  @vimalmohan59    False   \n",
      "299                            NaN            NaN    False   \n",
      "300                            NaN            NaN    False   \n",
      "301                    sficec.org/            NaN    False   \n",
      "302                            NaN            NaN    False   \n",
      "\n",
      "                                     preprocessed_text text_length word_count  \\\n",
      "0                          freeze dehradun uttarakhand          27          3   \n",
      "1    and ams 3870m12800 999person 7579079958 kuarip...         237         19   \n",
      "2    video told rishabh pants recent accident uttar...         100         14   \n",
      "3                    posted photo dehradun uttarakhand          33          4   \n",
      "4    health much better uttarakhand mesh kumar pant...         125         19   \n",
      "..                                                 ...         ...        ...   \n",
      "298  baba ram ahim gut feelings honey greet singh m...          86         14   \n",
      "299  know the what feel from inside many mix feelin...         135         22   \n",
      "300  turkish ice cream road mussoorie uttarakhand t...          79          9   \n",
      "301  students federation india wins seat treasurer ...         177         24   \n",
      "302  simple person hides thousand feelings behind h...          59          8   \n",
      "\n",
      "    emotional_intensity readability  \n",
      "0                     0         5.6  \n",
      "1                     0        22.5  \n",
      "2                     0         8.8  \n",
      "3                     0         9.6  \n",
      "4                     0        11.9  \n",
      "..                  ...         ...  \n",
      "298                   0         6.4  \n",
      "299                   0         8.3  \n",
      "300                   0        10.3  \n",
      "301                   0        16.2  \n",
      "302                   0        10.0  \n",
      "\n",
      "[303 rows x 85 columns]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import textstat\n",
    "\n",
    "# Load your dataset\n",
    "df = pd.read_csv(\"uttarakhand_clean.csv\")\n",
    "\n",
    "# Handle NaN values in 'preprocessed_text' column\n",
    "df['preprocessed_text'] = df['text'].fillna('')\n",
    "\n",
    "# Text Length\n",
    "df['text_length'] = df['text'].apply(lambda x: len(str(x)))\n",
    "\n",
    "# Word Count\n",
    "df['word_count'] = df['text'].apply(lambda x: len(str(x).split()))\n",
    "\n",
    "# Emotional Intensity\n",
    "df['emotional_intensity'] = df['text'].apply(lambda x: str(x).count('!'))\n",
    "\n",
    "# N-grams\n",
    "ngram_vectorizer = CountVectorizer(ngram_range=(1, 2))\n",
    "ngrams = ngram_vectorizer.fit_transform(df['text'].astype(str))\n",
    "\n",
    "# Readability Metrics\n",
    "df['readability'] = df['text'].apply(lambda x: textstat.flesch_kincaid_grade(str(x)))\n",
    "\n",
    "# Display the updated DataFrame\n",
    "print(df)\n",
    "\n",
    "df.to_csv(\"uttarakhand_with_features.csv\", index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d9e6805-f127-4045-9840-91726de3104a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}