prepare_medium_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-01T17:18:55.692543Z",
     "start_time": "2018-11-01T17:18:52.525203Z"
    }
   },
   "outputs": [],
   "source": [
    "# file system navigation\n",
    "from pathlib import Path\n",
    "\n",
    "# data transformation\n",
    "import pandas as pd\n",
    "\n",
    "# nlp\n",
    "import spacy\n",
    "from spacy_cld import LanguageDetector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:36:24.713454Z",
     "start_time": "2018-10-26T18:36:24.565640Z"
    }
   },
   "outputs": [],
   "source": [
    "articles = pd.read_csv(Path.cwd() / \"data\" / \"articles.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:36:24.775900Z",
     "start_time": "2018-10-26T18:36:24.719169Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(337, 6)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Find and remove duplicates based on author and title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:36:24.809221Z",
     "start_time": "2018-10-26T18:36:24.783106Z"
    }
   },
   "outputs": [],
   "source": [
    "articles = articles\\\n",
    "    .drop_duplicates(subset=[\"author\", \"title\"], keep=\"first\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:36:24.825229Z",
     "start_time": "2018-10-26T18:36:24.814354Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(230, 6)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert claps to integer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:36:24.898993Z",
     "start_time": "2018-10-26T18:36:24.829072Z"
    }
   },
   "outputs": [],
   "source": [
    "articles[\"claps\"] = articles[\"claps\"].apply(lambda s: int(float(s[:-1]) * 1000) if s[-1] == \"K\" else int(s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:36:25.144285Z",
     "start_time": "2018-10-26T18:36:25.060275Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>claps</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>link</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Justin Lee</td>\n",
       "      <td>8300</td>\n",
       "      <td>11</td>\n",
       "      <td>https://medium.com/swlh/chatbots-were-the-next...</td>\n",
       "      <td>Chatbots were the next big thing: what happene...</td>\n",
       "      <td>Oh, how the headlines blared:\\nChatbots were T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Conor Dewey</td>\n",
       "      <td>1400</td>\n",
       "      <td>7</td>\n",
       "      <td>https://towardsdatascience.com/python-for-data...</td>\n",
       "      <td>Python for Data Science: 8 Concepts You May Ha...</td>\n",
       "      <td>If you’ve ever found yourself looking up the s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>William Koehrsen</td>\n",
       "      <td>2800</td>\n",
       "      <td>11</td>\n",
       "      <td>https://towardsdatascience.com/automated-featu...</td>\n",
       "      <td>Automated Feature Engineering in Python – Towa...</td>\n",
       "      <td>Machine learning is increasingly moving from h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Gant Laborde</td>\n",
       "      <td>1300</td>\n",
       "      <td>7</td>\n",
       "      <td>https://medium.freecodecamp.org/machine-learni...</td>\n",
       "      <td>Machine Learning: how to go from Zero to Hero ...</td>\n",
       "      <td>If your understanding of A.I. and Machine Lear...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Emmanuel Ameisen</td>\n",
       "      <td>935</td>\n",
       "      <td>11</td>\n",
       "      <td>https://blog.insightdatascience.com/reinforcem...</td>\n",
       "      <td>Reinforcement Learning from scratch – Insight ...</td>\n",
       "      <td>Want to learn about applied Artificial Intelli...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             author  claps  reading_time  \\\n",
       "0        Justin Lee   8300            11   \n",
       "1       Conor Dewey   1400             7   \n",
       "2  William Koehrsen   2800            11   \n",
       "3      Gant Laborde   1300             7   \n",
       "4  Emmanuel Ameisen    935            11   \n",
       "\n",
       "                                                link  \\\n",
       "0  https://medium.com/swlh/chatbots-were-the-next...   \n",
       "1  https://towardsdatascience.com/python-for-data...   \n",
       "2  https://towardsdatascience.com/automated-featu...   \n",
       "3  https://medium.freecodecamp.org/machine-learni...   \n",
       "4  https://blog.insightdatascience.com/reinforcem...   \n",
       "\n",
       "                                               title  \\\n",
       "0  Chatbots were the next big thing: what happene...   \n",
       "1  Python for Data Science: 8 Concepts You May Ha...   \n",
       "2  Automated Feature Engineering in Python – Towa...   \n",
       "3  Machine Learning: how to go from Zero to Hero ...   \n",
       "4  Reinforcement Learning from scratch – Insight ...   \n",
       "\n",
       "                                                text  \n",
       "0  Oh, how the headlines blared:\\nChatbots were T...  \n",
       "1  If you’ve ever found yourself looking up the s...  \n",
       "2  Machine learning is increasingly moving from h...  \n",
       "3  If your understanding of A.I. and Machine Lear...  \n",
       "4  Want to learn about applied Artificial Intelli...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Remove articles, which are not written in english"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:36:28.124023Z",
     "start_time": "2018-10-26T18:36:26.301603Z"
    }
   },
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en\")\n",
    "language_detector = LanguageDetector()\n",
    "nlp.add_pipe(language_detector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:36:28.157065Z",
     "start_time": "2018-10-26T18:36:28.128995Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_en_score(df, col_name):\n",
    "    \n",
    "    data = df.copy()\n",
    "    data[\"en_score\"] = data[col_name].apply(lambda x: nlp(x)._.language_scores.get(\"en\", 0.0))\n",
    "    \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:39:15.396467Z",
     "start_time": "2018-10-26T18:36:28.164392Z"
    }
   },
   "outputs": [],
   "source": [
    "unique_articles_en = articles \\\n",
    "        .pipe(get_en_score, \"text\") \\\n",
    "        .query(\"en_score > 0.9\") \\\n",
    "        .drop(\"en_score\", axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:39:15.414699Z",
     "start_time": "2018-10-26T18:39:15.401520Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(224, 6)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_articles_en.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare and save data for labeling (author, claps and link only) as csv and for further processing as parquet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "LABEL_PATH = Path.cwd() / \"data\" / \"processed\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not (LABEL_PATH / \"articles_link_only.csv\").is_file:\n",
    "    LABEL_PATH.mkdir()\n",
    "    \n",
    "    unique_articles_en \\\n",
    "    .sort_values(by=\"claps\", ascending=False) \\\n",
    "    .loc[:, [\"author\", \"claps\", \"link\"]] \\\n",
    "    .to_csv(LABEL_PATH / \"articles_link_only.csv\", sep=\";\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not (LABEL_PATH / \"unique_articles.parquet\").is_file:\n",
    "    unique_articles_en \\\n",
    "    .to_parquet(LABEL_PATH / \"unique_articles.parquet\", engine=\"pyarrow\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Add labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:47:52.199007Z",
     "start_time": "2018-10-26T18:47:52.165085Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "labels = pd.read_csv(Path.cwd() / \"data\" / \"labels.csv\", sep=\";\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:47:53.438804Z",
     "start_time": "2018-10-26T18:47:53.416252Z"
    }
   },
   "outputs": [],
   "source": [
    "labeled_data = pd.merge(unique_articles_en, labels, how=\"inner\", left_index=True, right_on=\"id\").drop(\"id\", axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save for further use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "SHARING_PATH = Path.cwd() / \"data\" / \"shared\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-26T18:48:01.744015Z",
     "start_time": "2018-10-26T18:48:01.396275Z"
    }
   },
   "outputs": [],
   "source": [
    "if not (SHARING_PATH / \"train_data.parquet\").is_file:\n",
    "    SHARING_PATH.mkdir()\n",
    "    labeled_data \\\n",
    "    .to_parquet(Path.cwd() / \"data\" / \"processed\" / \"train_data.parquet\", engine=\"pyarrow\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}