AIM_Fetching.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import pandas as pd\n",
    "url = 'https://recruitment.aimtechnologies.co/ai-tasks'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(458197, 2)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ids_df = pd.read_csv('RawData/dialect_dataset.csv')\n",
    "ids_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic = {}\n",
    "epochs = len(ids_df)//1000\n",
    "\n",
    "for i in range(epochs+1):\n",
    "    data = json.dumps(list(map(str, ids_df[1000*(i):1000*(i+1)]['id'])))\n",
    "    r = requests.post(url = url, data = data)\n",
    "    dic.update(r.json())\n",
    "\n",
    "df = pd.DataFrame(list(dic.items()),columns = ['id','text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['id'] = df['id'].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_df = pd.merge(df,ids_df,on='id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>dialect</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28543</th>\n",
       "      <td>988563322227249152</td>\n",
       "      <td>@MalikKhalaf78 التطور سرق الوقت بشكل كبير, ساع...</td>\n",
       "      <td>LY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350130</th>\n",
       "      <td>1165529471056830464</td>\n",
       "      <td>لما المواطن يروح للحكومة و يعرض عليها مشروع في...</td>\n",
       "      <td>KW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136057</th>\n",
       "      <td>903542835504394240</td>\n",
       "      <td>@_Emanan ليش هي مش هيك؟؟!!!😂😂</td>\n",
       "      <td>SY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205780</th>\n",
       "      <td>944288399271387136</td>\n",
       "      <td>@ufatm_1 ما بينهم شي يا قلبي .. و المقطع اللي ...</td>\n",
       "      <td>SA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213748</th>\n",
       "      <td>937427170275848192</td>\n",
       "      <td>#تزهاك_البلديه_الثانيه_يابوحور\\n\\nالاوسكار ي م...</td>\n",
       "      <td>SA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         id  \\\n",
       "28543    988563322227249152   \n",
       "350130  1165529471056830464   \n",
       "136057   903542835504394240   \n",
       "205780   944288399271387136   \n",
       "213748   937427170275848192   \n",
       "\n",
       "                                                     text dialect  \n",
       "28543   @MalikKhalaf78 التطور سرق الوقت بشكل كبير, ساع...      LY  \n",
       "350130  لما المواطن يروح للحكومة و يعرض عليها مشروع في...      KW  \n",
       "136057                      @_Emanan ليش هي مش هيك؟؟!!!😂😂      SY  \n",
       "205780  @ufatm_1 ما بينهم شي يا قلبي .. و المقطع اللي ...      SA  \n",
       "213748  #تزهاك_البلديه_الثانيه_يابوحور\\n\\nالاوسكار ي م...      SA  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(458197, 3)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_df.to_csv('RawData/raw_data.csv',index = False, sep='\\t',encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}