Image Scraper( ASOS ).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib\n",
    "import urllib.request\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'\n",
    "\n",
    "headers={'User-Agent':user_agent,} \n",
    "\n",
    "def make_soup(url):\n",
    "    request= urllib.request.Request(url, None,headers) #The assembled request\n",
    "    thepage = urllib.request.urlopen(request)\n",
    "    soupdata = BeautifulSoup(thepage, \"html.parser\")\n",
    "    return soupdata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Page  0  done\n",
      "Page  1  done\n",
      "Page  2  done\n",
      "Page  3  done\n",
      "Page  4  done\n",
      "Page  5  done\n",
      "Page  6  done\n",
      "Page  7  done\n",
      "Page  8  done\n",
      "Page  9  done\n",
      "Page  10  done\n",
      "Page  11  done\n",
      "Page  12  done\n",
      "Page  13  done\n",
      "Page  14  done\n",
      "Page  15  done\n",
      "Page  16  done\n",
      "Page  17  done\n",
      "Page  18  done\n",
      "Page  19  done\n",
      "Page  20  done\n",
      "Page  21  done\n",
      "Page  22  done\n",
      "Page  23  done\n",
      "Page  24  done\n",
      "Page  25  done\n",
      "Page  26  done\n",
      "Page  27  done\n",
      "Page  28  done\n",
      "Page  29  done\n",
      "Page  30  done\n",
      "Page  31  done\n",
      "Page  32  done\n",
      "Page  33  done\n",
      "Page  34  done\n",
      "Page  35  done\n",
      "Page  36  done\n",
      "Page  37  done\n",
      "Page  38  done\n",
      "Page  39  done\n",
      "Page  40  done\n",
      "Page  41  done\n",
      "Page  42  done\n",
      "Page  43  done\n",
      "Page  44  done\n",
      "Page  45  done\n",
      "Page  46  done\n",
      "Page  47  done\n",
      "Page  48  done\n",
      "Page  49  done\n",
      "Page  50  done\n",
      "Page  51  done\n",
      "Page  52  done\n",
      "Page  53  done\n",
      "Page  54  done\n",
      "Page  55  done\n",
      "Page  56  done\n",
      "Page  57  done\n",
      "Page  58  done\n",
      "Page  59  done\n",
      "Page  60  done\n",
      "Page  61  done\n",
      "Page  62  done\n",
      "Page  63  done\n",
      "Page  64  done\n",
      "Page  65  done\n",
      "Page  66  done\n",
      "Page  67  done\n",
      "Page  68  done\n",
      "Page  69  done\n"
     ]
    }
   ],
   "source": [
    "urls = []\n",
    "for i in range(70):\n",
    "    site = 'http://www.asos.com/women/tops/cat/?cid=4169&nlid=ww|clothing|shop%20by%20product&page='\n",
    "    site = site + str(i)\n",
    "    soup = make_soup(site)\n",
    "    for img in soup.findAll('img'):\n",
    "        temp = img.get('src')\n",
    "        if temp is not None and \"products\" in temp:\n",
    "            urls.append(temp)\n",
    "    print('Page ', i, ' done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2520"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for i in range(len(urls)):\n",
    "    filename = str(i+1) + '_Tops'\n",
    "    path = 'ASOS/Tops/' + filename\n",
    "    imagefile = open(path + \".jpeg\", 'wb')\n",
    "    imagefile.write(urllib.request.urlopen(urls[i]).read())\n",
    "    imagefile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}