price_based_on_category.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Get a random sample from the training set\n",
    "# data = pd.read_table('/Users/David/Desktop/Mercari/train.tsv')\n",
    "# data_nrows = data.shape[0]\n",
    "# data['random'] = pd.Series(np.random.randn(data_nrows), index=data.index)\n",
    "# sorted_data = data.sort_values(by=['random'])\n",
    "# random_sample = sorted_data.head(148254)\n",
    "# random_sample.to_csv(\"/Users/David/Desktop/Mercari/random_sample.tsv\", sep='\\t', index=False)\n",
    "# data1.append(data2, ignore_index=True)\n",
    "\n",
    "# Load random sample after it's been written for the first time\n",
    "random_sample = pd.read_table('/Users/David/Desktop/Mercari/random_sample.tsv')\n",
    "\n",
    "# Load category metadata and get expected price per item based on category\n",
    "categories = pd.read_table('/Users/David/Desktop/Mercari/statsBySubcategory.csv', sep=\",\")\n",
    "\n",
    "# Predict price based on category\n",
    "predictions = random_sample.set_index('category_name').join(categories.set_index('Category'), how='left')\n",
    "deviations = abs(predictions['price'] - predictions['Median']) / predictions['price']\n",
    "\n",
    "# Return an estimate of accuracy of predictions\n",
    "import math\n",
    "accuracy = 1 - np.median(list(filter(lambda deviation: not math.isnan(deviation), deviations)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.59999999999999998"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('nighter', 43),\n",
       " ('cofounder', 2),\n",
       " ('berkus', 1),\n",
       " ('frustrating', 5),\n",
       " ('conditionmost', 1),\n",
       " ('psychobilly', 4),\n",
       " ('woody', 34),\n",
       " ('roughandtumble', 1),\n",
       " ('reflectors', 5),\n",
       " ('batmobile', 9),\n",
       " ('sleepwearhouse', 1),\n",
       " ('bursts', 7),\n",
       " ('spillfree', 2),\n",
       " ('leyya', 1),\n",
       " ('waverly', 1),\n",
       " ('dolphins', 21),\n",
       " ('cord', 1285),\n",
       " ('sjp', 2),\n",
       " ('mobiles', 6),\n",
       " ('pickets', 1),\n",
       " ('cagestrappy', 1),\n",
       " ('surethey', 1),\n",
       " ('pairsfree', 8),\n",
       " ('saturday', 264),\n",
       " ('enter', 443),\n",
       " ('purifiedwater', 1),\n",
       " ('sweaterish', 1),\n",
       " ('poem', 6),\n",
       " ('ta', 46543),\n",
       " ('tretinoin', 4),\n",
       " ('macyscom', 2),\n",
       " ('start', 640),\n",
       " ('folkmanis', 1),\n",
       " ('abreu', 2),\n",
       " ('drawing', 38),\n",
       " ('blinged', 10),\n",
       " ('careless', 3),\n",
       " ('chumsworth', 1),\n",
       " ('exhalation', 1),\n",
       " ('google', 92),\n",
       " ('cardio', 27),\n",
       " ('shippingpokmon', 1),\n",
       " ('rebuild', 5),\n",
       " ('luch', 2),\n",
       " ('dinotrux', 2),\n",
       " ('othwr', 1),\n",
       " ('nakedness', 1),\n",
       " ('poison', 27),\n",
       " ('input', 88),\n",
       " ('justshipit', 2),\n",
       " ('rac', 5887),\n",
       " ('columbia', 126),\n",
       " ('fadednot', 1),\n",
       " ('etchings', 1),\n",
       " ('hrvest', 1),\n",
       " ('marlin', 9),\n",
       " ('skinnothing', 1),\n",
       " ('missions', 4),\n",
       " ('motor', 149),\n",
       " ('maxus', 1),\n",
       " ('uvf', 1),\n",
       " ('procedure', 3),\n",
       " ('tactic', 16),\n",
       " ('nwtsize', 2),\n",
       " ('lend', 1217),\n",
       " ('frequncy', 3),\n",
       " ('sae', 4),\n",
       " ('transformable', 1),\n",
       " ('comfortwear', 1),\n",
       " ('surprises', 7),\n",
       " ('territory', 1),\n",
       " ('proms', 1),\n",
       " ('mightve', 1),\n",
       " ('hightops', 3),\n",
       " ('tagsbox', 3),\n",
       " ('abit', 39),\n",
       " ('setsrm', 2),\n",
       " ('bindage', 1),\n",
       " ('trigonometric', 1),\n",
       " ('mandalapurple', 1),\n",
       " ('smoke', 5886),\n",
       " ('flosser', 3),\n",
       " ('bgirl', 2),\n",
       " ('celin', 25),\n",
       " ('jananibae', 1),\n",
       " ('mspr', 1),\n",
       " ('illipe', 1),\n",
       " ('blauer', 1),\n",
       " ('playability', 3),\n",
       " ('elephants', 50),\n",
       " ('bradford', 6),\n",
       " ('ladylotusgear', 1),\n",
       " ('freebrow', 1),\n",
       " ('caravelle', 1),\n",
       " ('kawa', 190),\n",
       " ('scalloping', 1),\n",
       " ('equate', 11),\n",
       " ('bouncing', 6),\n",
       " ('raschel', 3),\n",
       " ('coif', 1),\n",
       " ('weeklys', 3),\n",
       " ('miamidolphins', 1),\n",
       " ('decorwedding', 1),\n",
       " ('brookstone', 2),\n",
       " ('remarks', 1),\n",
       " ('nightmarebeforechrnightmare', 1),\n",
       " ('looovvvveeee', 1),\n",
       " ('highlightblushbronzing', 1),\n",
       " ('montevideos', 1),\n",
       " ('jiminy', 1),\n",
       " ('profileshands', 1),\n",
       " ('blackbase', 2),\n",
       " ('ticke', 1198),\n",
       " ('wearholes', 1),\n",
       " ('coralwhite', 2),\n",
       " ('makeoverinstantly', 1),\n",
       " ('underwire', 216),\n",
       " ('infinity', 81),\n",
       " ('nutritive', 2),\n",
       " ('words', 235),\n",
       " ('kaciirow', 1),\n",
       " ('dane', 4),\n",
       " ('bottling', 2),\n",
       " ('hispanic', 1),\n",
       " ('instruments', 20),\n",
       " ('twostage', 1),\n",
       " ('switchable', 2),\n",
       " ('withot', 6),\n",
       " ('leggingslarge', 1),\n",
       " ('splity', 1),\n",
       " ('casesonly', 1),\n",
       " ('zeepbath', 1),\n",
       " ('dimples', 5),\n",
       " ('throh', 1),\n",
       " ('magcool', 1),\n",
       " ('pricethis', 2),\n",
       " ('shams', 36),\n",
       " ('whithout', 2),\n",
       " ('togel', 1),\n",
       " ('straightaway', 1),\n",
       " ('superwash', 1),\n",
       " ('wana', 1),\n",
       " ('questionable', 2),\n",
       " ('finn', 6),\n",
       " ('smudgefree', 4),\n",
       " ('cylon', 1),\n",
       " ('crithmum', 1),\n",
       " ('pegan', 3),\n",
       " ('designh', 2),\n",
       " ('kimport', 1),\n",
       " ('teamtiger', 1),\n",
       " ('eileenfisher', 2),\n",
       " ('copenhagen', 5),\n",
       " ('wont', 623),\n",
       " ('fates', 22),\n",
       " ('visuals', 2),\n",
       " ('conditionkeeps', 1),\n",
       " ('persons', 6),\n",
       " ('cyber', 38),\n",
       " ('common', 123),\n",
       " ('thermaplush', 1),\n",
       " ('nwton', 1),\n",
       " ('begay', 1),\n",
       " ('patroncito', 1),\n",
       " ('lacklusterlooking', 1),\n",
       " ('knifein', 1),\n",
       " ('maxruby', 1),\n",
       " ('bundledrm', 2),\n",
       " ('xxtrucoxx', 2),\n",
       " ('dentsdamagesscratches', 1),\n",
       " ('abriana', 1),\n",
       " ('pinpoint', 7),\n",
       " ('translucent', 95),\n",
       " ('slayin', 3),\n",
       " ('gravy', 6),\n",
       " ('lowprices', 1),\n",
       " ('duggal', 3),\n",
       " ('stroller', 60),\n",
       " ('asheville', 1),\n",
       " ('coralwhiteblack', 1),\n",
       " ('marchonyc', 1),\n",
       " ('rider', 30),\n",
       " ('sampletrial', 2),\n",
       " ('lilac', 97),\n",
       " ('twistup', 1),\n",
       " ('nimbus', 3),\n",
       " ('smsimessage', 1),\n",
       " ('retinol', 28),\n",
       " ('caudalie', 19),\n",
       " ('noitch', 1),\n",
       " ('helmetsjust', 1),\n",
       " ('etc', 3820),\n",
       " ('graffik', 1),\n",
       " ('thankful', 21),\n",
       " ('starlight', 9),\n",
       " ('whiteboard', 11),\n",
       " ('hour', 1614),\n",
       " ('figure', 596),\n",
       " ('simplydoanners', 1),\n",
       " ('privacythe', 1),\n",
       " ('charmander', 6),\n",
       " ('metered', 1),\n",
       " ('submersible', 8),\n",
       " ('selfish', 1),\n",
       " ('topshown', 1),\n",
       " ('lotionml', 1),\n",
       " ('cooler', 127),\n",
       " ('mixamp', 1),\n",
       " ('bungle', 2),\n",
       " ('craves', 2),\n",
       " ('ekone', 1),\n",
       " ('espadrilles', 22),\n",
       " ('mercri', 5),\n",
       " ('bronzing', 43),\n",
       " ('nourishing', 86),\n",
       " ('boysmens', 1),\n",
       " ('theater', 23),\n",
       " ('qualitycome', 1),\n",
       " ('sixers', 1),\n",
       " ('ssssss', 2),\n",
       " ('overjoyed', 1),\n",
       " ('realllly', 1),\n",
       " ('posey', 7),\n",
       " ('microdelivery', 4),\n",
       " ('goldmist', 1),\n",
       " ('noline', 2),\n",
       " ('hilow', 19),\n",
       " ('celebie', 1),\n",
       " ('hans', 60),\n",
       " ('camuto', 34),\n",
       " ('weiss', 2),\n",
       " ('getitnow', 1),\n",
       " ('contch', 1),\n",
       " ('storage', 679),\n",
       " ('fireworld', 1),\n",
       " ('mahnke', 1),\n",
       " ('cassies', 4),\n",
       " ('widecut', 1),\n",
       " ('electroplating', 1),\n",
       " ('sanity', 7),\n",
       " ('cambogia', 9),\n",
       " ('cohen', 1),\n",
       " ('cline', 19),\n",
       " ('itemmi', 3),\n",
       " ('cruisers', 3),\n",
       " ('give', 1622),\n",
       " ('jessicake', 1),\n",
       " ('bundleee', 1),\n",
       " ('veneer', 1),\n",
       " ('overhook', 2),\n",
       " ('waterpipe', 3),\n",
       " ('pushup', 263),\n",
       " ('smellyou', 1),\n",
       " ('yea', 1409),\n",
       " ('revised', 5),\n",
       " ('rump', 56),\n",
       " ('chihuahuas', 2),\n",
       " ('bootsrain', 1),\n",
       " ('waterproofed', 1),\n",
       " ('figet', 3),\n",
       " ('kpizzzle', 1),\n",
       " ('facitis', 1),\n",
       " ('latching', 1),\n",
       " ('ready', 1475),\n",
       " ('recurring', 2),\n",
       " ('hanger', 114),\n",
       " ('worse', 8),\n",
       " ('suspense', 5),\n",
       " ('crossroads', 2),\n",
       " ('thsnknyou', 1),\n",
       " ('qt', 89),\n",
       " ('spotsthe', 1),\n",
       " ('nagotiate', 1),\n",
       " ('smallerbreasted', 1),\n",
       " ('colicky', 1),\n",
       " ('bases', 14),\n",
       " ('dualcore', 4),\n",
       " ('cupcan', 1),\n",
       " ('defined', 34),\n",
       " ('oncemaybe', 1),\n",
       " ('benfica', 1),\n",
       " ('advertising', 2),\n",
       " ('dropthis', 1),\n",
       " ('naturallyderived', 9),\n",
       " ('krispies', 1),\n",
       " ('zaza', 1),\n",
       " ('antiwear', 2),\n",
       " ('colorblondebrown', 1),\n",
       " ('jacob', 120),\n",
       " ('limted', 2),\n",
       " ('brandgentle', 1),\n",
       " ('refered', 3),\n",
       " ('silverholographic', 1),\n",
       " ('retirement', 3),\n",
       " ('jagger', 2),\n",
       " ('mustardy', 2),\n",
       " ('maxen', 4),\n",
       " ('fanrm', 1),\n",
       " ('lnamed', 1),\n",
       " ('mitsubitshi', 1),\n",
       " ('senior', 11),\n",
       " ('iphonesplus', 14),\n",
       " ('gionara', 1),\n",
       " ('kmart', 4),\n",
       " ('pricing', 276),\n",
       " ('pealing', 17),\n",
       " ('jenners', 7),\n",
       " ('cfm', 1),\n",
       " ('monger', 1),\n",
       " ('shapehalf', 2),\n",
       " ('predrilled', 1),\n",
       " ('prynt', 1),\n",
       " ('yellow', 1558),\n",
       " ('slippersshoes', 1),\n",
       " ('banged', 12),\n",
       " ('montrose', 1),\n",
       " ('adudas', 1),\n",
       " ('ch', 42890),\n",
       " ('miscut', 1),\n",
       " ('imogen', 1),\n",
       " ('allclad', 1),\n",
       " ('beastbody', 1),\n",
       " ('gak', 5),\n",
       " ('salesthank', 1),\n",
       " ('chivechivette', 1),\n",
       " ('postagetracking', 4),\n",
       " ('usg', 16),\n",
       " ('dagashi', 1),\n",
       " ('juara', 2),\n",
       " ('streetstyle', 1),\n",
       " ('ime', 10577),\n",
       " ('joins', 1),\n",
       " ('belief', 1),\n",
       " ('pleating', 7),\n",
       " ('faulty', 7),\n",
       " ('foodie', 3),\n",
       " ('eft', 2076),\n",
       " ('suppressor', 1),\n",
       " ('unresealable', 1),\n",
       " ('plath', 2),\n",
       " ('wolly', 2),\n",
       " ('featuring', 202),\n",
       " ('blinking', 7),\n",
       " ('usy', 128),\n",
       " ('replacecorrect', 1),\n",
       " ('cuticles', 6),\n",
       " ('articuno', 3),\n",
       " ('fort', 3067),\n",
       " ('greengold', 4),\n",
       " ('sheiks', 1),\n",
       " ('stationtv', 1),\n",
       " ('tightensloosens', 1),\n",
       " ('leadville', 1),\n",
       " ('maxes', 4),\n",
       " ('ppops', 1),\n",
       " ('usateensupreme', 5),\n",
       " ('moment', 79),\n",
       " ('shamballa', 6),\n",
       " ('ivanka', 5),\n",
       " ('onnit', 1),\n",
       " ('jrd', 1),\n",
       " ('gentleman', 4),\n",
       " ('onsiepant', 2),\n",
       " ('mocsrose', 1),\n",
       " ('sandgria', 1),\n",
       " ('pumpedused', 1),\n",
       " ('screwbacks', 1),\n",
       " ('obsidianvoltpink', 1),\n",
       " ('boda', 1),\n",
       " ('subsitude', 1),\n",
       " ('leveled', 1),\n",
       " ('cubevit', 1),\n",
       " ('shick', 1),\n",
       " ('personalization', 6),\n",
       " ('winsor', 1),\n",
       " ('travelsize', 6),\n",
       " ('tinstar', 1),\n",
       " ('davidoff', 3),\n",
       " ('popularity', 7),\n",
       " ('beltsize', 1),\n",
       " ('tankie', 1),\n",
       " ('dermologica', 3),\n",
       " ('shapeups', 1),\n",
       " ('karmala', 1),\n",
       " ('fist', 21),\n",
       " ('uss', 917),\n",
       " ('armaldo', 1),\n",
       " ('privately', 1),\n",
       " ('traditional', 98),\n",
       " ('picker', 7),\n",
       " ('minimumsupport', 1),\n",
       " ('revolvers', 1),\n",
       " ('mlfl', 9),\n",
       " ('rickeybobby', 1),\n",
       " ('tsx', 4),\n",
       " ('geoffrey', 4),\n",
       " ('lowestlowballsprice', 1),\n",
       " ('jawdropping', 1),\n",
       " ('nwtfree', 2),\n",
       " ('zapper', 2),\n",
       " ('rainbows', 10),\n",
       " ('rhymes', 2),\n",
       " ('dragonfly', 16),\n",
       " ('seperating', 4),\n",
       " ('midrif', 2),\n",
       " ('alguronic', 1),\n",
       " ('tagsvansvolcomdiamondstussynike', 1),\n",
       " ('flared', 50),\n",
       " ('should', 1721),\n",
       " ('sunsetslimesco', 1),\n",
       " ('antioxidize', 1),\n",
       " ('deterioration', 1),\n",
       " ('barefooted', 1),\n",
       " ('schooling', 2),\n",
       " ('auvio', 1),\n",
       " ('monokini', 16),\n",
       " ('angk', 1),\n",
       " ('wantedneeded', 1),\n",
       " ('outgreat', 1),\n",
       " ('khumbu', 2),\n",
       " ('specsmens', 1),\n",
       " ('fitting', 285),\n",
       " ('includespurple', 1),\n",
       " ('salesman', 3),\n",
       " ('biti', 11),\n",
       " ('jefferycampbell', 1),\n",
       " ('durex', 1),\n",
       " ('stmped', 1),\n",
       " ('delux', 318),\n",
       " ('mintydelicious', 1),\n",
       " ('aspect', 6),\n",
       " ('purplepinkblack', 1),\n",
       " ('unbiased', 1),\n",
       " ('dvd', 468),\n",
       " ('tommybahama', 1),\n",
       " ('logos', 77),\n",
       " ('wornwashedtried', 4),\n",
       " ('drugs', 15),\n",
       " ('vs', 2286),\n",
       " ('smallhalf', 1),\n",
       " ('gomes', 2),\n",
       " ('shaver', 9),\n",
       " ('gerbils', 2),\n",
       " ('hammmoks', 1),\n",
       " ('tudors', 1),\n",
       " ('overlooked', 5),\n",
       " ('acnefighting', 1),\n",
       " ('hookups', 7),\n",
       " ('pinkacolada', 1),\n",
       " ('fingerless', 13),\n",
       " ('gravaton', 1),\n",
       " ('mantine', 1),\n",
       " ('sodi', 11),\n",
       " ('kinect', 17),\n",
       " ('ssplusplussse', 5),\n",
       " ('consuming', 9),\n",
       " ('prr', 1),\n",
       " ('holdss', 9),\n",
       " ('elemental', 1),\n",
       " ('belif', 4),\n",
       " ('squished', 11),\n",
       " ('cuisinarts', 1),\n",
       " ('phalates', 1),\n",
       " ('eachremover', 1),\n",
       " ('maddy', 2),\n",
       " ('piel', 3),\n",
       " ('recoup', 11),\n",
       " ('corpus', 1),\n",
       " ('wearpetsmoke', 1),\n",
       " ('including', 691),\n",
       " ('aaas', 1),\n",
       " ('droar', 1),\n",
       " ('confirm', 141),\n",
       " ('blow', 127),\n",
       " ('jewelry', 953),\n",
       " ('leotard', 48),\n",
       " ('holbrook', 3),\n",
       " ('mattewear', 4),\n",
       " ('adherent', 1),\n",
       " ('gingembre', 1),\n",
       " ('tuscaloosa', 1),\n",
       " ('aline', 105),\n",
       " ('draculaura', 4),\n",
       " ('likethank', 4),\n",
       " ('joggerreg', 1),\n",
       " ('noonsa', 1),\n",
       " ('eclipse', 21),\n",
       " ('titaniumis', 2),\n",
       " ('paraffins', 1),\n",
       " ('notrading', 3),\n",
       " ('task', 43),\n",
       " ('pcscar', 4),\n",
       " ('protectshine', 1),\n",
       " ('diet', 54),\n",
       " ('conforming', 2),\n",
       " ('silverzinc', 1),\n",
       " ('sharpen', 81),\n",
       " ('unwatched', 1),\n",
       " ('sauce', 32),\n",
       " ('kenway', 1),\n",
       " ('stayed', 10),\n",
       " ('rescued', 4),\n",
       " ('scuffscratchesmarksinitials', 1),\n",
       " ('needthank', 1),\n",
       " ('oxford', 25),\n",
       " ('intersection', 3),\n",
       " ('vein', 35),\n",
       " ('saxophone', 1),\n",
       " ('rmnike', 1),\n",
       " ('betrayal', 1),\n",
       " ('mariachi', 1),\n",
       " ('keeper', 21),\n",
       " ('memopad', 1),\n",
       " ('franchescas', 1),\n",
       " ('demension', 1),\n",
       " ('humans', 9),\n",
       " ('mayor', 6),\n",
       " ('giving', 260),\n",
       " ('rollerballs', 5),\n",
       " ('quavo', 1),\n",
       " ('wd', 1087),\n",
       " ('canteen', 4),\n",
       " ('payton', 5),\n",
       " ('dabbs', 1),\n",
       " ('pajamas', 168),\n",
       " ('pl', 28310),\n",
       " ('jeansdark', 1),\n",
       " ('feedba', 245),\n",
       " ('sinuses', 2),\n",
       " ('nightshirts', 1),\n",
       " ('winking', 2),\n",
       " ('pooped', 1),\n",
       " ('mikamarket', 1),\n",
       " ('wornwill', 1),\n",
       " ('played', 539),\n",
       " ('tupelo', 1),\n",
       " ('dressy', 89),\n",
       " ('disclosed', 6),\n",
       " ('requirement', 21),\n",
       " ('trstique', 1),\n",
       " ('sexxy', 1),\n",
       " ('selfsun', 1),\n",
       " ('mayb', 946),\n",
       " ('funktown', 1),\n",
       " ('freeunbranded', 1),\n",
       " ('dogeared', 8),\n",
       " ('tunisian', 2),\n",
       " ('redskin', 10),\n",
       " ('operatesimply', 1),\n",
       " ('hours', 1293),\n",
       " ('championcolumbia', 1),\n",
       " ('alpenglow', 1),\n",
       " ('earnings', 4),\n",
       " ('carbonite', 2),\n",
       " ('weft', 27),\n",
       " ('ccfl', 2),\n",
       " ('mintnear', 6),\n",
       " ('mater', 3621),\n",
       " ('comuter', 1),\n",
       " ('decorations', 61),\n",
       " ('exeggutor', 1),\n",
       " ('packrat', 1),\n",
       " ('miropure', 1),\n",
       " ('increases', 42),\n",
       " ('roombarshopcaferestaurant', 1),\n",
       " ('spescification', 1),\n",
       " ('philadelphia', 23),\n",
       " ('manufactactured', 1),\n",
       " ('cutlery', 7),\n",
       " ('comfortable', 1962),\n",
       " ('nevet', 1),\n",
       " ('elimination', 2),\n",
       " ('hometown', 3),\n",
       " ('another', 710),\n",
       " ('oncemore', 1),\n",
       " ('wanderlust', 9),\n",
       " ('tied', 106),\n",
       " ('eeboo', 2),\n",
       " ('extemely', 1),\n",
       " ('breakpoint', 4),\n",
       " ('maverik', 1),\n",
       " ('arealike', 1),\n",
       " ('mike', 44),\n",
       " ('celebrates', 4),\n",
       " ('shortsleeve', 18),\n",
       " ('apr', 541),\n",
       " ('fleck', 18),\n",
       " ('benjamins', 3),\n",
       " ('recreating', 1),\n",
       " ('sleevespack', 1),\n",
       " ('hadid', 3),\n",
       " ('neptune', 4),\n",
       " ('semimatte', 1),\n",
       " ('placevery', 1),\n",
       " ('ingredient', 364),\n",
       " ('ultrawaterproof', 1),\n",
       " ('countrythese', 1),\n",
       " ('maybelinne', 1),\n",
       " ('belongs', 9),\n",
       " ('lanyardsrm', 2),\n",
       " ('seasoning', 2),\n",
       " ('hipkini', 2),\n",
       " ('shockproof', 72),\n",
       " ('balmozml', 1),\n",
       " ('determined', 31),\n",
       " ('clfveff', 1),\n",
       " ('greeb', 1),\n",
       " ('allround', 1),\n",
       " ('medusa', 9),\n",
       " ('jars', 52),\n",
       " ('buttersoft', 1),\n",
       " ('steck', 1),\n",
       " ('fragrancesit', 1),\n",
       " ('agez', 2),\n",
       " ('unlaced', 2),\n",
       " ('jamaal', 1),\n",
       " ('wmatching', 2),\n",
       " ('llysine', 1),\n",
       " ('energizing', 7),\n",
       " ('elasticfree', 4),\n",
       " ('donkey', 27),\n",
       " ('leggingrm', 1),\n",
       " ('limitation', 3),\n",
       " ('snowpack', 1),\n",
       " ('crayons', 15),\n",
       " ('powervolume', 1),\n",
       " ('journee', 2),\n",
       " ('moshi', 1),\n",
       " ('editionbeautymakeupcosmeticeyeshadow', 1),\n",
       " ('acre', 19),\n",
       " ('fraiche', 2),\n",
       " ('instaxmini', 1),\n",
       " ('springlock', 1),\n",
       " ('frizzers', 1),\n",
       " ('redmedium', 1),\n",
       " ('bothers', 3),\n",
       " ('wristlet', 322),\n",
       " ('typeregular', 4),\n",
       " ('impulsive', 3),\n",
       " ('boyshortboxer', 1),\n",
       " ('collectionits', 1),\n",
       " ('fut', 108),\n",
       " ('strange', 37),\n",
       " ('lowlong', 2),\n",
       " ('preface', 1),\n",
       " ('solotica', 2),\n",
       " ('minami', 1),\n",
       " ('lf', 3354),\n",
       " ('hesss', 1),\n",
       " ('availbale', 5),\n",
       " ('pains', 8),\n",
       " ('dirtsome', 1),\n",
       " ('designsfrom', 1),\n",
       " ('sftroseta', 1),\n",
       " ('gm', 611),\n",
       " ('chilly', 27),\n",
       " ('remedial', 1),\n",
       " ('orangepink', 7),\n",
       " ('luthor', 1),\n",
       " ('cutsslits', 1),\n",
       " ('overlay', 84),\n",
       " ('troublemakin', 1),\n",
       " ('fashin', 2),\n",
       " ('tagscomes', 1),\n",
       " ('wornclimbing', 1),\n",
       " ('revitalizes', 3),\n",
       " ('squat', 14),\n",
       " ('lowballingno', 1),\n",
       " ('lexar', 2),\n",
       " ('meggie', 1),\n",
       " ('weighingdown', 1),\n",
       " ('todos', 1),\n",
       " ('frankenstein', 10),\n",
       " ('foams', 12),\n",
       " ('breakup', 4),\n",
       " ('shortsworn', 1),\n",
       " ('earlier', 9),\n",
       " ('atva', 2),\n",
       " ('heated', 27),\n",
       " ('msf', 1),\n",
       " ('timelessness', 1),\n",
       " ('prospects', 4),\n",
       " ('concoction', 1),\n",
       " ('greenblacktealorangepink', 1),\n",
       " ('bodysuittop', 1),\n",
       " ('loking', 2),\n",
       " ('snug', 221),\n",
       " ('tiramisu', 2),\n",
       " ('does', 2932),\n",
       " ('causes', 18),\n",
       " ('tacky', 4),\n",
       " ('unnoticeable', 71),\n",
       " ('soray', 1),\n",
       " ('newsome', 1),\n",
       " ('sweatproof', 11),\n",
       " ('sandi', 16),\n",
       " ('keplinger', 1),\n",
       " ('ssc', 146),\n",
       " ('troyer', 4),\n",
       " ('tomorrows', 2),\n",
       " ('familys', 3),\n",
       " ('penetrate', 21),\n",
       " ('cardigansize', 1),\n",
       " ('jdawn', 1),\n",
       " ('allot', 5),\n",
       " ('impressive', 7),\n",
       " ('prior', 726),\n",
       " ('furryfuzzy', 1),\n",
       " ('gomez', 5),\n",
       " ('inplace', 2),\n",
       " ('greensteals', 1),\n",
       " ('consistently', 10),\n",
       " ('peel', 486),\n",
       " ('composite', 14),\n",
       " ('egreen', 58),\n",
       " ('blushnude', 1),\n",
       " ('musicly', 1),\n",
       " ('liptick', 1),\n",
       " ('hanks', 2293),\n",
       " ('healing', 125),\n",
       " ('straining', 2),\n",
       " ('pluscase', 1),\n",
       " ('arrest', 4),\n",
       " ('silverspoon', 1),\n",
       " ('speedbox', 1),\n",
       " ('photosthe', 1),\n",
       " ('bittersweet', 8),\n",
       " ('piping', 44),\n",
       " ('singnature', 1),\n",
       " ('zzzzzz', 1),\n",
       " ('beauti', 3693),\n",
       " ('mkt', 2),\n",
       " ('airflow', 25),\n",
       " ('bother', 37),\n",
       " ('lightfair', 3),\n",
       " ('kilo', 7),\n",
       " ('sprocket', 1),\n",
       " ('meloncholy', 1),\n",
       " ('straightend', 1),\n",
       " ('frontblack', 2),\n",
       " ('wootcom', 1),\n",
       " ('envious', 2),\n",
       " ('excadrill', 1),\n",
       " ('stopper', 32),\n",
       " ('fita', 13),\n",
       " ('spansy', 1),\n",
       " ('licks', 20),\n",
       " ('mashems', 2),\n",
       " ('lakeshoredrive', 1),\n",
       " ('glasbake', 5),\n",
       " ('chokerfree', 1),\n",
       " ('beigeebonymocha', 1),\n",
       " ('hiphuggersbikini', 1),\n",
       " ('daniella', 1),\n",
       " ('bongo', 21),\n",
       " ('officeappropriate', 1),\n",
       " ('jock', 10),\n",
       " ('usedwashed', 1),\n",
       " ('athleisure', 4),\n",
       " ('hyacinth', 5),\n",
       " ('braveman', 1),\n",
       " ('wateraround', 1),\n",
       " ('hairs', 118),\n",
       " ('freetress', 2),\n",
       " ('anima', 537),\n",
       " ('freebird', 6),\n",
       " ('unflavored', 2),\n",
       " ('frenzy', 2),\n",
       " ('lamo', 87),\n",
       " ('harty', 1),\n",
       " ('edgar', 1),\n",
       " ('softside', 1),\n",
       " ('workplay', 1),\n",
       " ('webkins', 2),\n",
       " ('cheaply', 7),\n",
       " ('rorschach', 2),\n",
       " ('qualitycomfortable', 1),\n",
       " ('blinc', 1),\n",
       " ('aropostale', 3),\n",
       " ('boxreasonable', 1),\n",
       " ('seneserum', 1),\n",
       " ('wags', 1),\n",
       " ('blueredwhiteblack', 1),\n",
       " ('javelin', 1),\n",
       " ('lomo', 12),\n",
       " ('chipwrecked', 1),\n",
       " ('glecks', 1),\n",
       " ('diameter', 284),\n",
       " ('became', 11),\n",
       " ('footy', 2),\n",
       " ('frogskinz', 1),\n",
       " ('switchusb', 1),\n",
       " ('perch', 22),\n",
       " ('bandouliere', 2),\n",
       " ('answerend', 5),\n",
       " ('hygienically', 2),\n",
       " ('balancing', 25),\n",
       " ('theoretical', 1),\n",
       " ('miu', 450),\n",
       " ('friendship', 21),\n",
       " ('dl', 17663),\n",
       " ('ripstearsholesstainssmells', 1),\n",
       " ('sue', 1292),\n",
       " ('slxlxl', 1),\n",
       " ('closure', 1245),\n",
       " ('resembles', 6),\n",
       " ('zuilily', 1),\n",
       " ('sensuously', 1),\n",
       " ('fitabaa', 6),\n",
       " ('coffe', 414),\n",
       " ('strapsnever', 1),\n",
       " ('windstopper', 1),\n",
       " ('tester', 73),\n",
       " ('sanlorenzo', 1),\n",
       " ('intoxicates', 1),\n",
       " ('sellebsell', 1),\n",
       " ('skybox', 2),\n",
       " ('thickens', 9),\n",
       " ('wattage', 13),\n",
       " ('maintaining', 10),\n",
       " ('syncing', 6),\n",
       " ('maximizes', 12),\n",
       " ('swordready', 1),\n",
       " ('tantalizes', 1),\n",
       " ('nostains', 1),\n",
       " ('snowshoes', 1),\n",
       " ('burguny', 1),\n",
       " ('blacklace', 3),\n",
       " ('tex', 975),\n",
       " ('seeming', 3),\n",
       " ('lipsticklips', 5),\n",
       " ('occur', 108),\n",
       " ('smoothie', 22),\n",
       " ('aristocrats', 1),\n",
       " ('milkmaid', 1),\n",
       " ('moodlightening', 2),\n",
       " ('exposed', 122),\n",
       " ('skimming', 5),\n",
       " ('pail', 13),\n",
       " ('scraper', 7),\n",
       " ('textnow', 1),\n",
       " ('midlow', 2),\n",
       " ('silvercreamnavy', 2),\n",
       " ('turnlock', 7),\n",
       " ('gios', 1),\n",
       " ('fangon', 1),\n",
       " ('leau', 14),\n",
       " ('warner', 11),\n",
       " ('kymajesty', 2),\n",
       " ('legit', 29),\n",
       " ('discipline', 1),\n",
       " ('coloreddown', 1),\n",
       " ('aloneor', 1),\n",
       " ('arranged', 14),\n",
       " ('seedweed', 1),\n",
       " ('tabledepends', 1),\n",
       " ('vague', 1),\n",
       " ('supressed', 1),\n",
       " ('abgnac', 2),\n",
       " ('provides', 452),\n",
       " ('nevada', 3),\n",
       " ('platformheels', 1),\n",
       " ('aburn', 1),\n",
       " ('affiliated', 27),\n",
       " ('flora', 1256),\n",
       " ('landry', 2),\n",
       " ('farah', 2),\n",
       " ('heading', 5),\n",
       " ('ftm', 7),\n",
       " ('fanatics', 6),\n",
       " ('ent', 26836),\n",
       " ('routers', 2),\n",
       " ('cradling', 1),\n",
       " ('kidgets', 4),\n",
       " ('handset', 7),\n",
       " ('restocked', 8),\n",
       " ('attributes', 4),\n",
       " ('pippen', 1),\n",
       " ('uplifting', 12),\n",
       " ('pancake', 9),\n",
       " ('shap', 2076),\n",
       " ('facewash', 1),\n",
       " ('crybabys', 1),\n",
       " ('stylecat', 1),\n",
       " ('donruss', 14),\n",
       " ('danager', 1),\n",
       " ('environtment', 1),\n",
       " ('lepordprint', 1),\n",
       " ('significant', 50),\n",
       " ('disinfects', 2),\n",
       " ('picsmo', 1),\n",
       " ('isub', 1),\n",
       " ('belnavis', 2),\n",
       " ('paitient', 1),\n",
       " ('flexees', 2),\n",
       " ('waterproof', 704),\n",
       " ('crohns', 1),\n",
       " ('starpless', 1),\n",
       " ('sexycool', 1),\n",
       " ('shafted', 1),\n",
       " ('nightdateparty', 1),\n",
       " ('azzaro', 1),\n",
       " ('orangeyyellow', 1),\n",
       " ('cottonon', 3),\n",
       " ('structures', 1),\n",
       " ('comfortablesee', 1),\n",
       " ('marsk', 5),\n",
       " ('winged', 17),\n",
       " ('purposeful', 2),\n",
       " ('nwotdidnt', 1),\n",
       " ('cleverly', 3),\n",
       " ('tball', 395),\n",
       " ('vintageinspired', 6),\n",
       " ('exiting', 1),\n",
       " ('tharp', 1),\n",
       " ('vspinkcarolina', 3),\n",
       " ('purchasingasking', 4),\n",
       " ('fatale', 1),\n",
       " ('shoulderarm', 1),\n",
       " ('sldr', 1),\n",
       " ('lacroix', 2),\n",
       " ('christine', 3),\n",
       " ('heelwedged', 1),\n",
       " ('unnumbered', 1),\n",
       " ('bakerie', 2),\n",
       " ('geti', 28),\n",
       " ('xoxoxoxo', 3),\n",
       " ('handmadehandcrafted', 6),\n",
       " ('removablity', 1),\n",
       " ('colouration', 5),\n",
       " ('nestabilities', 1),\n",
       " ('changeid', 1),\n",
       " ('heliotrope', 3),\n",
       " ('faces', 115),\n",
       " ('enlivens', 2),\n",
       " ('spanielcavalier', 1),\n",
       " ('smurf', 7),\n",
       " ('marksflaws', 2),\n",
       " ('wnat', 1),\n",
       " ('mga', 2),\n",
       " ('herniated', 1),\n",
       " ('aforementioned', 1),\n",
       " ('mckee', 2),\n",
       " ('doghouse', 2),\n",
       " ('everlasting', 28),\n",
       " ('unable', 83),\n",
       " ('seminew', 2),\n",
       " ('aremakeof', 1),\n",
       " ('tagless', 9),\n",
       " ('resmed', 2),\n",
       " ('laurelwreathgnn', 1),\n",
       " ('flatten', 24),\n",
       " ('postconsumer', 1),\n",
       " ('straightphoto', 1),\n",
       " ('ailment', 6),\n",
       " ('sarongpareo', 1),\n",
       " ('lota', 7),\n",
       " ('effectsincluding', 1),\n",
       " ('navypeach', 1),\n",
       " ('jostled', 2),\n",
       " ('anybut', 1),\n",
       " ('minuets', 1),\n",
       " ('legions', 7),\n",
       " ('vgucjacket', 1),\n",
       " ('tealonly', 1),\n",
       " ('appsiri', 1),\n",
       " ('kutcher', 1),\n",
       " ('daywear', 1),\n",
       " ('presiona', 1),\n",
       " ('colorone', 1),\n",
       " ('unpunched', 1),\n",
       " ('ofra', 59),\n",
       " ('cagoule', 1),\n",
       " ('airsneakers', 1),\n",
       " ('blankey', 1),\n",
       " ('hamper', 6),\n",
       " ('flowers', 433),\n",
       " ('jeanmichel', 3),\n",
       " ('mzprince', 1),\n",
       " ('marvelous', 12),\n",
       " ('cupcake', 132),\n",
       " ('bootsw', 2),\n",
       " ('alm', 1641),\n",
       " ('topdown', 3),\n",
       " ('pantsjeans', 1),\n",
       " ('saragrosk', 1),\n",
       " ('silks', 2),\n",
       " ('greyfloral', 1),\n",
       " ('permedyeshuman', 1),\n",
       " ('adhesivestickon', 1),\n",
       " ('chloelove', 1),\n",
       " ('acids', 23),\n",
       " ('ducks', 10),\n",
       " ('castemate', 1),\n",
       " ('satinfinish', 1),\n",
       " ('luna', 86),\n",
       " ('kxxlly', 1),\n",
       " ('cylinder', 4),\n",
       " ('oreos', 2),\n",
       " ('whispy', 5),\n",
       " ...]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# Clean up descriptions\n",
    "alpha_descriptions = list(map(lambda description: re.sub(r\"[^a-zA-Z ]\", '', description.lower()), random_sample['item_description']))\n",
    "\n",
    "# Unique words\n",
    "split_descriptions = map(lambda alpha: alpha.split(), iter(alpha_descriptions))\n",
    "all_keywords = [keyword for description in split_descriptions for keyword in description]\n",
    "unique_keywords = set(all_keywords)\n",
    "\n",
    "# Number of items with each word (smaller sample)\n",
    "keyword_counts = pd.DataFrame(list(map(lambda keyword: (keyword, len([alpha for alpha in iter(alpha_descriptions) if keyword in alpha])), list(unique_keywords))))\n",
    "keyword_counts.columns = ['word', 'count']\n",
    "keyword_counts.sort_values(by=['count'], ascending=False).to_csv(\"/Users/David/Desktop/Mercari/keyword_counts.tsv\", sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29319</th>\n",
       "      <td>e</td>\n",
       "      <td>145619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41663</th>\n",
       "      <td>n</td>\n",
       "      <td>142016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42077</th>\n",
       "      <td>i</td>\n",
       "      <td>138924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5080</th>\n",
       "      <td>s</td>\n",
       "      <td>138580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17854</th>\n",
       "      <td>r</td>\n",
       "      <td>138063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59803</th>\n",
       "      <td>o</td>\n",
       "      <td>137047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46842</th>\n",
       "      <td>t</td>\n",
       "      <td>136152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21888</th>\n",
       "      <td>a</td>\n",
       "      <td>129472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3406</th>\n",
       "      <td>d</td>\n",
       "      <td>129011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59247</th>\n",
       "      <td>c</td>\n",
       "      <td>124543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30062</th>\n",
       "      <td>l</td>\n",
       "      <td>122147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39227</th>\n",
       "      <td>h</td>\n",
       "      <td>111269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46544</th>\n",
       "      <td>p</td>\n",
       "      <td>110424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32690</th>\n",
       "      <td>w</td>\n",
       "      <td>110054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1056</th>\n",
       "      <td>u</td>\n",
       "      <td>108081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52674</th>\n",
       "      <td>g</td>\n",
       "      <td>105915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17513</th>\n",
       "      <td>m</td>\n",
       "      <td>104351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33132</th>\n",
       "      <td>b</td>\n",
       "      <td>104018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39991</th>\n",
       "      <td>y</td>\n",
       "      <td>97498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6563</th>\n",
       "      <td>f</td>\n",
       "      <td>96322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16903</th>\n",
       "      <td>on</td>\n",
       "      <td>92726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14017</th>\n",
       "      <td>in</td>\n",
       "      <td>91995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13832</th>\n",
       "      <td>nd</td>\n",
       "      <td>91851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52060</th>\n",
       "      <td>an</td>\n",
       "      <td>88074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9972</th>\n",
       "      <td>er</td>\n",
       "      <td>86346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53004</th>\n",
       "      <td>or</td>\n",
       "      <td>85132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53836</th>\n",
       "      <td>re</td>\n",
       "      <td>84589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40377</th>\n",
       "      <td>k</td>\n",
       "      <td>83201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20849</th>\n",
       "      <td>it</td>\n",
       "      <td>82330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14251</th>\n",
       "      <td>th</td>\n",
       "      <td>79289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32103</th>\n",
       "      <td>ne</td>\n",
       "      <td>78989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27474</th>\n",
       "      <td>ti</td>\n",
       "      <td>78167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11001</th>\n",
       "      <td>es</td>\n",
       "      <td>75918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36084</th>\n",
       "      <td>le</td>\n",
       "      <td>74397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51347</th>\n",
       "      <td>v</td>\n",
       "      <td>72231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51334</th>\n",
       "      <td>and</td>\n",
       "      <td>69985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61188</th>\n",
       "      <td>ed</td>\n",
       "      <td>68393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14109</th>\n",
       "      <td>co</td>\n",
       "      <td>67271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10106</th>\n",
       "      <td>se</td>\n",
       "      <td>64769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27117</th>\n",
       "      <td>ea</td>\n",
       "      <td>63176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30982</th>\n",
       "      <td>st</td>\n",
       "      <td>63151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47370</th>\n",
       "      <td>he</td>\n",
       "      <td>63137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46565</th>\n",
       "      <td>si</td>\n",
       "      <td>62199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26523</th>\n",
       "      <td>al</td>\n",
       "      <td>61900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48130</th>\n",
       "      <td>ar</td>\n",
       "      <td>61558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24401</th>\n",
       "      <td>ll</td>\n",
       "      <td>61466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47614</th>\n",
       "      <td>ri</td>\n",
       "      <td>61344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44866</th>\n",
       "      <td>ra</td>\n",
       "      <td>60644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6039</th>\n",
       "      <td>te</td>\n",
       "      <td>60397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5300</th>\n",
       "      <td>ve</td>\n",
       "      <td>59984</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      word   count\n",
       "29319    e  145619\n",
       "41663    n  142016\n",
       "42077    i  138924\n",
       "5080     s  138580\n",
       "17854    r  138063\n",
       "59803    o  137047\n",
       "46842    t  136152\n",
       "21888    a  129472\n",
       "3406     d  129011\n",
       "59247    c  124543\n",
       "30062    l  122147\n",
       "39227    h  111269\n",
       "46544    p  110424\n",
       "32690    w  110054\n",
       "1056     u  108081\n",
       "52674    g  105915\n",
       "17513    m  104351\n",
       "33132    b  104018\n",
       "39991    y   97498\n",
       "6563     f   96322\n",
       "16903   on   92726\n",
       "14017   in   91995\n",
       "13832   nd   91851\n",
       "52060   an   88074\n",
       "9972    er   86346\n",
       "53004   or   85132\n",
       "53836   re   84589\n",
       "40377    k   83201\n",
       "20849   it   82330\n",
       "14251   th   79289\n",
       "32103   ne   78989\n",
       "27474   ti   78167\n",
       "11001   es   75918\n",
       "36084   le   74397\n",
       "51347    v   72231\n",
       "51334  and   69985\n",
       "61188   ed   68393\n",
       "14109   co   67271\n",
       "10106   se   64769\n",
       "27117   ea   63176\n",
       "30982   st   63151\n",
       "47370   he   63137\n",
       "46565   si   62199\n",
       "26523   al   61900\n",
       "48130   ar   61558\n",
       "24401   ll   61466\n",
       "47614   ri   61344\n",
       "44866   ra   60644\n",
       "6039    te   60397\n",
       "5300    ve   59984"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keyword_counts.sort_values(by=['count'], ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_id</th>\n",
       "      <th>name</th>\n",
       "      <th>item_condition_id</th>\n",
       "      <th>category_name</th>\n",
       "      <th>brand_name</th>\n",
       "      <th>price</th>\n",
       "      <th>shipping</th>\n",
       "      <th>item_description</th>\n",
       "      <th>random</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1039651</td>\n",
       "      <td>6 KLEANCOLOR MADLY MATTE LIQUID LIPSTICK</td>\n",
       "      <td>1</td>\n",
       "      <td>Beauty/Makeup/Lips</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0</td>\n",
       "      <td>New and sealed Kleancolor Madly Matte Lipgloss...</td>\n",
       "      <td>-4.943512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1183037</td>\n",
       "      <td>Clarks Brown Leather Loafers, Size 8.5 M</td>\n",
       "      <td>3</td>\n",
       "      <td>Women/Shoes/Loafers &amp; Slip-Ons</td>\n",
       "      <td>Clark's Botanicals</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Women's size 8.5 M. Leather upper, balance man...</td>\n",
       "      <td>-4.887879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1262715</td>\n",
       "      <td>Tory Burch Caroline Ballet Flats</td>\n",
       "      <td>3</td>\n",
       "      <td>Women/Shoes/Flats</td>\n",
       "      <td>Tory Burch</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Beautiful Caroline Ballet Flats by Tory Burch....</td>\n",
       "      <td>-4.719336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1048669</td>\n",
       "      <td>Lularoe foxes</td>\n",
       "      <td>1</td>\n",
       "      <td>Women/Athletic Apparel/Pants, Tights, Leggings</td>\n",
       "      <td>LuLaRoe</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Tc bnwt fox leggings!! Unicorn!!</td>\n",
       "      <td>-4.667091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>183748</td>\n",
       "      <td>Polo Ralph Lauren long sleeve</td>\n",
       "      <td>2</td>\n",
       "      <td>Men/Tops/T-shirts</td>\n",
       "      <td>Polo Ralph Lauren</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0</td>\n",
       "      <td>This is a white Polo Ralph Lauren long sleeve ...</td>\n",
       "      <td>-4.630884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1129241</td>\n",
       "      <td>Free Ship NWT PINK siz XS(AA-B) bralette</td>\n",
       "      <td>1</td>\n",
       "      <td>Women/Tops &amp; Blouses/Tank, Cami</td>\n",
       "      <td>Victoria's Secret</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Free Ship NWT PINK size XS(AA-B) bralette</td>\n",
       "      <td>-4.521999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>48634</td>\n",
       "      <td>GERARD COSMETICS *LAST DAY!! NO FEES</td>\n",
       "      <td>1</td>\n",
       "      <td>Beauty/Makeup/Face</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1</td>\n",
       "      <td>IS LISTING IS FOR 1 SETTING SPRAY GREEN TEA !!...</td>\n",
       "      <td>-4.471180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>811224</td>\n",
       "      <td>Exclusive for Nick!</td>\n",
       "      <td>3</td>\n",
       "      <td>Electronics/Video Games &amp; Consoles/Games</td>\n",
       "      <td>Nintendo</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Nick</td>\n",
       "      <td>-4.450309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>920549</td>\n",
       "      <td>4 Twilight movies</td>\n",
       "      <td>2</td>\n",
       "      <td>Electronics/Media/DVD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0</td>\n",
       "      <td>All 4 twilight movies except for Breaking Dawn...</td>\n",
       "      <td>-4.370084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1321060</td>\n",
       "      <td>NWT XS Lularoe Classic</td>\n",
       "      <td>1</td>\n",
       "      <td>Women/Tops &amp; Blouses/T-Shirts</td>\n",
       "      <td>LuLaRoe</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1</td>\n",
       "      <td>NWT XS Classic. Light blue</td>\n",
       "      <td>-4.347660</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_id                                      name  item_condition_id  \\\n",
       "0   1039651  6 KLEANCOLOR MADLY MATTE LIQUID LIPSTICK                  1   \n",
       "1   1183037  Clarks Brown Leather Loafers, Size 8.5 M                  3   \n",
       "2   1262715          Tory Burch Caroline Ballet Flats                  3   \n",
       "3   1048669                             Lularoe foxes                  1   \n",
       "4    183748             Polo Ralph Lauren long sleeve                  2   \n",
       "5   1129241  Free Ship NWT PINK siz XS(AA-B) bralette                  1   \n",
       "6     48634      GERARD COSMETICS *LAST DAY!! NO FEES                  1   \n",
       "7    811224                       Exclusive for Nick!                  3   \n",
       "8    920549                         4 Twilight movies                  2   \n",
       "9   1321060                    NWT XS Lularoe Classic                  1   \n",
       "\n",
       "                                    category_name          brand_name  price  \\\n",
       "0                              Beauty/Makeup/Lips                 NaN   10.0   \n",
       "1                  Women/Shoes/Loafers & Slip-Ons  Clark's Botanicals   41.0   \n",
       "2                               Women/Shoes/Flats          Tory Burch   26.0   \n",
       "3  Women/Athletic Apparel/Pants, Tights, Leggings             LuLaRoe   31.0   \n",
       "4                               Men/Tops/T-shirts   Polo Ralph Lauren   13.0   \n",
       "5                 Women/Tops & Blouses/Tank, Cami   Victoria's Secret   15.0   \n",
       "6                              Beauty/Makeup/Face                 NaN   20.0   \n",
       "7        Electronics/Video Games & Consoles/Games            Nintendo   10.0   \n",
       "8                           Electronics/Media/DVD                 NaN   10.0   \n",
       "9                   Women/Tops & Blouses/T-Shirts             LuLaRoe   20.0   \n",
       "\n",
       "   shipping                                   item_description    random  \n",
       "0         0  New and sealed Kleancolor Madly Matte Lipgloss... -4.943512  \n",
       "1         0  Women's size 8.5 M. Leather upper, balance man... -4.887879  \n",
       "2         0  Beautiful Caroline Ballet Flats by Tory Burch.... -4.719336  \n",
       "3         0                   Tc bnwt fox leggings!! Unicorn!! -4.667091  \n",
       "4         0  This is a white Polo Ralph Lauren long sleeve ... -4.630884  \n",
       "5         1          Free Ship NWT PINK size XS(AA-B) bralette -4.521999  \n",
       "6         1  IS LISTING IS FOR 1 SETTING SPRAY GREEN TEA !!... -4.471180  \n",
       "7         1                                               Nick -4.450309  \n",
       "8         0  All 4 twilight movies except for Breaking Dawn... -4.370084  \n",
       "9         1                         NWT XS Classic. Light blue -4.347660  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_sample.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2899"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reduced_random_sample = random_sample.loc[random_sample['category_name'].isin([\"Beauty/Makeup/Lips\", \"Women/Shoes/Loafers & Slip-Ons\", \"Men/Tops/T-shirts\"])]\n",
    "lips_sample = random_sample.loc[random_sample['category_name'].isin([\"Beauty/Makeup/Lips\"])]\n",
    "len(lips_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2112</th>\n",
       "      <td>e</td>\n",
       "      <td>2838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2854</th>\n",
       "      <td>n</td>\n",
       "      <td>2803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1023</th>\n",
       "      <td>s</td>\n",
       "      <td>2720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1990</th>\n",
       "      <td>r</td>\n",
       "      <td>2704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>i</td>\n",
       "      <td>2692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5793</th>\n",
       "      <td>d</td>\n",
       "      <td>2643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930</th>\n",
       "      <td>o</td>\n",
       "      <td>2614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3593</th>\n",
       "      <td>a</td>\n",
       "      <td>2584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4990</th>\n",
       "      <td>c</td>\n",
       "      <td>2467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5819</th>\n",
       "      <td>p</td>\n",
       "      <td>2441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4783</th>\n",
       "      <td>u</td>\n",
       "      <td>2291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4429</th>\n",
       "      <td>h</td>\n",
       "      <td>2244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1997</th>\n",
       "      <td>w</td>\n",
       "      <td>2196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>b</td>\n",
       "      <td>2171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2325</th>\n",
       "      <td>m</td>\n",
       "      <td>2153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3868</th>\n",
       "      <td>ne</td>\n",
       "      <td>2064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3304</th>\n",
       "      <td>in</td>\n",
       "      <td>1945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>li</td>\n",
       "      <td>1913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5671</th>\n",
       "      <td>an</td>\n",
       "      <td>1906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>540</th>\n",
       "      <td>g</td>\n",
       "      <td>1899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1502</th>\n",
       "      <td>f</td>\n",
       "      <td>1892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3546</th>\n",
       "      <td>ed</td>\n",
       "      <td>1837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5295</th>\n",
       "      <td>nd</td>\n",
       "      <td>1834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4600</th>\n",
       "      <td>k</td>\n",
       "      <td>1765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1910</th>\n",
       "      <td>or</td>\n",
       "      <td>1676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3710</th>\n",
       "      <td>and</td>\n",
       "      <td>1663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>on</td>\n",
       "      <td>1607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5148</th>\n",
       "      <td>st</td>\n",
       "      <td>1597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1522</th>\n",
       "      <td>lip</td>\n",
       "      <td>1585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4241</th>\n",
       "      <td>th</td>\n",
       "      <td>1559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1396</th>\n",
       "      <td>first</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4099</th>\n",
       "      <td>tarte</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5158</th>\n",
       "      <td>senegence</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5886</th>\n",
       "      <td>cover</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1860</th>\n",
       "      <td>without</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957</th>\n",
       "      <td>bar</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4800</th>\n",
       "      <td>lasts</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3428</th>\n",
       "      <td>bold</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>well</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2909</th>\n",
       "      <td>palette</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3591</th>\n",
       "      <td>lime</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3495</th>\n",
       "      <td>tube</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5868</th>\n",
       "      <td>rave</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3907</th>\n",
       "      <td>lower</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5805</th>\n",
       "      <td>holds</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2552</th>\n",
       "      <td>pretty</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4243</th>\n",
       "      <td>bundles</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5191</th>\n",
       "      <td>would</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5320</th>\n",
       "      <td>oops</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2680</th>\n",
       "      <td>jeffree</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4790</th>\n",
       "      <td>site</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5766</th>\n",
       "      <td>times</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2259</th>\n",
       "      <td>rad</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1193</th>\n",
       "      <td>gorgeous</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5358</th>\n",
       "      <td>apply</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5232</th>\n",
       "      <td>satin</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>good</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4026</th>\n",
       "      <td>depend</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>plump</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1528</th>\n",
       "      <td>ingredient</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>531 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            word  count\n",
       "2112           e   2838\n",
       "2854           n   2803\n",
       "1023           s   2720\n",
       "1990           r   2704\n",
       "21             i   2692\n",
       "5793           d   2643\n",
       "930            o   2614\n",
       "3593           a   2584\n",
       "4990           c   2467\n",
       "5819           p   2441\n",
       "4783           u   2291\n",
       "4429           h   2244\n",
       "1997           w   2196\n",
       "200            b   2171\n",
       "2325           m   2153\n",
       "3868          ne   2064\n",
       "3304          in   1945\n",
       "156           li   1913\n",
       "5671          an   1906\n",
       "540            g   1899\n",
       "1502           f   1892\n",
       "3546          ed   1837\n",
       "5295          nd   1834\n",
       "4600           k   1765\n",
       "1910          or   1676\n",
       "3710         and   1663\n",
       "177           on   1607\n",
       "5148          st   1597\n",
       "1522         lip   1585\n",
       "4241          th   1559\n",
       "...          ...    ...\n",
       "1396       first     55\n",
       "4099       tarte     55\n",
       "5158   senegence     55\n",
       "5886       cover     54\n",
       "1860     without     54\n",
       "1957         bar     54\n",
       "4800       lasts     54\n",
       "3428        bold     54\n",
       "291         well     54\n",
       "2909     palette     54\n",
       "3591        lime     54\n",
       "3495        tube     54\n",
       "5868        rave     53\n",
       "3907       lower     53\n",
       "5805       holds     53\n",
       "2552      pretty     52\n",
       "4243     bundles     52\n",
       "5191       would     52\n",
       "5320        oops     52\n",
       "2680     jeffree     52\n",
       "4790        site     52\n",
       "5766       times     52\n",
       "2259         rad     52\n",
       "1193    gorgeous     52\n",
       "5358       apply     51\n",
       "5232       satin     51\n",
       "13          good     51\n",
       "4026      depend     51\n",
       "437        plump     51\n",
       "1528  ingredient     51\n",
       "\n",
       "[531 rows x 2 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# Clean up descriptions\n",
    "alpha_descriptions = list(map(lambda description: re.sub(r\"[^a-zA-Z ]\", '', description.lower()), lips_sample['item_description']))\n",
    "\n",
    "# Unique words\n",
    "split_descriptions = map(lambda alpha: alpha.split(), iter(alpha_descriptions))\n",
    "all_keywords = [keyword for description in split_descriptions for keyword in description]\n",
    "unique_keywords = set(all_keywords)\n",
    "\n",
    "# Number of items with each word (smaller sample)\n",
    "keyword_counts = pd.DataFrame(list(map(lambda keyword: (keyword, len([alpha for alpha in iter(alpha_descriptions) if keyword in alpha])), list(unique_keywords))))\n",
    "keyword_counts.columns = ['word', 'count']\n",
    "significant_keywords = keyword_counts[keyword_counts['count'] > 50].sort_values(by=['count'], ascending=False)\n",
    "significant_keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TODO: Lowercase, strip your descriptions. Though I bet exclamation points are predictive. What about other characters/emojis? Spelling errors would be an intersting thing to model too\n",
    "lips_sample[lips_sample['item_description'].str.contains(\"apply\")].head(10)[\"item_description\"].to_csv(\"/Users/David/Desktop/Mercari/lips_sample_descriptions.tsv\", sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lips_sample.head(10)\n",
    "\n",
    "import re\n",
    "alpha_descriptions = list(map(lambda description: re.sub(r\"[^a-zA-Z ]\", '', description.lower()), lips_sample['item_description']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         New and sealed Kleancolor Madly Matte Lipgloss...\n",
       "30        Two Brand new in box MAC Lipglass viva glam Mi...\n",
       "202       Mac Mariah Carey DREAMLOVER Dirty pink shimmer...\n",
       "212       Completely brand new and 100% authentic, I hav...\n",
       "257                                      No description yet\n",
       "295       Bundle 2 colors for [rm] and 3 for [rm]! With ...\n",
       "311       KYLIE JENNER MATTE LIQUID LIPSTICK & LINER Con...\n",
       "341       Swatched once on the back of my hand Comes wit...\n",
       "352       Authentic SeneGence LipSense Bundle Color - Be...\n",
       "437                                             Never used.\n",
       "491       Brand New Free Shipping Includes 10 long lasti...\n",
       "507       Long lasting up to 18 hours! [rm] Glosses and ...\n",
       "538       3 Matte Liquid Lipsticks (0.11 fl oz./oz. liq ...\n",
       "622       LipSense by SeneGence Kiss Proof - Smudge Proo...\n",
       "663       This listing includes the liner and lipstick. ...\n",
       "664                Price for 4 lipsticks Choose your colors\n",
       "714       Brand New in Box. Limited Edition Tarte Tartei...\n",
       "726                             used once. great condition.\n",
       "820       Shades are: Teddy, Honeymoon, Embellishment, B...\n",
       "834       Bottle is new/unused. Apply to lips and wait 1...\n",
       "973       Price is for 2 lipsticks #2 embellishment is s...\n",
       "983       Jeffree Star 2015 Holiday collection, limited ...\n",
       "1002                                     No description yet\n",
       "1003                                                 All 10\n",
       "1034                 Used contains 60 -75 percent authentic\n",
       "1163      PLEASE READ DESCRIPTION ON 4TH PIC BEFORE PURC...\n",
       "1177      2 Matte Lipstick SingleS- Koko K and Dolce K B...\n",
       "1213      Lipsense glossy to go on top of any color to h...\n",
       "1249      NWT; Get whiter teeth in just 5 minutes with t...\n",
       "1347                                     No description yet\n",
       "                                ...                        \n",
       "146903    **BRAND NEW, IN BOX (the box is a bit beat up)...\n",
       "146984    Brand new, never swatched! Tarte Cosmetics Rai...\n",
       "146987    4 un one Lip Reviver Brand New Full Size Free ...\n",
       "146989    Limited Edition Rare Very Htf Cookies & Cream ...\n",
       "147033                                   No description yet\n",
       "147100                                   No description yet\n",
       "147157                                           Rose gloss\n",
       "147164                 3 flavored lipgloss and 1 pink gloss\n",
       "147182                   Beige champagne and Napa for Lisa!\n",
       "147209                            Pink lipstick. Brand new.\n",
       "147242    New Como la Flor mac Cosmetics Will ship as so...\n",
       "147251    ✨✨PLEASE READ MY BIO BEFORE BUYING FROM MY CLO...\n",
       "147293                    New comes w lipstick and lipliner\n",
       "147324                                   No description yet\n",
       "147590                           SCORPIO- Jeffree Star dupe\n",
       "147693                  \"Valentine\" Yes it's 100% authentic\n",
       "147792    Retail [rm] No NEGOTIATIONS or DISCOUNTS Alway...\n",
       "147855         Matte liquid lipgloss in Dusty Rose and Doll\n",
       "147868                                     Comes with all 3\n",
       "147892                      Never been open color is corset\n",
       "147915    Victorias Secret Beauty Rush Flavored Lip Glos...\n",
       "147935    2 brand new SEALED lipsticks! Colors are Magen...\n",
       "148030    LipSense by Senegence is a long lasting lip co...\n",
       "148073                             City Lips Plumping gloss\n",
       "148172                      Brand new Kourt k Free shipping\n",
       "148205    -Free Shipping + New Condition Benefits:moist ...\n",
       "148207    Full size Colors, Exposed, Bestie, Front Row, ...\n",
       "148217    SET OF 8 *RETAIL PRICE FOR EACH IS [rm] Perry ...\n",
       "148229    Brand New NYX Nude Matte Bundle With Nyx Lip L...\n",
       "148239                            New and sealed! Authentic\n",
       "Name: item_description, Length: 2899, dtype: object"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lips_sample['item_description']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "alpha_description_df = pd.DataFrame(alpha_descriptions)\n",
    "alpha_description_df[0]\n",
    "\n",
    "lips_sample['item_description'] = alpha_description_df[0].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         new and sealed kleancolor madly matte lipgloss...\n",
       "30        two brand new in box mac lipglass viva glam mi...\n",
       "202       mac mariah carey dreamlover dirty pink shimmer...\n",
       "212       completely brand new and  authentic i have the...\n",
       "257                                      no description yet\n",
       "295       bundle  colors for rm and  for rm with purchas...\n",
       "311       kylie jenner matte liquid lipstick  liner cont...\n",
       "341       swatched once on the back of my hand comes wit...\n",
       "352       authentic senegence lipsense bundle color  bel...\n",
       "437                                              never used\n",
       "491       brand new free shipping includes  long lasting...\n",
       "507       long lasting up to  hours rm glosses and other...\n",
       "538        matte liquid lipsticks  fl ozoz liq   ml each...\n",
       "622       lipsense by senegence kiss proof  smudge proof...\n",
       "663       this listing includes the liner and lipstick c...\n",
       "664                 price for  lipsticks choose your colors\n",
       "714       brand new in box limited edition tarte tarteis...\n",
       "726                               used once great condition\n",
       "820       shades are teddy honeymoon embellishment bedti...\n",
       "834       bottle is newunused apply to lips and wait  mi...\n",
       "973       price is for  lipsticks  embellishment is sold...\n",
       "983       jeffree star  holiday collection limited editi...\n",
       "1002                                     no description yet\n",
       "1003                                                   all \n",
       "1034                      used contains   percent authentic\n",
       "1163      please read description on th pic before purch...\n",
       "1177       matte lipstick singles koko k and dolce k bra...\n",
       "1213      lipsense glossy to go on top of any color to h...\n",
       "1249      nwt get whiter teeth in just  minutes with thi...\n",
       "1347                                     no description yet\n",
       "                                ...                        \n",
       "146903    brand new in box the box is a bit beat up neve...\n",
       "146984    brand new never swatched tarte cosmetics rainf...\n",
       "146987     un one lip reviver brand new full size free s...\n",
       "146989    limited edition rare very htf cookies  cream c...\n",
       "147033                                   no description yet\n",
       "147100                                   no description yet\n",
       "147157                                           rose gloss\n",
       "147164                    flavored lipgloss and  pink gloss\n",
       "147182                    beige champagne and napa for lisa\n",
       "147209                              pink lipstick brand new\n",
       "147242    new como la flor mac cosmetics will ship as so...\n",
       "147251    please read my bio before buying from my close...\n",
       "147293                    new comes w lipstick and lipliner\n",
       "147324                                   no description yet\n",
       "147590                            scorpio jeffree star dupe\n",
       "147693                         valentine yes its  authentic\n",
       "147792    retail rm no negotiations or discounts always ...\n",
       "147855         matte liquid lipgloss in dusty rose and doll\n",
       "147868                                      comes with all \n",
       "147892                      never been open color is corset\n",
       "147915    victorias secret beauty rush flavored lip glos...\n",
       "147935     brand new sealed lipsticks colors are magenta...\n",
       "148030    lipsense by senegence is a long lasting lip co...\n",
       "148073                             city lips plumping gloss\n",
       "148172                      brand new kourt k free shipping\n",
       "148205    free shipping  new condition benefitsmoist lip...\n",
       "148207    full size colors exposed bestie front row get ...\n",
       "148217    set of  retail price for each is rm perry pant...\n",
       "148229    brand new nyx nude matte bundle with nyx lip l...\n",
       "148239                             new and sealed authentic\n",
       "Name: item_description, Length: 2899, dtype: object"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lips_sample['item_description']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "multicharacter_keywords = list(filter(lambda row: len(row[0]) != 1, significant_keywords.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multicharacter_keywords[0][0] in lip_sample['item_descriptons'][0].split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "lips_sample['item_description'].values[0]\n",
    "\n",
    "split_description = [description.split() for description in lips_sample['item_description'].values]\n",
    "lips_sample['split_description'] = split_description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"kleancolor\" in lips_sample['split_description'].values\n",
    "\n",
    "kleancolor = [item for item in lips_sample.values if \"kleancolor\" in item[9]]\n",
    "\n",
    "import numpy as np\n",
    "np.median([item[5] for item in kleancolor])\n",
    "\n",
    "# This is to get all item associated with a keyword—you need to restructure multicharacter_keywords (it's actually keyword and count)\n",
    "# list(map(lambda keyword: (keyword, [item for item in lips_sample.values if keyword in item[9]]), multicharacter_keywords[0]))\n",
    "\n",
    "# This was an attempt to get median price by keyword\n",
    "# list(map(lambda keyword: (keyword, np.median([item[5] for item in lips_sample.values if keyword in item[9]])), multicharacter_keywords[0]))\n",
    "keyword_prices = list(map(lambda keyword: (keyword, np.median([item[5] for item in lips_sample.values if keyword in item[9]])), [record[0] for record in multicharacter_keywords]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>deviation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ne</td>\n",
       "      <td>13.00</td>\n",
       "      <td>-1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>in</td>\n",
       "      <td>14.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>li</td>\n",
       "      <td>31.00</td>\n",
       "      <td>17.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>an</td>\n",
       "      <td>15.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ed</td>\n",
       "      <td>11.00</td>\n",
       "      <td>-3.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>nd</td>\n",
       "      <td>17.00</td>\n",
       "      <td>3.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>or</td>\n",
       "      <td>14.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>and</td>\n",
       "      <td>15.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>on</td>\n",
       "      <td>13.00</td>\n",
       "      <td>-1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>st</td>\n",
       "      <td>10.00</td>\n",
       "      <td>-4.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>lip</td>\n",
       "      <td>13.00</td>\n",
       "      <td>-1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>th</td>\n",
       "      <td>20.00</td>\n",
       "      <td>6.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>he</td>\n",
       "      <td>14.50</td>\n",
       "      <td>0.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>new</td>\n",
       "      <td>15.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>co</td>\n",
       "      <td>7.00</td>\n",
       "      <td>-7.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>le</td>\n",
       "      <td>18.00</td>\n",
       "      <td>4.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>sh</td>\n",
       "      <td>17.50</td>\n",
       "      <td>3.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>at</td>\n",
       "      <td>13.00</td>\n",
       "      <td>-1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>it</td>\n",
       "      <td>14.25</td>\n",
       "      <td>0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ll</td>\n",
       "      <td>15.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>us</td>\n",
       "      <td>11.00</td>\n",
       "      <td>-3.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>is</td>\n",
       "      <td>14.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ea</td>\n",
       "      <td>14.50</td>\n",
       "      <td>0.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>hi</td>\n",
       "      <td>26.00</td>\n",
       "      <td>12.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>nt</td>\n",
       "      <td>10.00</td>\n",
       "      <td>-4.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>the</td>\n",
       "      <td>14.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>de</td>\n",
       "      <td>39.50</td>\n",
       "      <td>25.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ps</td>\n",
       "      <td>28.00</td>\n",
       "      <td>14.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>me</td>\n",
       "      <td>15.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>as</td>\n",
       "      <td>13.00</td>\n",
       "      <td>-1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>first</td>\n",
       "      <td>20.00</td>\n",
       "      <td>6.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480</th>\n",
       "      <td>tarte</td>\n",
       "      <td>15.50</td>\n",
       "      <td>1.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>481</th>\n",
       "      <td>senegence</td>\n",
       "      <td>27.00</td>\n",
       "      <td>13.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482</th>\n",
       "      <td>cover</td>\n",
       "      <td>19.00</td>\n",
       "      <td>5.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483</th>\n",
       "      <td>without</td>\n",
       "      <td>12.50</td>\n",
       "      <td>-1.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>bar</td>\n",
       "      <td>13.00</td>\n",
       "      <td>-1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>lasts</td>\n",
       "      <td>20.00</td>\n",
       "      <td>6.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486</th>\n",
       "      <td>bold</td>\n",
       "      <td>16.00</td>\n",
       "      <td>2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>well</td>\n",
       "      <td>12.00</td>\n",
       "      <td>-2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>palette</td>\n",
       "      <td>12.00</td>\n",
       "      <td>-2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489</th>\n",
       "      <td>lime</td>\n",
       "      <td>12.00</td>\n",
       "      <td>-2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490</th>\n",
       "      <td>tube</td>\n",
       "      <td>15.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>rave</td>\n",
       "      <td>22.00</td>\n",
       "      <td>8.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492</th>\n",
       "      <td>lower</td>\n",
       "      <td>12.00</td>\n",
       "      <td>-2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>holds</td>\n",
       "      <td>14.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>pretty</td>\n",
       "      <td>16.00</td>\n",
       "      <td>2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>bundles</td>\n",
       "      <td>13.00</td>\n",
       "      <td>-1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>would</td>\n",
       "      <td>15.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>oops</td>\n",
       "      <td>25.00</td>\n",
       "      <td>11.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>jeffree</td>\n",
       "      <td>18.00</td>\n",
       "      <td>4.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>site</td>\n",
       "      <td>11.00</td>\n",
       "      <td>-3.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>times</td>\n",
       "      <td>14.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>501</th>\n",
       "      <td>rad</td>\n",
       "      <td>16.00</td>\n",
       "      <td>2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>502</th>\n",
       "      <td>gorgeous</td>\n",
       "      <td>11.00</td>\n",
       "      <td>-3.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503</th>\n",
       "      <td>apply</td>\n",
       "      <td>10.00</td>\n",
       "      <td>-4.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>504</th>\n",
       "      <td>satin</td>\n",
       "      <td>12.00</td>\n",
       "      <td>-2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>505</th>\n",
       "      <td>good</td>\n",
       "      <td>13.00</td>\n",
       "      <td>-1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>506</th>\n",
       "      <td>depend</td>\n",
       "      <td>12.00</td>\n",
       "      <td>-2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507</th>\n",
       "      <td>plump</td>\n",
       "      <td>10.00</td>\n",
       "      <td>-4.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>508</th>\n",
       "      <td>ingredient</td>\n",
       "      <td>17.50</td>\n",
       "      <td>3.50</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>509 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              0      1  deviation\n",
       "0            ne  13.00      -1.00\n",
       "1            in  14.00       0.00\n",
       "2            li  31.00      17.00\n",
       "3            an  15.00       1.00\n",
       "4            ed  11.00      -3.00\n",
       "5            nd  17.00       3.00\n",
       "6            or  14.00       0.00\n",
       "7           and  15.00       1.00\n",
       "8            on  13.00      -1.00\n",
       "9            st  10.00      -4.00\n",
       "10          lip  13.00      -1.00\n",
       "11           th  20.00       6.00\n",
       "12           he  14.50       0.50\n",
       "13          new  15.00       1.00\n",
       "14           co   7.00      -7.00\n",
       "15           le  18.00       4.00\n",
       "16           sh  17.50       3.50\n",
       "17           at  13.00      -1.00\n",
       "18           it  14.25       0.25\n",
       "19           ll  15.00       1.00\n",
       "20           us  11.00      -3.00\n",
       "21           is  14.00       0.00\n",
       "22           ea  14.50       0.50\n",
       "23           hi  26.00      12.00\n",
       "24           nt  10.00      -4.00\n",
       "25          the  14.00       0.00\n",
       "26           de  39.50      25.50\n",
       "27           ps  28.00      14.00\n",
       "28           me  15.00       1.00\n",
       "29           as  13.00      -1.00\n",
       "..          ...    ...        ...\n",
       "479       first  20.00       6.00\n",
       "480       tarte  15.50       1.50\n",
       "481   senegence  27.00      13.00\n",
       "482       cover  19.00       5.00\n",
       "483     without  12.50      -1.50\n",
       "484         bar  13.00      -1.00\n",
       "485       lasts  20.00       6.00\n",
       "486        bold  16.00       2.00\n",
       "487        well  12.00      -2.00\n",
       "488     palette  12.00      -2.00\n",
       "489        lime  12.00      -2.00\n",
       "490        tube  15.00       1.00\n",
       "491        rave  22.00       8.00\n",
       "492       lower  12.00      -2.00\n",
       "493       holds  14.00       0.00\n",
       "494      pretty  16.00       2.00\n",
       "495     bundles  13.00      -1.00\n",
       "496       would  15.00       1.00\n",
       "497        oops  25.00      11.00\n",
       "498     jeffree  18.00       4.00\n",
       "499        site  11.00      -3.00\n",
       "500       times  14.00       0.00\n",
       "501         rad  16.00       2.00\n",
       "502    gorgeous  11.00      -3.00\n",
       "503       apply  10.00      -4.00\n",
       "504       satin  12.00      -2.00\n",
       "505        good  13.00      -1.00\n",
       "506      depend  12.00      -2.00\n",
       "507       plump  10.00      -4.00\n",
       "508  ingredient  17.50       3.50\n",
       "\n",
       "[509 rows x 3 columns]"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "category_price = categories[categories['Category'] == \"Beauty/Makeup/Lips\"]['Median'].values[0]\n",
    "\n",
    "keyword_price_df = pd.DataFrame(keyword_prices)\n",
    "keyword_price_df['deviation'] = (keyword_price_df[1] - category_price)\n",
    "keyword_price_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/numpy/core/fromnumeric.py:2909: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:80: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "[kp[1] for kp in keyword_price_df.values if kp[0] in \"lovely clean lips li\".split()]\n",
    "\n",
    "average_price_from_keywords = list(map(lambda description: np.mean([kp[1] for kp in keyword_price_df.values if kp[0] in description]), lips_sample['split_description']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2899"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(average_price_from_keywords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2899"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(lips_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.62134146341463414"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "deviations2 = abs(lips_sample['price'] - average_price_from_keywords) / lips_sample['price']\n",
    "accuracy2 = 1 - np.median(list(filter(lambda deviation: not math.isnan(deviation), deviations2)))\n",
    "accuracy2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "465"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Things to try to boost predictive power: \n",
    "# Select keywords with higher frequency\n",
    "# Ignore stopwords\n",
    "# Maybe ignore keywords that don't deviate much—is standard deviation on predictions lower than actual prices?\n",
    "# but after trying for other categories\n",
    "\n",
    "# Clean up redundant code, extract functions\n",
    "\n",
    "slipons_sample = random_sample.loc[random_sample['category_name'].isin([\"Women/Shoes/Loafers & Slip-Ons\"])]\n",
    "len(slipons_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:25: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:30: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "/anaconda3/lib/python3.6/site-packages/numpy/core/fromnumeric.py:2909: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:80: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.65625"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reduced_random_sample = random_sample.loc[random_sample['category_name'].isin([\"Beauty/Makeup/Lips\", \"Women/Shoes/Loafers & Slip-Ons\", \"Men/Tops/T-shirts\"])]\n",
    "slipons_sample = random_sample.loc[random_sample['category_name'].isin([\"Women/Shoes/Loafers & Slip-Ons\"])]\n",
    "\n",
    "\n",
    "import re\n",
    "\n",
    "# Clean up descriptions\n",
    "alpha_descriptions = list(map(lambda description: re.sub(r\"[^a-zA-Z ]\", '', description.lower()), slipons_sample['item_description']))\n",
    "\n",
    "# Unique words\n",
    "split_descriptions = map(lambda alpha: alpha.split(), iter(alpha_descriptions))\n",
    "all_keywords = [keyword for description in split_descriptions for keyword in description]\n",
    "unique_keywords = set(all_keywords)\n",
    "\n",
    "# Number of items with each word (smaller sample)\n",
    "keyword_counts = pd.DataFrame(list(map(lambda keyword: (keyword, len([alpha for alpha in iter(alpha_descriptions) if keyword in alpha])), list(unique_keywords))))\n",
    "keyword_counts.columns = ['word', 'count']\n",
    "significant_keywords = keyword_counts[keyword_counts['count'] > 50].sort_values(by=['count'], ascending=False)\n",
    "significant_keywords\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "alpha_description_df = pd.DataFrame(alpha_descriptions)\n",
    "alpha_description_df[0]\n",
    "slipons_sample['item_description'] = alpha_description_df[0].values\n",
    "\n",
    "\n",
    "multicharacter_keywords = list(filter(lambda row: len(row[0]) != 1, significant_keywords.values))\n",
    "split_description = [description.split() for description in slipons_sample['item_description'].values]\n",
    "slipons_sample['split_description'] = split_description\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "# This is to get all item associated with a keyword—you need to restructure multicharacter_keywords (it's actually keyword and count)\n",
    "# list(map(lambda keyword: (keyword, [item for item in slipons_sample.values if keyword in item[9]]), multicharacter_keywords[0]))\n",
    "\n",
    "# This was an attempt to get median price by keyword\n",
    "# list(map(lambda keyword: (keyword, np.median([item[5] for item in slipons_sample.values if keyword in item[9]])), multicharacter_keywords[0]))\n",
    "keyword_prices = list(map(lambda keyword: (keyword, np.median([item[5] for item in slipons_sample.values if keyword in item[9]])), [record[0] for record in multicharacter_keywords]))\n",
    "\n",
    "\n",
    "category_price = categories[categories['Category'] == \"Women/Shoes/Loafers & Slip-Ons\"]['Median'].values[0]\n",
    "\n",
    "keyword_price_df = pd.DataFrame(keyword_prices)\n",
    "keyword_price_df['deviation'] = (keyword_price_df[1] - category_price)\n",
    "keyword_price_df\n",
    "\n",
    "\n",
    "average_price_from_keywords = list(map(lambda description: np.mean([kp[1] for kp in keyword_price_df.values if kp[0] in description]), slipons_sample['split_description']))\n",
    "\n",
    "deviations3 = abs(slipons_sample['price'] - average_price_from_keywords) / slipons_sample['price']\n",
    "accuracy3 = 1 - np.median(list(filter(lambda deviation: not math.isnan(deviation), deviations3)))\n",
    "accuracy3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "465"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(average_price_from_keywords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:25: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:30: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "/anaconda3/lib/python3.6/site-packages/numpy/core/fromnumeric.py:2909: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:80: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.63516865079365081"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reduced_random_sample = random_sample.loc[random_sample['category_name'].isin([\"Beauty/Makeup/Lips\", \"Women/Shoes/Loafers & Slip-Ons\", \"Men/Tops/T-shirts\"])]\n",
    "mens_shirts_sample = random_sample.loc[random_sample['category_name'].isin([\"Men/Tops/T-shirts\"])]\n",
    "\n",
    "\n",
    "import re\n",
    "\n",
    "# Clean up descriptions\n",
    "alpha_descriptions = list(map(lambda description: re.sub(r\"[^a-zA-Z ]\", '', description.lower()), mens_shirts_sample['item_description']))\n",
    "\n",
    "# Unique words\n",
    "split_descriptions = map(lambda alpha: alpha.split(), iter(alpha_descriptions))\n",
    "all_keywords = [keyword for description in split_descriptions for keyword in description]\n",
    "unique_keywords = set(all_keywords)\n",
    "\n",
    "# Number of items with each word (smaller sample)\n",
    "keyword_counts = pd.DataFrame(list(map(lambda keyword: (keyword, len([alpha for alpha in iter(alpha_descriptions) if keyword in alpha])), list(unique_keywords))))\n",
    "keyword_counts.columns = ['word', 'count']\n",
    "significant_keywords = keyword_counts[keyword_counts['count'] > 50].sort_values(by=['count'], ascending=False)\n",
    "significant_keywords\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "alpha_description_df = pd.DataFrame(alpha_descriptions)\n",
    "alpha_description_df[0]\n",
    "mens_shirts_sample['item_description'] = alpha_description_df[0].values\n",
    "\n",
    "\n",
    "multicharacter_keywords = list(filter(lambda row: len(row[0]) != 1, significant_keywords.values))\n",
    "split_description = [description.split() for description in mens_shirts_sample['item_description'].values]\n",
    "mens_shirts_sample['split_description'] = split_description\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "# This is to get all item associated with a keyword—you need to restructure multicharacter_keywords (it's actually keyword and count)\n",
    "# list(map(lambda keyword: (keyword, [item for item in mens_shirts_sample.values if keyword in item[9]]), multicharacter_keywords[0]))\n",
    "\n",
    "# This was an attempt to get median price by keyword\n",
    "# list(map(lambda keyword: (keyword, np.median([item[5] for item in mens_shirts_sample.values if keyword in item[9]])), multicharacter_keywords[0]))\n",
    "keyword_prices = list(map(lambda keyword: (keyword, np.median([item[5] for item in mens_shirts_sample.values if keyword in item[9]])), [record[0] for record in multicharacter_keywords]))\n",
    "\n",
    "\n",
    "category_price = categories[categories['Category'] == \"Men/Tops/T-shirts\"]['Median'].values[0]\n",
    "\n",
    "keyword_price_df = pd.DataFrame(keyword_prices)\n",
    "keyword_price_df['deviation'] = (keyword_price_df[1] - category_price)\n",
    "keyword_price_df\n",
    "\n",
    "\n",
    "average_price_from_keywords = list(map(lambda description: np.mean([kp[1] for kp in keyword_price_df.values if kp[0] in description]), mens_shirts_sample['split_description']))\n",
    "\n",
    "deviations3 = abs(mens_shirts_sample['price'] - average_price_from_keywords) / mens_shirts_sample['price']\n",
    "accuracy3 = 1 - np.median(list(filter(lambda deviation: not math.isnan(deviation), deviations3)))\n",
    "accuracy3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}