create_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This jupyter notebook is written to get the true definition of the words from Merriam-Webster dictionary.\n",
    "\n",
    "## We ended up using the balderdash_words1.csv files.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/parsa/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import requests\n",
    "\n",
    "from tqdm import tqdm\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "tqdm.pandas()\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "nltk.download(\"stopwords\")\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abandon',\n",
       " 'ability',\n",
       " 'able',\n",
       " 'abortion',\n",
       " 'abroad',\n",
       " 'absence',\n",
       " 'absolute',\n",
       " 'absolutely',\n",
       " 'absorb',\n",
       " 'abuse',\n",
       " 'academic',\n",
       " 'accept',\n",
       " 'access',\n",
       " 'accident',\n",
       " 'accompany',\n",
       " 'accomplish',\n",
       " 'according',\n",
       " 'account',\n",
       " 'accurate',\n",
       " 'accuse',\n",
       " 'achieve',\n",
       " 'achievement',\n",
       " 'acid',\n",
       " 'acknowledge',\n",
       " 'acquire',\n",
       " 'across',\n",
       " 'act',\n",
       " 'action',\n",
       " 'active',\n",
       " 'activist',\n",
       " 'activity',\n",
       " 'actor',\n",
       " 'actress',\n",
       " 'actual',\n",
       " 'actually',\n",
       " 'ad',\n",
       " 'adapt',\n",
       " 'add',\n",
       " 'addition',\n",
       " 'additional',\n",
       " 'address',\n",
       " 'adequate',\n",
       " 'adjust',\n",
       " 'adjustment',\n",
       " 'administration',\n",
       " 'administrator',\n",
       " 'admire',\n",
       " 'admission',\n",
       " 'admit',\n",
       " 'adolescent',\n",
       " 'adopt',\n",
       " 'adult',\n",
       " 'advance',\n",
       " 'advanced',\n",
       " 'advantage',\n",
       " 'adventure',\n",
       " 'advertising',\n",
       " 'advice',\n",
       " 'advise',\n",
       " 'adviser',\n",
       " 'advocate',\n",
       " 'affair',\n",
       " 'affect',\n",
       " 'afford',\n",
       " 'afraid',\n",
       " 'African',\n",
       " 'African-American',\n",
       " 'afternoon',\n",
       " 'age',\n",
       " 'agency',\n",
       " 'agenda',\n",
       " 'agent',\n",
       " 'aggressive',\n",
       " 'ago',\n",
       " 'agree',\n",
       " 'agreement',\n",
       " 'agricultural',\n",
       " 'ah',\n",
       " 'ahead',\n",
       " 'aid',\n",
       " 'aide',\n",
       " 'AIDS',\n",
       " 'aim',\n",
       " 'air',\n",
       " 'aircraft',\n",
       " 'airline',\n",
       " 'airport',\n",
       " 'album',\n",
       " 'alcohol',\n",
       " 'alive',\n",
       " 'alliance',\n",
       " 'allow',\n",
       " 'ally',\n",
       " 'almost',\n",
       " 'alone',\n",
       " 'along',\n",
       " 'already',\n",
       " 'also',\n",
       " 'alter',\n",
       " 'alternative',\n",
       " 'although',\n",
       " 'always',\n",
       " 'AM',\n",
       " 'amazing',\n",
       " 'American',\n",
       " 'among',\n",
       " 'amount',\n",
       " 'analysis',\n",
       " 'analyst',\n",
       " 'analyze',\n",
       " 'ancient',\n",
       " 'anger',\n",
       " 'angle',\n",
       " 'angry',\n",
       " 'animal',\n",
       " 'anniversary',\n",
       " 'announce',\n",
       " 'annual',\n",
       " 'another',\n",
       " 'answer',\n",
       " 'anticipate',\n",
       " 'anxiety',\n",
       " 'anybody',\n",
       " 'anymore',\n",
       " 'anyone',\n",
       " 'anything',\n",
       " 'anyway',\n",
       " 'anywhere',\n",
       " 'apart',\n",
       " 'apartment',\n",
       " 'apparent',\n",
       " 'apparently',\n",
       " 'appeal',\n",
       " 'appear',\n",
       " 'appearance',\n",
       " 'apple',\n",
       " 'application',\n",
       " 'apply',\n",
       " 'appoint',\n",
       " 'appointment',\n",
       " 'appreciate',\n",
       " 'approach',\n",
       " 'appropriate',\n",
       " 'approval',\n",
       " 'approve',\n",
       " 'approximately',\n",
       " 'Arab',\n",
       " 'architect',\n",
       " 'area',\n",
       " 'argue',\n",
       " 'argument',\n",
       " 'arise',\n",
       " 'arm',\n",
       " 'armed',\n",
       " 'army',\n",
       " 'around',\n",
       " 'arrange',\n",
       " 'arrangement',\n",
       " 'arrest',\n",
       " 'arrival',\n",
       " 'arrive',\n",
       " 'art',\n",
       " 'article',\n",
       " 'artist',\n",
       " 'artistic',\n",
       " 'Asian',\n",
       " 'aside',\n",
       " 'ask',\n",
       " 'asleep',\n",
       " 'aspect',\n",
       " 'assault',\n",
       " 'assert',\n",
       " 'assess',\n",
       " 'assessment',\n",
       " 'asset',\n",
       " 'assign',\n",
       " 'assignment',\n",
       " 'assist',\n",
       " 'assistance',\n",
       " 'assistant',\n",
       " 'associate',\n",
       " 'association',\n",
       " 'assume',\n",
       " 'assumption',\n",
       " 'assure',\n",
       " 'athlete',\n",
       " 'athletic',\n",
       " 'atmosphere',\n",
       " 'attach',\n",
       " 'attack',\n",
       " 'attempt',\n",
       " 'attend',\n",
       " 'attention',\n",
       " 'attitude',\n",
       " 'attorney',\n",
       " 'attract',\n",
       " 'attractive',\n",
       " 'attribute',\n",
       " 'audience',\n",
       " 'author',\n",
       " 'authority',\n",
       " 'auto',\n",
       " 'available',\n",
       " 'average',\n",
       " 'avoid',\n",
       " 'award',\n",
       " 'aware',\n",
       " 'awareness',\n",
       " 'away',\n",
       " 'awful',\n",
       " 'baby',\n",
       " 'back',\n",
       " 'background',\n",
       " 'bad',\n",
       " 'badly',\n",
       " 'bag',\n",
       " 'bake',\n",
       " 'balance',\n",
       " 'ball',\n",
       " 'ban',\n",
       " 'band',\n",
       " 'bank',\n",
       " 'bar',\n",
       " 'barely',\n",
       " 'barrel',\n",
       " 'barrier',\n",
       " 'base',\n",
       " 'baseball',\n",
       " 'basic',\n",
       " 'basically',\n",
       " 'basis',\n",
       " 'basket',\n",
       " 'basketball',\n",
       " 'bathroom',\n",
       " 'battery',\n",
       " 'battle',\n",
       " 'beach',\n",
       " 'bean',\n",
       " 'bear',\n",
       " 'beat',\n",
       " 'beautiful',\n",
       " 'beauty',\n",
       " 'become',\n",
       " 'bed',\n",
       " 'bedroom',\n",
       " 'beer',\n",
       " 'begin',\n",
       " 'beginning',\n",
       " 'behavior',\n",
       " 'behind',\n",
       " 'belief',\n",
       " 'believe',\n",
       " 'bell',\n",
       " 'belong',\n",
       " 'belt',\n",
       " 'bench',\n",
       " 'bend',\n",
       " 'beneath',\n",
       " 'benefit',\n",
       " 'beside',\n",
       " 'besides',\n",
       " 'best',\n",
       " 'bet',\n",
       " 'better',\n",
       " 'beyond',\n",
       " 'Bible',\n",
       " 'big',\n",
       " 'bike',\n",
       " 'bill',\n",
       " 'billion',\n",
       " 'bind',\n",
       " 'biological',\n",
       " 'bird',\n",
       " 'birth',\n",
       " 'birthday',\n",
       " 'bit',\n",
       " 'bite',\n",
       " 'black',\n",
       " 'blade',\n",
       " 'blame',\n",
       " 'blanket',\n",
       " 'blind',\n",
       " 'block',\n",
       " 'blood',\n",
       " 'blow',\n",
       " 'blue',\n",
       " 'board',\n",
       " 'boat',\n",
       " 'body',\n",
       " 'bomb',\n",
       " 'bombing',\n",
       " 'bond',\n",
       " 'bone',\n",
       " 'book',\n",
       " 'boom',\n",
       " 'boot',\n",
       " 'border',\n",
       " 'born',\n",
       " 'borrow',\n",
       " 'boss',\n",
       " 'bother',\n",
       " 'bottle',\n",
       " 'bottom',\n",
       " 'boundary',\n",
       " 'bowl',\n",
       " 'box',\n",
       " 'boy',\n",
       " 'boyfriend',\n",
       " 'brain',\n",
       " 'branch',\n",
       " 'brand',\n",
       " 'bread',\n",
       " 'break',\n",
       " 'breakfast',\n",
       " 'breast',\n",
       " 'breath',\n",
       " 'breathe',\n",
       " 'brick',\n",
       " 'bridge',\n",
       " 'brief',\n",
       " 'briefly',\n",
       " 'bright',\n",
       " 'brilliant',\n",
       " 'bring',\n",
       " 'British',\n",
       " 'broad',\n",
       " 'broken',\n",
       " 'brother',\n",
       " 'brown',\n",
       " 'brush',\n",
       " 'buck',\n",
       " 'budget',\n",
       " 'build',\n",
       " 'building',\n",
       " 'bullet',\n",
       " 'bunch',\n",
       " 'burden',\n",
       " 'burn',\n",
       " 'bury',\n",
       " 'bus',\n",
       " 'business',\n",
       " 'busy',\n",
       " 'butter',\n",
       " 'button',\n",
       " 'buy',\n",
       " 'buyer',\n",
       " 'cabin',\n",
       " 'cabinet',\n",
       " 'cable',\n",
       " 'cake',\n",
       " 'calculate',\n",
       " 'call',\n",
       " 'camera',\n",
       " 'camp',\n",
       " 'campaign',\n",
       " 'campus',\n",
       " 'Canadian',\n",
       " 'cancer',\n",
       " 'candidate',\n",
       " 'cap',\n",
       " 'capability',\n",
       " 'capable',\n",
       " 'capacity',\n",
       " 'capital',\n",
       " 'captain',\n",
       " 'capture',\n",
       " 'car',\n",
       " 'carbon',\n",
       " 'card',\n",
       " 'care',\n",
       " 'career',\n",
       " 'careful',\n",
       " 'carefully',\n",
       " 'carrier',\n",
       " 'carry',\n",
       " 'case',\n",
       " 'cash',\n",
       " 'cast',\n",
       " 'cat',\n",
       " 'catch',\n",
       " 'category',\n",
       " 'Catholic',\n",
       " 'cause',\n",
       " 'ceiling',\n",
       " 'celebrate',\n",
       " 'celebration',\n",
       " 'celebrity',\n",
       " 'cell',\n",
       " 'center',\n",
       " 'central',\n",
       " 'century',\n",
       " 'CEO',\n",
       " 'ceremony',\n",
       " 'certain',\n",
       " 'certainly',\n",
       " 'chain',\n",
       " 'chair',\n",
       " 'chairman',\n",
       " 'challenge',\n",
       " 'chamber',\n",
       " 'champion',\n",
       " 'championship',\n",
       " 'chance',\n",
       " 'change',\n",
       " 'changing',\n",
       " 'channel',\n",
       " 'chapter',\n",
       " 'character',\n",
       " 'characteristic',\n",
       " 'characterize',\n",
       " 'charge',\n",
       " 'charity',\n",
       " 'chart',\n",
       " 'chase',\n",
       " 'cheap',\n",
       " 'check',\n",
       " 'cheek',\n",
       " 'cheese',\n",
       " 'chef',\n",
       " 'chemical',\n",
       " 'chest',\n",
       " 'chicken',\n",
       " 'chief',\n",
       " 'child',\n",
       " 'childhood',\n",
       " 'Chinese',\n",
       " 'chip',\n",
       " 'chocolate',\n",
       " 'choice',\n",
       " 'cholesterol',\n",
       " 'choose',\n",
       " 'Christian',\n",
       " 'Christmas',\n",
       " 'church',\n",
       " 'cigarette',\n",
       " 'circle',\n",
       " 'circumstance',\n",
       " 'cite',\n",
       " 'citizen',\n",
       " 'city',\n",
       " 'civil',\n",
       " 'civilian',\n",
       " 'claim',\n",
       " 'class',\n",
       " 'classic',\n",
       " 'classroom',\n",
       " 'clean',\n",
       " 'clear',\n",
       " 'clearly',\n",
       " 'client',\n",
       " 'climate',\n",
       " 'climb',\n",
       " 'clinic',\n",
       " 'clinical',\n",
       " 'clock',\n",
       " 'close',\n",
       " 'closely',\n",
       " 'closer',\n",
       " 'clothes',\n",
       " 'clothing',\n",
       " 'cloud',\n",
       " 'club',\n",
       " 'clue',\n",
       " 'cluster',\n",
       " 'coach',\n",
       " 'coal',\n",
       " 'coalition',\n",
       " 'coast',\n",
       " 'coat',\n",
       " 'code',\n",
       " 'coffee',\n",
       " 'cognitive',\n",
       " 'cold',\n",
       " 'collapse',\n",
       " 'colleague',\n",
       " 'collect',\n",
       " 'collection',\n",
       " 'collective',\n",
       " 'college',\n",
       " 'colonial',\n",
       " 'color',\n",
       " 'column',\n",
       " 'combination',\n",
       " 'combine',\n",
       " 'come',\n",
       " 'comedy',\n",
       " 'comfort',\n",
       " 'comfortable',\n",
       " 'command',\n",
       " 'commander',\n",
       " 'comment',\n",
       " 'commercial',\n",
       " 'commission',\n",
       " 'commit',\n",
       " 'commitment',\n",
       " 'committee',\n",
       " 'common',\n",
       " 'communicate',\n",
       " 'communication',\n",
       " 'community',\n",
       " 'company',\n",
       " 'compare',\n",
       " 'comparison',\n",
       " 'compete',\n",
       " 'competition',\n",
       " 'competitive',\n",
       " 'competitor',\n",
       " 'complain',\n",
       " 'complaint',\n",
       " 'complete',\n",
       " 'completely',\n",
       " 'complex',\n",
       " 'complicated',\n",
       " 'component',\n",
       " 'compose',\n",
       " 'composition',\n",
       " 'comprehensive',\n",
       " 'computer',\n",
       " 'concentrate',\n",
       " 'concentration',\n",
       " 'concept',\n",
       " 'concern',\n",
       " 'concerned',\n",
       " 'concert',\n",
       " 'conclude',\n",
       " 'conclusion',\n",
       " 'concrete',\n",
       " 'condition',\n",
       " 'conduct',\n",
       " 'conference',\n",
       " 'confidence',\n",
       " 'confident',\n",
       " 'confirm',\n",
       " 'conflict',\n",
       " 'confront',\n",
       " 'confusion',\n",
       " 'Congress',\n",
       " 'congressional',\n",
       " 'connect',\n",
       " 'connection',\n",
       " 'consciousness',\n",
       " 'consensus',\n",
       " 'consequence',\n",
       " 'conservative',\n",
       " 'consider',\n",
       " 'considerable',\n",
       " 'consideration',\n",
       " 'consist',\n",
       " 'consistent',\n",
       " 'constant',\n",
       " 'constantly',\n",
       " 'constitute',\n",
       " 'constitutional',\n",
       " 'construct',\n",
       " 'construction',\n",
       " 'consultant',\n",
       " 'consume',\n",
       " 'consumer',\n",
       " 'consumption',\n",
       " 'contact',\n",
       " 'contain',\n",
       " 'container',\n",
       " 'contemporary',\n",
       " 'content',\n",
       " 'contest',\n",
       " 'context',\n",
       " 'continue',\n",
       " 'continued',\n",
       " 'contract',\n",
       " 'contrast',\n",
       " 'contribute',\n",
       " 'contribution',\n",
       " 'control',\n",
       " 'controversial',\n",
       " 'controversy',\n",
       " 'convention',\n",
       " 'conventional',\n",
       " 'conversation',\n",
       " 'convert',\n",
       " 'conviction',\n",
       " 'convince',\n",
       " 'cook',\n",
       " 'cookie',\n",
       " 'cooking',\n",
       " 'cool',\n",
       " 'cooperation',\n",
       " 'cop',\n",
       " 'cope',\n",
       " 'copy',\n",
       " 'core',\n",
       " 'corn',\n",
       " 'corner',\n",
       " 'corporate',\n",
       " 'corporation',\n",
       " 'correct',\n",
       " 'correspondent',\n",
       " 'cost',\n",
       " 'cotton',\n",
       " 'couch',\n",
       " 'could',\n",
       " 'council',\n",
       " 'counselor',\n",
       " 'count',\n",
       " 'counter',\n",
       " 'country',\n",
       " 'county',\n",
       " 'couple',\n",
       " 'courage',\n",
       " 'course',\n",
       " 'court',\n",
       " 'cousin',\n",
       " 'cover',\n",
       " 'coverage',\n",
       " 'cow',\n",
       " 'crack',\n",
       " 'craft',\n",
       " 'crash',\n",
       " 'crazy',\n",
       " 'cream',\n",
       " 'create',\n",
       " 'creation',\n",
       " 'creative',\n",
       " 'creature',\n",
       " 'credit',\n",
       " 'crew',\n",
       " 'crime',\n",
       " 'criminal',\n",
       " 'crisis',\n",
       " 'criteria',\n",
       " 'critic',\n",
       " 'critical',\n",
       " 'criticism',\n",
       " 'criticize',\n",
       " 'crop',\n",
       " 'cross',\n",
       " 'crowd',\n",
       " 'crucial',\n",
       " 'cry',\n",
       " 'cultural',\n",
       " 'culture',\n",
       " 'cup',\n",
       " 'curious',\n",
       " 'current',\n",
       " 'currently',\n",
       " 'curriculum',\n",
       " 'custom',\n",
       " 'customer',\n",
       " 'cut',\n",
       " 'cycle',\n",
       " 'dad',\n",
       " 'daily',\n",
       " 'damage',\n",
       " 'dance',\n",
       " 'danger',\n",
       " 'dangerous',\n",
       " 'dare',\n",
       " 'dark',\n",
       " 'darkness',\n",
       " 'data',\n",
       " 'date',\n",
       " 'daughter',\n",
       " 'day',\n",
       " 'dead',\n",
       " 'deal',\n",
       " 'dealer',\n",
       " 'dear',\n",
       " 'death',\n",
       " 'debate',\n",
       " 'debt',\n",
       " 'decade',\n",
       " 'decide',\n",
       " 'decision',\n",
       " 'deck',\n",
       " 'declare',\n",
       " 'decline',\n",
       " 'decrease',\n",
       " 'deep',\n",
       " 'deeply',\n",
       " 'deer',\n",
       " 'defeat',\n",
       " 'defend',\n",
       " 'defendant',\n",
       " 'defense',\n",
       " 'defensive',\n",
       " 'deficit',\n",
       " 'define',\n",
       " 'definitely',\n",
       " 'definition',\n",
       " 'degree',\n",
       " 'delay',\n",
       " 'deliver',\n",
       " 'delivery',\n",
       " 'demand',\n",
       " 'democracy',\n",
       " 'Democrat',\n",
       " 'democratic',\n",
       " 'demonstrate',\n",
       " 'demonstration',\n",
       " 'deny',\n",
       " 'department',\n",
       " 'depend',\n",
       " 'dependent',\n",
       " 'depending',\n",
       " 'depict',\n",
       " 'depression',\n",
       " 'depth',\n",
       " 'deputy',\n",
       " 'derive',\n",
       " 'describe',\n",
       " 'description',\n",
       " 'desert',\n",
       " 'deserve',\n",
       " 'design',\n",
       " 'designer',\n",
       " 'desire',\n",
       " 'desk',\n",
       " 'desperate',\n",
       " 'despite',\n",
       " 'destroy',\n",
       " 'destruction',\n",
       " 'detail',\n",
       " 'detailed',\n",
       " 'detect',\n",
       " 'determine',\n",
       " 'develop',\n",
       " 'developing',\n",
       " 'development',\n",
       " 'device',\n",
       " 'devote',\n",
       " 'dialogue',\n",
       " 'die',\n",
       " 'diet',\n",
       " 'differ',\n",
       " 'difference',\n",
       " 'different',\n",
       " 'differently',\n",
       " 'difficult',\n",
       " 'difficulty',\n",
       " 'dig',\n",
       " 'digital',\n",
       " 'dimension',\n",
       " 'dining',\n",
       " 'dinner',\n",
       " 'direct',\n",
       " 'direction',\n",
       " 'directly',\n",
       " 'director',\n",
       " 'dirt',\n",
       " 'dirty',\n",
       " 'disability',\n",
       " 'disagree',\n",
       " 'disappear',\n",
       " 'disaster',\n",
       " 'discipline',\n",
       " 'discourse',\n",
       " 'discover',\n",
       " 'discovery',\n",
       " 'discrimination',\n",
       " 'discuss',\n",
       " 'discussion',\n",
       " 'disease',\n",
       " 'dish',\n",
       " 'dismiss',\n",
       " 'disorder',\n",
       " 'display',\n",
       " 'dispute',\n",
       " 'distance',\n",
       " 'distant',\n",
       " 'distinct',\n",
       " 'distinction',\n",
       " 'distinguish',\n",
       " 'distribute',\n",
       " 'distribution',\n",
       " 'district',\n",
       " 'diverse',\n",
       " 'diversity',\n",
       " 'divide',\n",
       " 'division',\n",
       " 'divorce',\n",
       " 'DNA',\n",
       " 'doctor',\n",
       " 'document',\n",
       " 'dog',\n",
       " 'domestic',\n",
       " 'dominant',\n",
       " 'dominate',\n",
       " 'door',\n",
       " 'double',\n",
       " 'doubt',\n",
       " 'downtown',\n",
       " 'dozen',\n",
       " 'draft',\n",
       " 'drag',\n",
       " 'drama',\n",
       " 'dramatic',\n",
       " 'dramatically',\n",
       " 'draw',\n",
       " 'drawing',\n",
       " 'dream',\n",
       " 'dress',\n",
       " 'drink',\n",
       " 'drive',\n",
       " 'driver',\n",
       " 'drop',\n",
       " 'drug',\n",
       " 'dry',\n",
       " 'due',\n",
       " 'dust',\n",
       " 'duty',\n",
       " 'eager',\n",
       " 'ear',\n",
       " 'early',\n",
       " 'earn',\n",
       " 'earnings',\n",
       " 'earth',\n",
       " 'ease',\n",
       " 'easily',\n",
       " 'east',\n",
       " 'eastern',\n",
       " 'easy',\n",
       " 'eat',\n",
       " 'economic',\n",
       " 'economics',\n",
       " 'economist',\n",
       " 'economy',\n",
       " 'edge',\n",
       " 'edition',\n",
       " 'editor',\n",
       " 'educate',\n",
       " 'education',\n",
       " 'educational',\n",
       " 'educator',\n",
       " 'effect',\n",
       " 'effective',\n",
       " 'effectively',\n",
       " 'efficiency',\n",
       " 'efficient',\n",
       " 'effort',\n",
       " 'egg',\n",
       " 'eight',\n",
       " 'either',\n",
       " 'elderly',\n",
       " 'elect',\n",
       " 'election',\n",
       " 'electric',\n",
       " 'electricity',\n",
       " 'electronic',\n",
       " 'element',\n",
       " 'elementary',\n",
       " 'eliminate',\n",
       " 'elite',\n",
       " 'else',\n",
       " 'elsewhere',\n",
       " 'e-mail',\n",
       " 'embrace',\n",
       " 'emerge',\n",
       " 'emergency',\n",
       " 'emission',\n",
       " 'emotion',\n",
       " 'emotional',\n",
       " 'emphasis',\n",
       " 'emphasize',\n",
       " 'employ',\n",
       " 'employee',\n",
       " 'employer',\n",
       " 'employment',\n",
       " 'empty',\n",
       " 'enable',\n",
       " 'encounter',\n",
       " 'encourage',\n",
       " 'end',\n",
       " 'enemy',\n",
       " 'energy',\n",
       " 'enforcement',\n",
       " 'engage',\n",
       " 'engine',\n",
       " 'engineer',\n",
       " 'engineering',\n",
       " 'English',\n",
       " 'enhance',\n",
       " 'enjoy',\n",
       " 'enormous',\n",
       " 'enough',\n",
       " 'ensure',\n",
       " 'enter',\n",
       " 'enterprise',\n",
       " 'entertainment',\n",
       " 'entire',\n",
       " 'entirely',\n",
       " 'entrance',\n",
       " 'entry',\n",
       " 'environment',\n",
       " 'environmental',\n",
       " 'episode',\n",
       " 'equal',\n",
       " 'equally',\n",
       " 'equipment',\n",
       " 'era',\n",
       " 'error',\n",
       " 'escape',\n",
       " 'especially',\n",
       " 'essay',\n",
       " 'essential',\n",
       " 'essentially',\n",
       " 'establish',\n",
       " 'establishment',\n",
       " 'estate',\n",
       " 'estimate',\n",
       " 'etc',\n",
       " 'ethics',\n",
       " 'ethnic',\n",
       " 'European',\n",
       " 'evaluate',\n",
       " 'evaluation',\n",
       " 'even',\n",
       " 'evening',\n",
       " 'event',\n",
       " 'eventually',\n",
       " 'ever',\n",
       " 'every',\n",
       " 'everybody',\n",
       " 'everyday',\n",
       " 'everyone',\n",
       " 'everything',\n",
       " 'everywhere',\n",
       " 'evidence',\n",
       " 'evolution',\n",
       " 'evolve',\n",
       " 'exact',\n",
       " 'exactly',\n",
       " 'examination',\n",
       " 'examine',\n",
       " 'example',\n",
       " 'exceed',\n",
       " 'excellent',\n",
       " 'except',\n",
       " 'exception',\n",
       " 'exchange',\n",
       " 'exciting',\n",
       " 'executive',\n",
       " 'exercise',\n",
       " 'exhibit',\n",
       " 'exhibition',\n",
       " 'exist',\n",
       " 'existence',\n",
       " 'existing',\n",
       " 'expand',\n",
       " 'expansion',\n",
       " 'expect',\n",
       " 'expectation',\n",
       " 'expense',\n",
       " 'expensive',\n",
       " 'experience',\n",
       " 'experiment',\n",
       " 'expert',\n",
       " 'explain',\n",
       " 'explanation',\n",
       " 'explode',\n",
       " 'explore',\n",
       " 'explosion',\n",
       " 'expose',\n",
       " 'exposure',\n",
       " 'express',\n",
       " 'expression',\n",
       " 'extend',\n",
       " 'extension',\n",
       " 'extensive',\n",
       " 'extent',\n",
       " 'external',\n",
       " 'extra',\n",
       " 'extraordinary',\n",
       " 'extreme',\n",
       " 'extremely',\n",
       " 'eye',\n",
       " 'fabric',\n",
       " 'face',\n",
       " 'facility',\n",
       " 'fact',\n",
       " 'factor',\n",
       " 'factory',\n",
       " 'faculty',\n",
       " 'fade',\n",
       " 'fail',\n",
       " 'failure',\n",
       " 'fair',\n",
       " 'fairly',\n",
       " 'faith',\n",
       " 'fall',\n",
       " 'false',\n",
       " 'familiar',\n",
       " 'family',\n",
       " 'famous',\n",
       " 'fan',\n",
       " 'fantasy',\n",
       " 'far',\n",
       " 'farm',\n",
       " 'farmer',\n",
       " 'fashion',\n",
       " 'fast',\n",
       " 'fat',\n",
       " ...]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read 'data/basic_english_words.csv' file and read the column \"words\"\n",
    "df = pd.read_csv(\"data/basic_english_words.csv\")\n",
    "words = df[\"words\"].tolist()\n",
    "\n",
    "words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'a',\n",
       " 'about',\n",
       " 'above',\n",
       " 'after',\n",
       " 'again',\n",
       " 'against',\n",
       " 'ain',\n",
       " 'all',\n",
       " 'am',\n",
       " 'an',\n",
       " 'and',\n",
       " 'any',\n",
       " 'are',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'as',\n",
       " 'at',\n",
       " 'be',\n",
       " 'because',\n",
       " 'been',\n",
       " 'before',\n",
       " 'being',\n",
       " 'below',\n",
       " 'between',\n",
       " 'both',\n",
       " 'but',\n",
       " 'by',\n",
       " 'can',\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'd',\n",
       " 'did',\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'do',\n",
       " 'does',\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'doing',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'down',\n",
       " 'during',\n",
       " 'each',\n",
       " 'few',\n",
       " 'for',\n",
       " 'from',\n",
       " 'further',\n",
       " 'had',\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'has',\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'have',\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'having',\n",
       " 'he',\n",
       " 'her',\n",
       " 'here',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'him',\n",
       " 'himself',\n",
       " 'his',\n",
       " 'how',\n",
       " 'i',\n",
       " 'if',\n",
       " 'in',\n",
       " 'into',\n",
       " 'is',\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'just',\n",
       " 'll',\n",
       " 'm',\n",
       " 'ma',\n",
       " 'me',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'more',\n",
       " 'most',\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'my',\n",
       " 'myself',\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'now',\n",
       " 'o',\n",
       " 'of',\n",
       " 'off',\n",
       " 'on',\n",
       " 'once',\n",
       " 'only',\n",
       " 'or',\n",
       " 'other',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'out',\n",
       " 'over',\n",
       " 'own',\n",
       " 're',\n",
       " 's',\n",
       " 'same',\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'so',\n",
       " 'some',\n",
       " 'such',\n",
       " 't',\n",
       " 'than',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'the',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'them',\n",
       " 'themselves',\n",
       " 'then',\n",
       " 'there',\n",
       " 'these',\n",
       " 'they',\n",
       " 'this',\n",
       " 'those',\n",
       " 'through',\n",
       " 'to',\n",
       " 'too',\n",
       " 'under',\n",
       " 'until',\n",
       " 'up',\n",
       " 've',\n",
       " 'very',\n",
       " 'was',\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'we',\n",
       " 'were',\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'what',\n",
       " 'when',\n",
       " 'where',\n",
       " 'which',\n",
       " 'while',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'why',\n",
       " 'will',\n",
       " 'with',\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\",\n",
       " 'y',\n",
       " 'you',\n",
       " \"you'd\",\n",
       " \"you'll\",\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words = set(stopwords.words(\"english\"))\n",
    "stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of words: 2895\n"
     ]
    }
   ],
   "source": [
    "# Filter out the stop words from the list of words\n",
    "words = [word for word in words if word not in stop_words]\n",
    "print(f\"Length of words: {len(words)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>def</th>\n",
       "      <th>POS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>abandon</td>\n",
       "      <td>['to give up to the control or influence of an...</td>\n",
       "      <td>['verb', 'verb', 'verb', 'verb', 'verb', 'verb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ability</td>\n",
       "      <td>['the quality or state of being able', 'compet...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun suffix']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>able</td>\n",
       "      <td>['having sufficient power, skill, or resources...</td>\n",
       "      <td>['adjective', 'adjective', 'adjective', 'adjec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>abortion</td>\n",
       "      <td>['spontaneous expulsion of a human fetus durin...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>abroad</td>\n",
       "      <td>[\"beyond the boundaries of one's country in or...</td>\n",
       "      <td>['adverb or adjective', 'adverb or adjective',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2890</th>\n",
       "      <td>yet</td>\n",
       "      <td>['in addition besides', 'even', 'on top of eve...</td>\n",
       "      <td>['adverb', 'adverb', 'adverb', 'adverb', 'adve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2891</th>\n",
       "      <td>yield</td>\n",
       "      <td>['to bear or bring forth as a natural product ...</td>\n",
       "      <td>['verb', 'verb', 'verb', 'verb', 'verb', 'verb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2892</th>\n",
       "      <td>young</td>\n",
       "      <td>['being in the first or an early stage of life...</td>\n",
       "      <td>['adjective', 'adjective', 'adjective', 'adjec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2893</th>\n",
       "      <td>youth</td>\n",
       "      <td>['the time of life when one is young', 'the ea...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2894</th>\n",
       "      <td>zone</td>\n",
       "      <td>[\"any of five great divisions of the earth's s...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2895 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         words                                                def  \\\n",
       "0      abandon  ['to give up to the control or influence of an...   \n",
       "1      ability  ['the quality or state of being able', 'compet...   \n",
       "2         able  ['having sufficient power, skill, or resources...   \n",
       "3     abortion  ['spontaneous expulsion of a human fetus durin...   \n",
       "4       abroad  [\"beyond the boundaries of one's country in or...   \n",
       "...        ...                                                ...   \n",
       "2890       yet  ['in addition besides', 'even', 'on top of eve...   \n",
       "2891     yield  ['to bear or bring forth as a natural product ...   \n",
       "2892     young  ['being in the first or an early stage of life...   \n",
       "2893     youth  ['the time of life when one is young', 'the ea...   \n",
       "2894      zone  [\"any of five great divisions of the earth's s...   \n",
       "\n",
       "                                                    POS  \n",
       "0     ['verb', 'verb', 'verb', 'verb', 'verb', 'verb...  \n",
       "1               ['noun', 'noun', 'noun', 'noun suffix']  \n",
       "2     ['adjective', 'adjective', 'adjective', 'adjec...  \n",
       "3     ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "4     ['adverb or adjective', 'adverb or adjective',...  \n",
       "...                                                 ...  \n",
       "2890  ['adverb', 'adverb', 'adverb', 'adverb', 'adve...  \n",
       "2891  ['verb', 'verb', 'verb', 'verb', 'verb', 'verb...  \n",
       "2892  ['adjective', 'adjective', 'adjective', 'adjec...  \n",
       "2893  ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "2894  ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "\n",
       "[2895 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>def</th>\n",
       "      <th>POS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>abandon</td>\n",
       "      <td>['to give up to the control or influence of an...</td>\n",
       "      <td>['verb', 'verb', 'verb', 'verb', 'verb', 'verb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ability</td>\n",
       "      <td>['the quality or state of being able', 'compet...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun suffix']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>able</td>\n",
       "      <td>['having sufficient power, skill, or resources...</td>\n",
       "      <td>['adjective', 'adjective', 'adjective', 'adjec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>abortion</td>\n",
       "      <td>['spontaneous expulsion of a human fetus durin...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>abroad</td>\n",
       "      <td>[\"beyond the boundaries of one's country in or...</td>\n",
       "      <td>['adverb or adjective', 'adverb or adjective',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2890</th>\n",
       "      <td>yet</td>\n",
       "      <td>['in addition besides', 'even', 'on top of eve...</td>\n",
       "      <td>['adverb', 'adverb', 'adverb', 'adverb', 'adve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2891</th>\n",
       "      <td>yield</td>\n",
       "      <td>['to bear or bring forth as a natural product ...</td>\n",
       "      <td>['verb', 'verb', 'verb', 'verb', 'verb', 'verb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2892</th>\n",
       "      <td>young</td>\n",
       "      <td>['being in the first or an early stage of life...</td>\n",
       "      <td>['adjective', 'adjective', 'adjective', 'adjec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2893</th>\n",
       "      <td>youth</td>\n",
       "      <td>['the time of life when one is young', 'the ea...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2894</th>\n",
       "      <td>zone</td>\n",
       "      <td>[\"any of five great divisions of the earth's s...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2865 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         words                                                def  \\\n",
       "0      abandon  ['to give up to the control or influence of an...   \n",
       "1      ability  ['the quality or state of being able', 'compet...   \n",
       "2         able  ['having sufficient power, skill, or resources...   \n",
       "3     abortion  ['spontaneous expulsion of a human fetus durin...   \n",
       "4       abroad  [\"beyond the boundaries of one's country in or...   \n",
       "...        ...                                                ...   \n",
       "2890       yet  ['in addition besides', 'even', 'on top of eve...   \n",
       "2891     yield  ['to bear or bring forth as a natural product ...   \n",
       "2892     young  ['being in the first or an early stage of life...   \n",
       "2893     youth  ['the time of life when one is young', 'the ea...   \n",
       "2894      zone  [\"any of five great divisions of the earth's s...   \n",
       "\n",
       "                                                    POS  \n",
       "0     ['verb', 'verb', 'verb', 'verb', 'verb', 'verb...  \n",
       "1               ['noun', 'noun', 'noun', 'noun suffix']  \n",
       "2     ['adjective', 'adjective', 'adjective', 'adjec...  \n",
       "3     ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "4     ['adverb or adjective', 'adverb or adjective',...  \n",
       "...                                                 ...  \n",
       "2890  ['adverb', 'adverb', 'adverb', 'adverb', 'adve...  \n",
       "2891  ['verb', 'verb', 'verb', 'verb', 'verb', 'verb...  \n",
       "2892  ['adjective', 'adjective', 'adjective', 'adjec...  \n",
       "2893  ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "2894  ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "\n",
       "[2865 rows x 3 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop the rows containing -1 in \"def\" column\n",
    "df = df[df[\"def\"] != '-1']\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "853    True\n",
       "Name: def, dtype: bool"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show row for the word \"e-mail\"\n",
    "df[df[\"words\"] == \"e-mail\"]['def'] == '-1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_definition_text(text):\n",
    "    \"\"\"\n",
    "    Cleans the definition text by removing or replacing special tokens.\n",
    "    \"\"\"\n",
    "    # Remove bold, italics, smallcaps, and other formatting\n",
    "    text = re.sub(r\"\\{/?it\\}\", \"\", text)  # Remove italic tags\n",
    "    text = re.sub(r\"\\{/?b\\}\", \"\", text)  # Remove bold tags\n",
    "    text = re.sub(r\"\\{/?sc\\}\", \"\", text)  # Remove smallcaps tags\n",
    "    text = re.sub(r\"\\{/?sup\\}\", \"\", text)  # Remove superscript tags\n",
    "    text = re.sub(r\"\\{/?inf\\}\", \"\", text)  # Remove subscript tags\n",
    "\n",
    "    # Replace punctuation tokens\n",
    "    text = re.sub(r\"\\{bc\\}\", \"\", text)  # Replace {bc} with colon and space\n",
    "    text = re.sub(r\"\\{ldquo\\}\", '\"', text)  # Replace left double quote\n",
    "    text = re.sub(r\"\\{rdquo\\}\", '\"', text)  # Replace right double quote\n",
    "\n",
    "    # Replace link and cross-reference tokens, showing only the linked term\n",
    "    text = re.sub(r\"\\{[a-z_]+_link\\|([^|}]+)\\|?[^}]*\\}\", r\"\\1\", text)\n",
    "\n",
    "    # Replace cross-reference tokens like {dxt|...|...|}, removing extra details\n",
    "    text = re.sub(r\"\\{dxt\\|([^|}]+)\\|[^}]*\\}\", r\"\\1\", text)\n",
    "\n",
    "    # Replace synonymous cross-reference {sx|term|...} with 'term'\n",
    "    text = re.sub(r\"\\{sx\\|([^|}]+)\\|[^}]*\\}\", r\"\\1\", text)\n",
    "\n",
    "    # Handle nested complex tokens like {gloss|...} and similar\n",
    "    text = re.sub(r\"\\{[^}]+\\}([^{}]+)\\{\\/[^}]+\\}\", r\"\\1\", text)\n",
    "\n",
    "    return text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d9461dddb6b4de9bcb2e51a44728f95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2895 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>def</th>\n",
       "      <th>POS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>abandon</td>\n",
       "      <td>['to give up to the control or influence of an...</td>\n",
       "      <td>['verb', 'verb', 'verb', 'verb', 'verb', 'verb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ability</td>\n",
       "      <td>['the quality or state of being able', 'compet...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun suffix']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>able</td>\n",
       "      <td>['having sufficient power, skill, or resources...</td>\n",
       "      <td>['adjective', 'adjective', 'adjective', 'adjec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>abortion</td>\n",
       "      <td>['spontaneous expulsion of a human fetus durin...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>abroad</td>\n",
       "      <td>[\"beyond the boundaries of one's country in or...</td>\n",
       "      <td>['adverb or adjective', 'adverb or adjective',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2890</th>\n",
       "      <td>yet</td>\n",
       "      <td>['in addition besides', 'even', 'on top of eve...</td>\n",
       "      <td>['adverb', 'adverb', 'adverb', 'adverb', 'adve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2891</th>\n",
       "      <td>yield</td>\n",
       "      <td>['to bear or bring forth as a natural product ...</td>\n",
       "      <td>['verb', 'verb', 'verb', 'verb', 'verb', 'verb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2892</th>\n",
       "      <td>young</td>\n",
       "      <td>['being in the first or an early stage of life...</td>\n",
       "      <td>['adjective', 'adjective', 'adjective', 'adjec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2893</th>\n",
       "      <td>youth</td>\n",
       "      <td>['the time of life when one is young', 'the ea...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2894</th>\n",
       "      <td>zone</td>\n",
       "      <td>[\"any of five great divisions of the earth's s...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2895 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         words                                                def  \\\n",
       "0      abandon  ['to give up to the control or influence of an...   \n",
       "1      ability  ['the quality or state of being able', 'compet...   \n",
       "2         able  ['having sufficient power, skill, or resources...   \n",
       "3     abortion  ['spontaneous expulsion of a human fetus durin...   \n",
       "4       abroad  [\"beyond the boundaries of one's country in or...   \n",
       "...        ...                                                ...   \n",
       "2890       yet  ['in addition besides', 'even', 'on top of eve...   \n",
       "2891     yield  ['to bear or bring forth as a natural product ...   \n",
       "2892     young  ['being in the first or an early stage of life...   \n",
       "2893     youth  ['the time of life when one is young', 'the ea...   \n",
       "2894      zone  [\"any of five great divisions of the earth's s...   \n",
       "\n",
       "                                                    POS  \n",
       "0     ['verb', 'verb', 'verb', 'verb', 'verb', 'verb...  \n",
       "1               ['noun', 'noun', 'noun', 'noun suffix']  \n",
       "2     ['adjective', 'adjective', 'adjective', 'adjec...  \n",
       "3     ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "4     ['adverb or adjective', 'adverb or adjective',...  \n",
       "...                                                 ...  \n",
       "2890  ['adverb', 'adverb', 'adverb', 'adverb', 'adve...  \n",
       "2891  ['verb', 'verb', 'verb', 'verb', 'verb', 'verb...  \n",
       "2892  ['adjective', 'adjective', 'adjective', 'adjec...  \n",
       "2893  ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "2894  ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "\n",
       "[2895 rows x 3 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"def\"] = df[\"def\"].progress_apply(clean_definition_text)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>def</th>\n",
       "      <th>POS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>abandon</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ability</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>able</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>abortion</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>abroad</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2890</th>\n",
       "      <td>yet</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2891</th>\n",
       "      <td>yield</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2892</th>\n",
       "      <td>young</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2893</th>\n",
       "      <td>youth</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2894</th>\n",
       "      <td>zone</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2895 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         words def POS\n",
       "0      abandon        \n",
       "1      ability        \n",
       "2         able        \n",
       "3     abortion        \n",
       "4       abroad        \n",
       "...        ...  ..  ..\n",
       "2890       yet        \n",
       "2891     yield        \n",
       "2892     young        \n",
       "2893     youth        \n",
       "2894      zone        \n",
       "\n",
       "[2895 rows x 3 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a pandas dataframe with columns \"words\" \"def\" and \"POS\" and fill the \"words\" column with the words\n",
    "df = pd.DataFrame(words, columns=[\"words\"])\n",
    "df[\"def\"] = \"\"\n",
    "df[\"POS\"] = \"\"\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_definition_and_pos(word, api_key):\n",
    "    \"\"\"Retrieve detailed definitions and their corresponding parts of speech for a given word from Merriam-Webster API.\"\"\"\n",
    "    url = f\"https://dictionaryapi.com/api/v3/references/collegiate/json/{word}?key={api_key}\"\n",
    "    response = requests.get(url)\n",
    "    if response.status_code != 200:\n",
    "        return None, None\n",
    "\n",
    "    data = response.json()\n",
    "    definition_pos_pairs = []\n",
    "\n",
    "    for entry in data:\n",
    "        if \"meta\" in entry and \"id\" in entry[\"meta\"] and word in entry[\"meta\"][\"id\"]:\n",
    "            pos = entry.get(\"fl\", None)\n",
    "            if \"def\" in entry:\n",
    "                # Extract definitions from each sense in 'sseq'\n",
    "                for sense_block in entry[\"def\"]:\n",
    "                    for sense in sense_block.get(\"sseq\", []):\n",
    "                        for sense_detail in sense:\n",
    "                            if sense_detail[0] == \"sense\":\n",
    "                                definition_text = sense_detail[1].get(\"dt\", [])\n",
    "                                # Collect each textual definition\n",
    "                                for text in definition_text:\n",
    "                                    if text[0] == \"text\":\n",
    "                                        clean_definition = text[1].strip()\n",
    "                                        definition_pos_pairs.append((clean_definition, pos))\n",
    "\n",
    "    return definition_pos_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0c2eb8fd3fb4586ac4fa4401c9d57eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2895 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>def</th>\n",
       "      <th>POS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>abandon</td>\n",
       "      <td>['{bc}to give up to the control or influence o...</td>\n",
       "      <td>['verb', 'verb', 'verb', 'verb', 'verb', 'verb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ability</td>\n",
       "      <td>['{bc}the quality or state of being able', '{b...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun suffix']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>able</td>\n",
       "      <td>['{bc}having sufficient power, skill, or resou...</td>\n",
       "      <td>['adjective', 'adjective', 'adjective', 'adjec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>abortion</td>\n",
       "      <td>['{bc}spontaneous expulsion of a human fetus d...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>abroad</td>\n",
       "      <td>[\"{bc}beyond the boundaries of one's country {...</td>\n",
       "      <td>['adverb or adjective', 'adverb or adjective',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2890</th>\n",
       "      <td>yet</td>\n",
       "      <td>['{bc}in addition {bc}{sx|besides||}', '{bc}{s...</td>\n",
       "      <td>['adverb', 'adverb', 'adverb', 'adverb', 'adve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2891</th>\n",
       "      <td>yield</td>\n",
       "      <td>['{bc}to bear or bring forth as a natural prod...</td>\n",
       "      <td>['verb', 'verb', 'verb', 'verb', 'verb', 'verb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2892</th>\n",
       "      <td>young</td>\n",
       "      <td>['{bc}being in the first or an early stage of ...</td>\n",
       "      <td>['adjective', 'adjective', 'adjective', 'adjec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2893</th>\n",
       "      <td>youth</td>\n",
       "      <td>['{bc}the time of life when one is young', '{b...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2894</th>\n",
       "      <td>zone</td>\n",
       "      <td>[\"{bc}any of five great divisions of the earth...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2895 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         words                                                def  \\\n",
       "0      abandon  ['{bc}to give up to the control or influence o...   \n",
       "1      ability  ['{bc}the quality or state of being able', '{b...   \n",
       "2         able  ['{bc}having sufficient power, skill, or resou...   \n",
       "3     abortion  ['{bc}spontaneous expulsion of a human fetus d...   \n",
       "4       abroad  [\"{bc}beyond the boundaries of one's country {...   \n",
       "...        ...                                                ...   \n",
       "2890       yet  ['{bc}in addition {bc}{sx|besides||}', '{bc}{s...   \n",
       "2891     yield  ['{bc}to bear or bring forth as a natural prod...   \n",
       "2892     young  ['{bc}being in the first or an early stage of ...   \n",
       "2893     youth  ['{bc}the time of life when one is young', '{b...   \n",
       "2894      zone  [\"{bc}any of five great divisions of the earth...   \n",
       "\n",
       "                                                    POS  \n",
       "0     ['verb', 'verb', 'verb', 'verb', 'verb', 'verb...  \n",
       "1               ['noun', 'noun', 'noun', 'noun suffix']  \n",
       "2     ['adjective', 'adjective', 'adjective', 'adjec...  \n",
       "3     ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "4     ['adverb or adjective', 'adverb or adjective',...  \n",
       "...                                                 ...  \n",
       "2890  ['adverb', 'adverb', 'adverb', 'adverb', 'adve...  \n",
       "2891  ['verb', 'verb', 'verb', 'verb', 'verb', 'verb...  \n",
       "2892  ['adjective', 'adjective', 'adjective', 'adjec...  \n",
       "2893  ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "2894  ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "\n",
       "[2895 rows x 3 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Assume df is your DataFrame and it has a column named 'words'\n",
    "progress_bar = tqdm(df.iterrows(), total=df.shape[0])\n",
    "\n",
    "for index, row in progress_bar:\n",
    "    word = row[\"words\"]\n",
    "    api_key = os.getenv(\"DICTIONARY_API_KEY\")\n",
    "    definition_pos_pairs = get_word_definition_and_pos(word, api_key)\n",
    "    if definition_pos_pairs:\n",
    "        # Separate definitions and POSs\n",
    "        definitions = [pair[0] for pair in definition_pos_pairs]\n",
    "        poss = [pair[1] for pair in definition_pos_pairs if pair[1] is not None]\n",
    "        # make string version of definitions and POSs\n",
    "        df.at[index, \"def\"] = definitions.__str__()\n",
    "        df.at[index, \"POS\"] = poss.__str__()\n",
    "    else:\n",
    "        # Update progress bar to show the word was not found\n",
    "        progress_bar.set_postfix({\"word\": word, \"status\": \"not found\"})\n",
    "        df.at[index, \"def\"] = -1  # Use None or a specific string if -1 is not appropriate\n",
    "        df.at[index, \"POS\"] = -1  # Same as above\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"./data/basic_english_words.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/parsahejabi/Codebases/GitHub_Repositories/USC-CSCI699-HistoryofLanguage-AI_Balderdash/data/balderdash_words1.csv'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "balderdash_words1_path = os.path.join(os.getcwd(), \"data\", \"balderdash_words1.csv\")\n",
    "balderdash_words1_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/parsahejabi/Codebases/GitHub_Repositories/USC-CSCI699-HistoryofLanguage-AI_Balderdash/data/balderdash_words2.csv'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "balderdash_words2_path = os.path.join(os.getcwd(), \"data\", \"balderdash_words2.csv\")\n",
    "balderdash_words2_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>def</th>\n",
       "      <th>POS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wadmiltilt</td>\n",
       "      <td>['A strong rough woolen cloth employed to cove...</td>\n",
       "      <td>['noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>scrivello</td>\n",
       "      <td>[\"An elephant's tusk of less than 20 pounds in...</td>\n",
       "      <td>['noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scapulimancy</td>\n",
       "      <td>['Divination by means of a shoulder-blade: sam...</td>\n",
       "      <td>['noun', 'noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>queez-madam</td>\n",
       "      <td>['The cuisse-madam, a French jargonelle pear.']</td>\n",
       "      <td>['noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>progger</td>\n",
       "      <td>['One who progs; a rambling or aimless searche...</td>\n",
       "      <td>['noun', 'noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>doromania</td>\n",
       "      <td>['An unusual urge or preoccupation with giving...</td>\n",
       "      <td>['noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>pogonip</td>\n",
       "      <td>['A frozen fog, formed in the coldest weather ...</td>\n",
       "      <td>['noun', 'noun', 'noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>oogonium</td>\n",
       "      <td>['Any of the cells that give rise through mito...</td>\n",
       "      <td>['noun', 'noun', 'noun', 'noun', 'noun', 'noun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>decussated</td>\n",
       "      <td>['Simple past tense and past participle of dec...</td>\n",
       "      <td>['verb']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>emunction</td>\n",
       "      <td>[\"The act of blowing one's nose.\"]</td>\n",
       "      <td>['noun']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>233 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            words                                                def  \\\n",
       "0      wadmiltilt  ['A strong rough woolen cloth employed to cove...   \n",
       "1       scrivello  [\"An elephant's tusk of less than 20 pounds in...   \n",
       "2    scapulimancy  ['Divination by means of a shoulder-blade: sam...   \n",
       "3     queez-madam    ['The cuisse-madam, a French jargonelle pear.']   \n",
       "4         progger  ['One who progs; a rambling or aimless searche...   \n",
       "..            ...                                                ...   \n",
       "228     doromania  ['An unusual urge or preoccupation with giving...   \n",
       "229       pogonip  ['A frozen fog, formed in the coldest weather ...   \n",
       "230      oogonium  ['Any of the cells that give rise through mito...   \n",
       "231    decussated  ['Simple past tense and past participle of dec...   \n",
       "232     emunction                 [\"The act of blowing one's nose.\"]   \n",
       "\n",
       "                                                   POS  \n",
       "0                                             ['noun']  \n",
       "1                                             ['noun']  \n",
       "2                                     ['noun', 'noun']  \n",
       "3                                             ['noun']  \n",
       "4                                     ['noun', 'noun']  \n",
       "..                                                 ...  \n",
       "228                                           ['noun']  \n",
       "229                           ['noun', 'noun', 'noun']  \n",
       "230  ['noun', 'noun', 'noun', 'noun', 'noun', 'noun...  \n",
       "231                                           ['verb']  \n",
       "232                                           ['noun']  \n",
       "\n",
       "[233 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "balderdash_words1_df = pd.read_csv(balderdash_words1_path)\n",
    "balderdash_words1_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>def</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Abibliophobia</td>\n",
       "      <td>The fear of running out of reading material.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Absquatulate</td>\n",
       "      <td>To leave or abscond with something.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Allegator</td>\n",
       "      <td>Some who alleges.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Anencephalous</td>\n",
       "      <td>Lacking a brain.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Argle-bargle</td>\n",
       "      <td>A loud row or quarrel.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>Unremacadamized</td>\n",
       "      <td>Having not been repaved with macadam.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>Vomitory</td>\n",
       "      <td>An exit or outlet.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>Wabbit</td>\n",
       "      <td>Exhausted, tired, worn out.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>Widdershins</td>\n",
       "      <td>In a contrary or counterclockwise direction.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>Yahoo</td>\n",
       "      <td>A rube, a country bumpkin.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               words                                           def\n",
       "0      Abibliophobia  The fear of running out of reading material.\n",
       "1       Absquatulate           To leave or abscond with something.\n",
       "2          Allegator                             Some who alleges.\n",
       "3      Anencephalous                              Lacking a brain.\n",
       "4       Argle-bargle                        A loud row or quarrel.\n",
       "..               ...                                           ...\n",
       "97   Unremacadamized         Having not been repaved with macadam.\n",
       "98          Vomitory                            An exit or outlet.\n",
       "99            Wabbit                   Exhausted, tired, worn out.\n",
       "100      Widdershins  In a contrary or counterclockwise direction.\n",
       "101            Yahoo                    A rube, a country bumpkin.\n",
       "\n",
       "[102 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "balderdash_words2_df = pd.read_csv(balderdash_words2_path)\n",
    "balderdash_words2_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_definition_and_pos(word, api_key):\n",
    "    \"\"\"\n",
    "    Makes a request to the Merriam-Webster Dictionary API for a given word and returns the first definition\n",
    "    and part of speech (POS) if available.\n",
    "\n",
    "    Parameters:\n",
    "    word (str): The word for which the definition and POS are requested.\n",
    "    api_key (str): Your Merriam-Webster API key.\n",
    "\n",
    "    Returns:\n",
    "    tuple: A tuple containing the first definition and POS of the word. If multiple\n",
    "    definitions or POS tags are present, only the first is returned. If no definition\n",
    "    or POS is found, None is returned for each.\n",
    "    \"\"\"\n",
    "    url = f\"https://dictionaryapi.com/api/v3/references/collegiate/json/{word}?key={api_key}\"\n",
    "    response = requests.get(url)\n",
    "    response_data = response.json()\n",
    "\n",
    "    # Check if the response contains data and extract the first definition and POS\n",
    "    if (\n",
    "        response_data\n",
    "        and isinstance(response_data, list)\n",
    "        and \"shortdef\" in response_data[0]\n",
    "        and \"fl\" in response_data[0]\n",
    "    ):\n",
    "        first_definition = response_data[0][\"shortdef\"][0] if response_data[0][\"shortdef\"] else None\n",
    "        pos = response_data[0][\"fl\"] if response_data[0][\"fl\"] else None\n",
    "        return first_definition, pos\n",
    "    else:\n",
    "        # Handle cases where the word might not be found or API response is unexpected\n",
    "        return None, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>def</th>\n",
       "      <th>POS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wadmiltilt</td>\n",
       "      <td>['A strong rough woolen cloth employed to cove...</td>\n",
       "      <td>['noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>scrivello</td>\n",
       "      <td>[\"An elephant's tusk of less than 20 pounds in...</td>\n",
       "      <td>['noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scapulimancy</td>\n",
       "      <td>['Divination by means of a shoulder-blade: sam...</td>\n",
       "      <td>['noun', 'noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>queez-madam</td>\n",
       "      <td>['The cuisse-madam, a French jargonelle pear.']</td>\n",
       "      <td>['noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>progger</td>\n",
       "      <td>['One who progs; a rambling or aimless searche...</td>\n",
       "      <td>['noun', 'noun']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330</th>\n",
       "      <td>Unremacadamized</td>\n",
       "      <td>Having not been repaved with macadam.</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>Vomitory</td>\n",
       "      <td>An exit or outlet.</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>Wabbit</td>\n",
       "      <td>Exhausted, tired, worn out.</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>Widdershins</td>\n",
       "      <td>In a contrary or counterclockwise direction.</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>Yahoo</td>\n",
       "      <td>A rube, a country bumpkin.</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>335 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               words                                                def  \\\n",
       "0         wadmiltilt  ['A strong rough woolen cloth employed to cove...   \n",
       "1          scrivello  [\"An elephant's tusk of less than 20 pounds in...   \n",
       "2       scapulimancy  ['Divination by means of a shoulder-blade: sam...   \n",
       "3        queez-madam    ['The cuisse-madam, a French jargonelle pear.']   \n",
       "4            progger  ['One who progs; a rambling or aimless searche...   \n",
       "..               ...                                                ...   \n",
       "330  Unremacadamized              Having not been repaved with macadam.   \n",
       "331         Vomitory                                 An exit or outlet.   \n",
       "332           Wabbit                        Exhausted, tired, worn out.   \n",
       "333      Widdershins       In a contrary or counterclockwise direction.   \n",
       "334            Yahoo                         A rube, a country bumpkin.   \n",
       "\n",
       "                  POS  \n",
       "0            ['noun']  \n",
       "1            ['noun']  \n",
       "2    ['noun', 'noun']  \n",
       "3            ['noun']  \n",
       "4    ['noun', 'noun']  \n",
       "..                ...  \n",
       "330               NaN  \n",
       "331               NaN  \n",
       "332               NaN  \n",
       "333               NaN  \n",
       "334               NaN  \n",
       "\n",
       "[335 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make a single dataframe containing all the words from both CSV files\n",
    "balderdash_words_df = pd.concat([balderdash_words1_df, balderdash_words2_df], ignore_index=True)\n",
    "balderdash_words_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>def</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wadmiltilt</td>\n",
       "      <td>['A strong rough woolen cloth employed to cove...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>scrivello</td>\n",
       "      <td>[\"An elephant's tusk of less than 20 pounds in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scapulimancy</td>\n",
       "      <td>['Divination by means of a shoulder-blade: sam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>queez-madam</td>\n",
       "      <td>['The cuisse-madam, a French jargonelle pear.']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>progger</td>\n",
       "      <td>['One who progs; a rambling or aimless searche...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330</th>\n",
       "      <td>Unremacadamized</td>\n",
       "      <td>Having not been repaved with macadam.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>Vomitory</td>\n",
       "      <td>An exit or outlet.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>Wabbit</td>\n",
       "      <td>Exhausted, tired, worn out.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>Widdershins</td>\n",
       "      <td>In a contrary or counterclockwise direction.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>Yahoo</td>\n",
       "      <td>A rube, a country bumpkin.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>335 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               words                                                def\n",
       "0         wadmiltilt  ['A strong rough woolen cloth employed to cove...\n",
       "1          scrivello  [\"An elephant's tusk of less than 20 pounds in...\n",
       "2       scapulimancy  ['Divination by means of a shoulder-blade: sam...\n",
       "3        queez-madam    ['The cuisse-madam, a French jargonelle pear.']\n",
       "4            progger  ['One who progs; a rambling or aimless searche...\n",
       "..               ...                                                ...\n",
       "330  Unremacadamized              Having not been repaved with macadam.\n",
       "331         Vomitory                                 An exit or outlet.\n",
       "332           Wabbit                        Exhausted, tired, worn out.\n",
       "333      Widdershins       In a contrary or counterclockwise direction.\n",
       "334            Yahoo                         A rube, a country bumpkin.\n",
       "\n",
       "[335 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop the POS column\n",
    "balderdash_words_df = balderdash_words_df.drop(columns=[\"POS\"])\n",
    "balderdash_words_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>dictionary_definition</th>\n",
       "      <th>pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wadmiltilt</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>scrivello</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scapulimancy</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>queez-madam</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>progger</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330</th>\n",
       "      <td>Unremacadamized</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>Vomitory</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>Wabbit</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>Widdershins</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>Yahoo</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>335 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                word dictionary_definition   pos\n",
       "0         wadmiltilt                  None  None\n",
       "1          scrivello                  None  None\n",
       "2       scapulimancy                  None  None\n",
       "3        queez-madam                  None  None\n",
       "4            progger                  None  None\n",
       "..               ...                   ...   ...\n",
       "330  Unremacadamized                  None  None\n",
       "331         Vomitory                  None  None\n",
       "332           Wabbit                  None  None\n",
       "333      Widdershins                  None  None\n",
       "334            Yahoo                  None  None\n",
       "\n",
       "[335 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rename \"words\" to \"word\" and \"def\" to \"dictionary_definition\"\n",
    "balderdash_words_df = balderdash_words_df.rename(columns={\"words\": \"word\", \"def\": \"dictionary_definition\"})\n",
    "# Clear the \"dictionary_definition\" column\n",
    "balderdash_words_df[\"dictionary_definition\"] = None\n",
    "# Create a new column \"pos\" and fill it with None\n",
    "balderdash_words_df[\"pos\"] = None\n",
    "balderdash_words_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb5784baf4704c72977978bf4665cf1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/335 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>dictionary_definition</th>\n",
       "      <th>pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wadmiltilt</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>scrivello</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scapulimancy</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>queez-madam</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>progger</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330</th>\n",
       "      <td>Unremacadamized</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>Vomitory</td>\n",
       "      <td>an entrance piercing the banks of seats of a t...</td>\n",
       "      <td>noun</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>Wabbit</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>Widdershins</td>\n",
       "      <td>in a left-handed, wrong, or contrary direction...</td>\n",
       "      <td>adverb</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>Yahoo</td>\n",
       "      <td>a member of a race of brutes in Swift's Gulliv...</td>\n",
       "      <td>noun</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>335 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                word                              dictionary_definition  \\\n",
       "0         wadmiltilt                                                 -1   \n",
       "1          scrivello                                                 -1   \n",
       "2       scapulimancy                                                 -1   \n",
       "3        queez-madam                                                 -1   \n",
       "4            progger                                                 -1   \n",
       "..               ...                                                ...   \n",
       "330  Unremacadamized                                                 -1   \n",
       "331         Vomitory  an entrance piercing the banks of seats of a t...   \n",
       "332           Wabbit                                                 -1   \n",
       "333      Widdershins  in a left-handed, wrong, or contrary direction...   \n",
       "334            Yahoo  a member of a race of brutes in Swift's Gulliv...   \n",
       "\n",
       "        pos  \n",
       "0        -1  \n",
       "1        -1  \n",
       "2        -1  \n",
       "3        -1  \n",
       "4        -1  \n",
       "..      ...  \n",
       "330      -1  \n",
       "331    noun  \n",
       "332      -1  \n",
       "333  adverb  \n",
       "334    noun  \n",
       "\n",
       "[335 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For each word in the dataframe, get the definition and POS from the Merriam-Webster API\n",
    "# and update the corresponding columns in the dataframe\n",
    "progress_bar = tqdm(balderdash_words_df.iterrows(), total=balderdash_words_df.shape[0])\n",
    "\n",
    "for index, row in progress_bar:\n",
    "    word = row[\"word\"]\n",
    "    definition, pos = get_word_definition_and_pos(word, os.getenv(\"DICTIONARY_API_KEY\"))\n",
    "    if definition is not None:\n",
    "        balderdash_words_df.at[index, \"dictionary_definition\"] = definition\n",
    "        balderdash_words_df.at[index, \"pos\"] = pos\n",
    "    else:\n",
    "        # Write in tqdm progress bar: \"Word {word} not found in the dictionary.\"\n",
    "        progress_bar.set_postfix({\"word\": word, \"status\": \"not found\"})\n",
    "        # Write -1 in the dictionary_definition and pos column if the word is not found\n",
    "        balderdash_words_df.at[index, \"dictionary_definition\"] = -1\n",
    "        balderdash_words_df.at[index, \"pos\"] = -1\n",
    "\n",
    "balderdash_words_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the updated dataframe to a new CSV file\n",
    "balderdash_words_with_definitions_path = os.path.join(\n",
    "    os.getcwd(), \"data\", \"balderdash_words_with_definitions.csv\"\n",
    ")\n",
    "balderdash_words_df.to_csv(balderdash_words_with_definitions_path, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}