Lexile Determination.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# References"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- http://allselenium.info/working-with-checkboxes-using-python-selenium-webdriver/ \n",
    "- https://selenium-python.readthedocs.io/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Obtain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import time\n",
    "import pickle\n",
    "from math import floor\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Full Texts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Acquire Available Titles, Authors, and URLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()\n",
    "driver.implicitly_wait(5)\n",
    "driver.get(\"https://www.gutenberg.org/wiki/Children%27s_Literature_(Bookshelf)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Aunt Jo's Scrap-Bag, Vol. 5; Jimmy's Cruise in the Pinafore, Etc.\",\n",
       " 'Eight Cousins',\n",
       " 'Flower Fables',\n",
       " 'A Garland for Girls',\n",
       " 'Jack and Jill',\n",
       " \"Jo's Boys\",\n",
       " \"Kitty's Class Day and Other Stories\",\n",
       " 'Little Men',\n",
       " 'Little Women',\n",
       " 'The Louisa Alcott Reader: a Supplementary Reader for the Fourth Year of School',\n",
       " \"Marjorie's Three Gifts\",\n",
       " 'A Modern Cinderella',\n",
       " 'The Mysterious Key and What It Opened',\n",
       " 'An Old-Fashioned Girl',\n",
       " 'Onder Moeders Vleugels',\n",
       " 'Op Eigen Wieken',\n",
       " 'Rose in Bloom',\n",
       " \"Shawl-Straps; A Second Series of Aunt Jo's Scrap-Bag\",\n",
       " 'Under the Lilacs',\n",
       " 'Peter Pan',\n",
       " 'American Fairy Tales',\n",
       " 'Illustrated',\n",
       " 'The Emerald City of Oz',\n",
       " 'The Enchanted Island of Yew',\n",
       " 'Glinda of Oz',\n",
       " 'A Kidnapped Santa Claus',\n",
       " 'Life and Adventures of Santa Claus',\n",
       " 'Little Wizard Stories of Oz',\n",
       " 'The Lost Princess of Oz',\n",
       " 'The Magic of Oz',\n",
       " 'The Marvelous Land of Oz',\n",
       " 'Mary Louise',\n",
       " 'Mary Louise in the Country',\n",
       " 'The Master Key, an Electrical Fairy Tale Founded Upon the Mysteries of Electricity',\n",
       " 'Mother Goose in Prose',\n",
       " 'Ozma of Oz',\n",
       " 'The Patchwork Girl of Oz',\n",
       " 'Rinkitink in Oz',\n",
       " 'The Road to Oz',\n",
       " 'The Scarecrow of Oz',\n",
       " 'The Sea Fairies',\n",
       " \"Sky Island: being the further exciting adventures of Trot and Cap'n Bill after their visit to the sea fairies\",\n",
       " 'The Surprising Adventures of the Magical Monarch of Mo and His People',\n",
       " 'Tik-Tok of Oz',\n",
       " 'The Tin Woodman of Oz',\n",
       " 'The Woggle-Bug Book',\n",
       " 'The Wonderful Wizard of Oz',\n",
       " 'Ebook',\n",
       " 'Rab and His Friends',\n",
       " 'The Adventures of Grandfather Frog',\n",
       " 'The Adventures of Jerry Muskrat',\n",
       " 'The Adventures of Jimmy Skunk',\n",
       " 'The Adventures of Johnny Chuck',\n",
       " 'The Adventures of Lightfoot the Deer',\n",
       " 'The Adventures of Mr. Mocker',\n",
       " 'The Adventures of Old Mr. Toad',\n",
       " 'The Adventures of Paddy Beaver',\n",
       " 'The Adventures of Paddy the Beaver',\n",
       " 'The Adventures of Poor Mrs. Quack',\n",
       " 'The Adventures of Prickly Porky',\n",
       " 'Adventures of Reddy Fox',\n",
       " \"The Adventures of Unc' Billy Possum\",\n",
       " 'Blacky the Crow',\n",
       " 'Bowser the Hound',\n",
       " 'Happy Jack',\n",
       " 'Lightfoot the Deer',\n",
       " \"Mother West Wind's Children\",\n",
       " 'Mother West Wind \"Where\" Stories',\n",
       " \"Mother West Wind 'Why' Stories\",\n",
       " 'Mrs. Peter Rabbit',\n",
       " 'Old Granny Fox',\n",
       " 'Old Mother West Wind',\n",
       " 'Whitefoot the Wood Mouse',\n",
       " 'The Dawn of a To-morrow',\n",
       " 'The Land of the Blue Flower',\n",
       " 'The Little Hunchback Zia',\n",
       " 'Little Lord Fauntleroy',\n",
       " 'A Little Princess',\n",
       " 'Little Saint Elizabeth and Other Stories',\n",
       " 'The Lost Prince',\n",
       " 'My Robin',\n",
       " \"Sara Crewe: or, What happened at Miss Minchin's boarding school\",\n",
       " 'Racketty-Packetty House',\n",
       " 'The Secret Garden',\n",
       " 'Understood Betsy',\n",
       " \"Alice's Adventures in Wonderland\",\n",
       " \"Alice's Abenteuer im Wunderland\",\n",
       " 'La Aventuroj de Alicio en Mirlando',\n",
       " 'Alice\\'s Adventures Under Ground; Being a facsimile of the original Ms. book afterwards developed into \"Alice\\'s Adventures in Wonderland\"',\n",
       " 'The Hunting of the Snark',\n",
       " 'Through the Looking-Glass',\n",
       " 'What Katy Did',\n",
       " 'What Katy Did Next',\n",
       " 'What Katy Did at School',\n",
       " 'Clover',\n",
       " 'Verses',\n",
       " 'Cantique de Noël',\n",
       " 'The Cricket on the Hearth',\n",
       " 'Brothers of Pity and Other Tales of Beasts and Men',\n",
       " 'The Brownies and Other Tales',\n",
       " 'A Flat Iron for a Farthing\\nor Some Passages in the Life of an only Son',\n",
       " 'A Great Emergency and Other Tales',\n",
       " 'Jackanapes',\n",
       " \"Jackanapes, Daddy Darwin's Dovecot and Other Stories\",\n",
       " 'Jan of the Windmill',\n",
       " \"Mary's Meadow\\nAnd Other Tales of Fields and Flowers\",\n",
       " \"Melchior's Dream and Other Tales\",\n",
       " \"Mrs. Overtheway's Remembrances\",\n",
       " 'Old-Fashioned Fairy Tales',\n",
       " 'The Peace Egg and Other tales',\n",
       " 'Six to Sixteen\\nA Story for Girls',\n",
       " 'Verses for Children\\nand Songs for Music',\n",
       " 'We and the World, Part I\\nA Book for Boys',\n",
       " 'We and the World, Part II\\nA Book for Boys',\n",
       " 'The Wind in the Willows',\n",
       " 'A Little Union Scout',\n",
       " 'Mingo And Other Sketches in Black and White',\n",
       " 'Little Mr. Thimblefinger and His Queer Country',\n",
       " 'Uncle Remus, his songs and his sayings',\n",
       " 'Nights With Uncle Remus',\n",
       " 'Beneath an Umbrella (From \"Twice Told Tales\")',\n",
       " 'Chippings with a Chisel (From \"Twice Told Tales\")',\n",
       " 'Edward Fane\\'s Rosebud (From \"Twice Told Tales\")',\n",
       " 'Fancy\\'s Show-Box (From \"Twice Told Tales\")',\n",
       " 'Footprints on the Sea-Shore (From \"Twice Told Tales\")',\n",
       " 'From Twice Told Tales',\n",
       " 'The Gorgon\\'s Head (From: \"A Wonder-Book for Girls and Boys\")',\n",
       " 'The Haunted Mind (From \"Twice Told Tales\")',\n",
       " 'The Lily\\'s Quest (From \"Twice Told Tales\")',\n",
       " 'Little Annie\\'s Ramble (From \"Twice Told Tales\")',\n",
       " 'The Miraculous Pitcher (From: \"A Wonder-Book for Girls and Boys\")',\n",
       " 'The Paradise of Children (From: \"A Wonder-Book for Girls and Boys\")',\n",
       " 'The Seven Vagabonds (From \"Twice Told Tales\")',\n",
       " 'Sights from a Steeple (From \"Twice Told Tales\")',\n",
       " 'The Sister Years (From \"Twice Told Tales\")',\n",
       " 'Snow Flakes (From \"Twice Told Tales\")',\n",
       " 'Sunday at Home (From \"Twice Told Tales\")',\n",
       " 'Tanglewood Tales',\n",
       " 'The Threefold Destiny (From \"Twice Told Tales\")',\n",
       " 'The Three Golden Apples (From: \"A Wonder-Book for Girls and Boys\")',\n",
       " 'The Toll Gatherer\\'s Day (From \"Twice Told Tales\")',\n",
       " 'Twice Told Tales',\n",
       " 'The Village Uncle (From \"Twice Told Tales\")',\n",
       " 'The Vision of the Fountain (From \"Twice Told Tales\")',\n",
       " 'The White Old Maid (From \"Twice Told Tales\")',\n",
       " 'The Legend of Sleepy Hollow',\n",
       " 'La Legendo de Dorm-Valeto',\n",
       " 'The Water-Babies; A Fairy Tale for a Land-Baby',\n",
       " 'The Jungle Book',\n",
       " 'The Second Jungle Book',\n",
       " 'Just So Stories',\n",
       " 'Version 1',\n",
       " 'Kim',\n",
       " 'The Blue Fairy Book',\n",
       " 'The Brown Fairy Book',\n",
       " 'The Crimson Fairy Book',\n",
       " 'The Green Fairy Book',\n",
       " 'The Grey Fairy Book',\n",
       " 'The Lilac Fairy Book',\n",
       " 'The Orange Fairy Book',\n",
       " 'The Pink Fairy Book',\n",
       " 'The Red Fairy Book',\n",
       " 'The Violet Fairy Book',\n",
       " 'The Yellow Fairy Book',\n",
       " 'Prince Prigio; From \"His Own Fairy Book\"',\n",
       " \"Prince Ricardo of Pantouflia; being the adventures of Prince Prigio's son\",\n",
       " 'The Story of Doctor Dolittle',\n",
       " 'The Voyages of Dr. Dolittle',\n",
       " \"The Children's Own Longfellow\",\n",
       " 'The Song of Hiawatha',\n",
       " 'Text-only original',\n",
       " 'The Light Princess',\n",
       " 'The Light Princess and Other Fairy Stories',\n",
       " 'The Princess and Curdie',\n",
       " 'The Princess and the Goblin',\n",
       " 'Children of the New Forest',\n",
       " 'Anne of Avonlea',\n",
       " 'Anne of Green Gables',\n",
       " 'Anne of the Island',\n",
       " \"Anne's House of Dreams\",\n",
       " 'Chronicles of Avonlea',\n",
       " 'Further Chronicles of Avonlea',\n",
       " 'The Golden Road',\n",
       " 'Kilmeny of the Orchard',\n",
       " 'Rainbow Valley',\n",
       " 'Rilla of Ingleside',\n",
       " 'The Story Girl',\n",
       " 'The Little Lame Prince',\n",
       " 'The Little Lame Prince; Rewritten for Young Readers by Margaret Waters',\n",
       " 'Beautiful Stories from Shakespeare',\n",
       " 'The Book of Dragons',\n",
       " 'The Enchanted Castle',\n",
       " 'Five Children and It',\n",
       " 'In Homespun',\n",
       " 'The Magic City',\n",
       " 'New Treasure Seekers; or, The Bastable Children in Search of a Fortune',\n",
       " 'The Phoenix and the Carpet',\n",
       " 'The Railway Children',\n",
       " 'The Rainbow and the Rose',\n",
       " 'The Story of the Amulet',\n",
       " 'The Story of the Treasure Seekers',\n",
       " 'The Wouldbegoods',\n",
       " 'Bebee',\n",
       " 'Bimbi: Stories for Children',\n",
       " 'A Dog of Flanders',\n",
       " 'The Nürnberg stove',\n",
       " 'Contes pour les petits garçons',\n",
       " 'Black Beauty - The Autobiography of a Horse',\n",
       " \"Black Beauty, Young Folks' Edition\",\n",
       " 'Five Little Peppers and How They Grew',\n",
       " 'Five Little Peppers Midway',\n",
       " 'Five Little Peppers Grown Up',\n",
       " 'The Adventures of Joel Pepper',\n",
       " 'Five Little Peppers Abroad',\n",
       " 'Five Little Peppers and Their Friends',\n",
       " 'Cornelli',\n",
       " 'Erick and Sally',\n",
       " \"Gritli's Children\",\n",
       " 'Heidi',\n",
       " 'Heidi kann brauchen, was es gelernt hat',\n",
       " 'Heidis Lehr- und Wanderjahre',\n",
       " 'Heidis Lehr- und Wanderjahre',\n",
       " 'Heimatlos; Geschichten für Kinder und auch für solche, welche die Kinder lieb haben, 1. Band',\n",
       " 'Maezli; A Story of the Swiss Valleys',\n",
       " 'Moni the Goat-boy',\n",
       " 'Rico and Wiseli',\n",
       " 'Toni, the Little Woodcarver',\n",
       " 'Uncle Titus and His Visit to the Country',\n",
       " 'Veronica And Other Friends; Two Stories For Children',\n",
       " 'Vom This, der doch etwas wird Erzählung',\n",
       " 'Was die Großmutter gelehrt hat Erzählung',\n",
       " 'What Sami Sings with the Birds',\n",
       " 'Wie Wiselis Weg gefunden wird Erzählung',\n",
       " 'Wo Gritlis Kinder hingekommen sind Geschichten für Kinder und auch für solche, welche die Kinder lieb haben, 8. Band',\n",
       " 'The Black Arrow',\n",
       " 'Kidnapped',\n",
       " 'Treasure Island',\n",
       " \"Uncle Tom's Cabin, Young Folks' Edition\",\n",
       " 'The Arabian Nights: Their Best-known Tales',\n",
       " \"The Birds' Christmas Carol\",\n",
       " \"Children's Rights: A book of nursery logic\",\n",
       " 'The Diary of a Goose Girl',\n",
       " 'The Flag-Raising',\n",
       " 'Homespun Tales',\n",
       " 'Marm Lisa',\n",
       " \"Mother Carey's Chickens\",\n",
       " 'New Chronicles of Rebecca',\n",
       " \"Penelope's English Experiences\",\n",
       " \"Penelope's Experiences in Scotland\",\n",
       " \"Penelope's Irish Experiences\",\n",
       " \"Penelope's Postscripts\",\n",
       " 'Rebecca of Sunnybrook Farm',\n",
       " \"Rose O' the River\",\n",
       " 'The Story Hour',\n",
       " 'The Story of Patsy',\n",
       " 'Tales of Wonder Every Child Should Know',\n",
       " 'The Talking Beasts',\n",
       " \"Timothy's Quest\",\n",
       " 'The Happy Prince and Other Tales',\n",
       " 'Children and Their Books',\n",
       " 'A History of Nursery Rhymes',\n",
       " \"Children's Literature; A Textbook of Sources for Teachers and Teacher-Training Classes\"]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titles = []\n",
    "for x, y  in enumerate(driver.find_elements_by_xpath('./html/body/div[1]/div[2]/div[2]/div/ul/li')):\n",
    "    try:\n",
    "        if [i.get_attribute('alt') for i in y.find_elements_by_xpath('./a[@class=\"image\"]/img')][0] == 'BookIcon.png':\n",
    "            titles.append(y.find_elements_by_xpath('./a')[0].text)\n",
    "        else:\n",
    "            continue\n",
    "    except:\n",
    "        continue\n",
    "titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Louisa May Alcott, 1832-1888\n",
      "J. M. Barrie, 1832-1888\n",
      "Frank L. Baum (Lyman Frank), 1856-1919\n",
      "Margery Williams Bianco 1880-1944\n",
      "John Brown (1810-1882)\n",
      "Thornton W. Burgess 1874-1965\n",
      "Frances Hodgson Burnett (1849-1924)\n",
      "Dorothy Canfield (Fisher) (1879-1958)\n",
      "Carroll, Lewis, 1832-1898\n",
      "Susan Coolidge (1835-1905)\n",
      "Charles Dickens (1812-1870)\n",
      "Ewing, Juliana Horatia Gatty, 1841-1885\n",
      "Kenneth Grahame (1859-1932)\n",
      "Joel Chandler Harris (1848-1908)\n",
      "Nathaniel Hawthorne (1804-1864)\n",
      "Washington Irving (1783-1859)\n",
      "Charles Kingsley (1819-1875)\n",
      "Rudyard Kipling (1865-1936)\n",
      "Andrew Lang (1844-1912)\n",
      "Hugh Lofting (1886-1947)\n",
      "Henry Wadsworth Longfellow (1807-1882)\n",
      "George MacDonald (1824-1905)\n",
      "Frederick Marryat (1792-1848)\n",
      "Lucy Maud Montgomery (1874-1942)\n",
      "Mulock, Dinah Maria 1826-1887\n",
      "E. Nesbit (1858 - 1924)\n",
      "Ouida (Ramée, Louise de la) (1839-1908)\n",
      "Christoph von Schmid, (1768-1854)\n",
      "Anna Sewell (1820-1878)\n",
      "Margaret Sidney (1844-1924)\n",
      "Johanna Spyri (1827-1901)\n",
      "Robert Louis Stevenson (1850-1894)\n",
      "Harriet Beecher Stowe (1811-1896)\n",
      "Kate Douglas Wiggin (1856-1923)\n",
      "Wilde, Oscar, 1854-1900\n",
      "About Children's Books\n"
     ]
    }
   ],
   "source": [
    "num_titles = []\n",
    "for x in driver.find_elements_by_xpath('./html/body/div[1]/div[2]/div[2]/div/h2'):\n",
    "    print(x.text)\n",
    "# num_titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Louisa May Alcott,',\n",
       " 'J. M. Barrie,',\n",
       " 'Frank L. Baum (Lyman Frank),',\n",
       " 'Margery Williams Bianco',\n",
       " 'John Brown',\n",
       " 'Thornton W. Burgess',\n",
       " 'Frances Hodgson Burnett',\n",
       " 'Dorothy Canfield (Fisher)',\n",
       " 'Carroll, Lewis,',\n",
       " 'Susan Coolidge',\n",
       " 'Charles Dickens',\n",
       " 'Ewing, Juliana Horatia Gatty,',\n",
       " 'Kenneth Grahame',\n",
       " 'Joel Chandler Harris',\n",
       " 'Nathaniel Hawthorne',\n",
       " 'Washington Irving',\n",
       " 'Charles Kingsley',\n",
       " 'Rudyard Kipling',\n",
       " 'Andrew Lang',\n",
       " 'Hugh Lofting',\n",
       " 'Henry Wadsworth Longfellow',\n",
       " 'George MacDonald',\n",
       " 'Frederick Marryat',\n",
       " 'Lucy Maud Montgomery',\n",
       " 'Mulock, Dinah Maria',\n",
       " 'E. Nesbit',\n",
       " 'Ouida (Ramée, Louise de la)',\n",
       " 'Christoph von Schmid,',\n",
       " 'Anna Sewell',\n",
       " 'Margaret Sidney',\n",
       " 'Johanna Spyri',\n",
       " 'Robert Louis Stevenson',\n",
       " 'Harriet Beecher Stowe',\n",
       " 'Kate Douglas Wiggin',\n",
       " 'Wilde, Oscar,',\n",
       " \"About Children's Books\"]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authors = []\n",
    "authors_source = driver.find_elements_by_class_name('mw-headline')\n",
    "for x in authors_source:\n",
    "    authors.append(x.text.split(' (1')[0].split(' 1')[0])\n",
    "authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.gutenberg.org/ebooks/22234',\n",
       " 'https://www.gutenberg.org/ebooks/2726',\n",
       " 'https://www.gutenberg.org/ebooks/163',\n",
       " 'https://www.gutenberg.org/ebooks/5830',\n",
       " 'https://www.gutenberg.org/ebooks/2786',\n",
       " 'https://www.gutenberg.org/ebooks/3499',\n",
       " 'https://www.gutenberg.org/ebooks/10360',\n",
       " 'https://www.gutenberg.org/ebooks/2788',\n",
       " 'https://www.gutenberg.org/ebooks/514',\n",
       " 'https://www.gutenberg.org/ebooks/7425',\n",
       " 'https://www.gutenberg.org/ebooks/5352',\n",
       " 'https://www.gutenberg.org/ebooks/3806',\n",
       " 'https://www.gutenberg.org/ebooks/8188',\n",
       " 'https://www.gutenberg.org/ebooks/2787',\n",
       " 'https://www.gutenberg.org/ebooks/17337',\n",
       " 'https://www.gutenberg.org/ebooks/21946',\n",
       " 'https://www.gutenberg.org/ebooks/2804',\n",
       " 'https://www.gutenberg.org/ebooks/22022',\n",
       " 'https://www.gutenberg.org/ebooks/3795',\n",
       " 'https://www.gutenberg.org/ebooks/16',\n",
       " 'https://www.gutenberg.org/ebooks/4357',\n",
       " 'https://www.gutenberg.org/ebooks/22566',\n",
       " 'https://www.gutenberg.org/ebooks/517',\n",
       " 'https://www.gutenberg.org/ebooks/518',\n",
       " 'https://www.gutenberg.org/ebooks/961',\n",
       " 'https://www.gutenberg.org/ebooks/519',\n",
       " 'https://www.gutenberg.org/ebooks/520',\n",
       " 'https://www.gutenberg.org/ebooks/25519',\n",
       " 'https://www.gutenberg.org/ebooks/24459',\n",
       " 'https://www.gutenberg.org/ebooks/419',\n",
       " 'https://www.gutenberg.org/ebooks/54',\n",
       " 'https://www.gutenberg.org/ebooks/5660',\n",
       " 'https://www.gutenberg.org/ebooks/22225',\n",
       " 'https://www.gutenberg.org/ebooks/436',\n",
       " 'https://www.gutenberg.org/ebooks/5312',\n",
       " 'https://www.gutenberg.org/ebooks/486',\n",
       " 'https://www.gutenberg.org/ebooks/955',\n",
       " 'https://www.gutenberg.org/ebooks/25581',\n",
       " 'https://www.gutenberg.org/ebooks/485',\n",
       " 'https://www.gutenberg.org/ebooks/957',\n",
       " 'https://www.gutenberg.org/ebooks/4358',\n",
       " 'https://www.gutenberg.org/ebooks/4356',\n",
       " 'https://www.gutenberg.org/ebooks/16259',\n",
       " 'https://www.gutenberg.org/ebooks/956',\n",
       " 'https://www.gutenberg.org/ebooks/960',\n",
       " 'https://www.gutenberg.org/ebooks/21914',\n",
       " 'https://www.gutenberg.org/ebooks/55',\n",
       " 'https://www.gutenberg.org/ebooks/11757',\n",
       " 'https://www.gutenberg.org/ebooks/5420',\n",
       " 'https://www.gutenberg.org/ebooks/14375',\n",
       " 'https://www.gutenberg.org/ebooks/5110',\n",
       " 'https://www.gutenberg.org/ebooks/21015',\n",
       " 'https://www.gutenberg.org/ebooks/5844',\n",
       " 'https://www.gutenberg.org/ebooks/19079',\n",
       " 'https://www.gutenberg.org/ebooks/11915',\n",
       " 'https://www.gutenberg.org/ebooks/12630',\n",
       " 'https://www.gutenberg.org/ebooks/19092',\n",
       " 'https://www.gutenberg.org/ebooks/2493',\n",
       " 'https://www.gutenberg.org/ebooks/5846',\n",
       " 'https://www.gutenberg.org/ebooks/15521',\n",
       " 'https://www.gutenberg.org/ebooks/1825',\n",
       " 'https://www.gutenberg.org/ebooks/14732',\n",
       " 'https://www.gutenberg.org/ebooks/4979',\n",
       " 'https://www.gutenberg.org/ebooks/15168',\n",
       " 'https://www.gutenberg.org/ebooks/13355',\n",
       " 'https://www.gutenberg.org/ebooks/4670',\n",
       " 'https://www.gutenberg.org/ebooks/20877',\n",
       " 'https://www.gutenberg.org/ebooks/17250',\n",
       " 'https://www.gutenberg.org/ebooks/14958',\n",
       " 'https://www.gutenberg.org/ebooks/5791',\n",
       " 'https://www.gutenberg.org/ebooks/4980',\n",
       " 'https://www.gutenberg.org/ebooks/2557',\n",
       " 'https://www.gutenberg.org/ebooks/4698',\n",
       " 'https://www.gutenberg.org/ebooks/460',\n",
       " 'https://www.gutenberg.org/ebooks/5302',\n",
       " 'https://www.gutenberg.org/ebooks/5303',\n",
       " 'https://www.gutenberg.org/ebooks/479',\n",
       " 'https://www.gutenberg.org/ebooks/146',\n",
       " 'https://www.gutenberg.org/ebooks/10466',\n",
       " 'https://www.gutenberg.org/ebooks/384',\n",
       " 'https://www.gutenberg.org/ebooks/5304',\n",
       " 'https://www.gutenberg.org/ebooks/24772',\n",
       " 'https://www.gutenberg.org/ebooks/8574',\n",
       " 'https://www.gutenberg.org/ebooks/113',\n",
       " 'https://www.gutenberg.org/ebooks/5347',\n",
       " 'https://www.gutenberg.org/ebooks/19033',\n",
       " 'https://www.gutenberg.org/ebooks/19778',\n",
       " 'https://www.gutenberg.org/ebooks/17482',\n",
       " 'https://www.gutenberg.org/ebooks/19002',\n",
       " 'https://www.gutenberg.org/ebooks/13',\n",
       " 'https://www.gutenberg.org/ebooks/12',\n",
       " 'https://www.gutenberg.org/ebooks/8994',\n",
       " 'https://www.gutenberg.org/ebooks/8995',\n",
       " 'https://www.gutenberg.org/ebooks/5141',\n",
       " 'https://www.gutenberg.org/ebooks/15798',\n",
       " 'https://www.gutenberg.org/ebooks/4560',\n",
       " 'https://www.gutenberg.org/ebooks/16021',\n",
       " 'https://www.gutenberg.org/ebooks/20795',\n",
       " 'https://www.gutenberg.org/ebooks/16121',\n",
       " 'https://www.gutenberg.org/ebooks/16052',\n",
       " 'https://www.gutenberg.org/ebooks/19859',\n",
       " 'https://www.gutenberg.org/ebooks/17069',\n",
       " 'https://www.gutenberg.org/ebooks/20351',\n",
       " 'https://www.gutenberg.org/ebooks/7865',\n",
       " 'https://www.gutenberg.org/ebooks/5601',\n",
       " 'https://www.gutenberg.org/ebooks/19644',\n",
       " 'https://www.gutenberg.org/ebooks/16540',\n",
       " 'https://www.gutenberg.org/ebooks/17772',\n",
       " 'https://www.gutenberg.org/ebooks/15592',\n",
       " 'https://www.gutenberg.org/ebooks/20425',\n",
       " 'https://www.gutenberg.org/ebooks/19360',\n",
       " 'https://www.gutenberg.org/ebooks/16686',\n",
       " 'https://www.gutenberg.org/ebooks/18077',\n",
       " 'https://www.gutenberg.org/ebooks/18156',\n",
       " 'https://www.gutenberg.org/ebooks/289',\n",
       " 'https://www.gutenberg.org/ebooks/23871',\n",
       " 'https://www.gutenberg.org/ebooks/19648',\n",
       " 'https://www.gutenberg.org/ebooks/23869',\n",
       " 'https://www.gutenberg.org/ebooks/2306',\n",
       " 'https://www.gutenberg.org/ebooks/24430',\n",
       " 'https://www.gutenberg.org/ebooks/9216',\n",
       " 'https://www.gutenberg.org/ebooks/9215',\n",
       " 'https://www.gutenberg.org/ebooks/9219',\n",
       " 'https://www.gutenberg.org/ebooks/9208',\n",
       " 'https://www.gutenberg.org/ebooks/9218',\n",
       " 'https://www.gutenberg.org/ebooks/508',\n",
       " 'https://www.gutenberg.org/ebooks/9255',\n",
       " 'https://www.gutenberg.org/ebooks/9209',\n",
       " 'https://www.gutenberg.org/ebooks/9217',\n",
       " 'https://www.gutenberg.org/ebooks/9202',\n",
       " 'https://www.gutenberg.org/ebooks/9258',\n",
       " 'https://www.gutenberg.org/ebooks/9256',\n",
       " 'https://www.gutenberg.org/ebooks/9213',\n",
       " 'https://www.gutenberg.org/ebooks/9205',\n",
       " 'https://www.gutenberg.org/ebooks/9211',\n",
       " 'https://www.gutenberg.org/ebooks/9212',\n",
       " 'https://www.gutenberg.org/ebooks/9201',\n",
       " 'https://www.gutenberg.org/ebooks/976',\n",
       " 'https://www.gutenberg.org/ebooks/9220',\n",
       " 'https://www.gutenberg.org/ebooks/9257',\n",
       " 'https://www.gutenberg.org/ebooks/9206',\n",
       " 'https://www.gutenberg.org/ebooks/13707',\n",
       " 'https://www.gutenberg.org/ebooks/9210',\n",
       " 'https://www.gutenberg.org/ebooks/9207',\n",
       " 'https://www.gutenberg.org/ebooks/9214',\n",
       " 'https://www.gutenberg.org/ebooks/41',\n",
       " 'https://www.gutenberg.org/ebooks/19293',\n",
       " 'https://www.gutenberg.org/ebooks/25564',\n",
       " 'https://www.gutenberg.org/ebooks/236',\n",
       " 'https://www.gutenberg.org/ebooks/1937',\n",
       " 'https://www.gutenberg.org/ebooks/2781',\n",
       " 'https://www.gutenberg.org/ebooks/557',\n",
       " 'https://www.gutenberg.org/ebooks/2226',\n",
       " 'https://www.gutenberg.org/ebooks/503',\n",
       " 'https://www.gutenberg.org/ebooks/3282',\n",
       " 'https://www.gutenberg.org/ebooks/2435',\n",
       " 'https://www.gutenberg.org/ebooks/7277',\n",
       " 'https://www.gutenberg.org/ebooks/6746',\n",
       " 'https://www.gutenberg.org/ebooks/3454',\n",
       " 'https://www.gutenberg.org/ebooks/3027',\n",
       " 'https://www.gutenberg.org/ebooks/5615',\n",
       " 'https://www.gutenberg.org/ebooks/540',\n",
       " 'https://www.gutenberg.org/ebooks/641',\n",
       " 'https://www.gutenberg.org/ebooks/640',\n",
       " 'https://www.gutenberg.org/ebooks/21935',\n",
       " 'https://www.gutenberg.org/ebooks/21994',\n",
       " 'https://www.gutenberg.org/ebooks/501',\n",
       " 'https://www.gutenberg.org/ebooks/1154',\n",
       " 'https://www.gutenberg.org/ebooks/9080',\n",
       " 'https://www.gutenberg.org/ebooks/19',\n",
       " 'https://www.gutenberg.org/ebooks/225',\n",
       " 'https://www.gutenberg.org/ebooks/697',\n",
       " 'https://www.gutenberg.org/ebooks/18811',\n",
       " 'https://www.gutenberg.org/ebooks/709',\n",
       " 'https://www.gutenberg.org/ebooks/708',\n",
       " 'https://www.gutenberg.org/ebooks/6471',\n",
       " 'https://www.gutenberg.org/ebooks/47',\n",
       " 'https://www.gutenberg.org/ebooks/45',\n",
       " 'https://www.gutenberg.org/ebooks/51',\n",
       " 'https://www.gutenberg.org/ebooks/544',\n",
       " 'https://www.gutenberg.org/ebooks/1354',\n",
       " 'https://www.gutenberg.org/ebooks/5340',\n",
       " 'https://www.gutenberg.org/ebooks/316',\n",
       " 'https://www.gutenberg.org/ebooks/5341',\n",
       " 'https://www.gutenberg.org/ebooks/5343',\n",
       " 'https://www.gutenberg.org/ebooks/3796',\n",
       " 'https://www.gutenberg.org/ebooks/5342',\n",
       " 'https://www.gutenberg.org/ebooks/496',\n",
       " 'https://www.gutenberg.org/ebooks/24053',\n",
       " 'https://www.gutenberg.org/ebooks/1430',\n",
       " 'https://www.gutenberg.org/ebooks/23661',\n",
       " 'https://www.gutenberg.org/ebooks/3536',\n",
       " 'https://www.gutenberg.org/ebooks/778',\n",
       " 'https://www.gutenberg.org/ebooks/4378',\n",
       " 'https://www.gutenberg.org/ebooks/20606',\n",
       " 'https://www.gutenberg.org/ebooks/25496',\n",
       " 'https://www.gutenberg.org/ebooks/836',\n",
       " 'https://www.gutenberg.org/ebooks/1874',\n",
       " 'https://www.gutenberg.org/ebooks/4513',\n",
       " 'https://www.gutenberg.org/ebooks/837',\n",
       " 'https://www.gutenberg.org/ebooks/770',\n",
       " 'https://www.gutenberg.org/ebooks/794',\n",
       " 'https://www.gutenberg.org/ebooks/13912',\n",
       " 'https://www.gutenberg.org/ebooks/5834',\n",
       " 'https://www.gutenberg.org/ebooks/7766',\n",
       " 'https://www.gutenberg.org/ebooks/20997',\n",
       " 'https://www.gutenberg.org/ebooks/14069',\n",
       " 'https://www.gutenberg.org/ebooks/271',\n",
       " 'https://www.gutenberg.org/ebooks/11860',\n",
       " 'https://www.gutenberg.org/ebooks/2770',\n",
       " 'https://www.gutenberg.org/ebooks/5632',\n",
       " 'https://www.gutenberg.org/ebooks/7498',\n",
       " 'https://www.gutenberg.org/ebooks/7434',\n",
       " 'https://www.gutenberg.org/ebooks/6987',\n",
       " 'https://www.gutenberg.org/ebooks/6418',\n",
       " 'https://www.gutenberg.org/ebooks/6380',\n",
       " 'https://www.gutenberg.org/ebooks/10436',\n",
       " 'https://www.gutenberg.org/ebooks/15727',\n",
       " 'https://www.gutenberg.org/ebooks/20781',\n",
       " 'https://www.gutenberg.org/ebooks/7512',\n",
       " 'https://www.gutenberg.org/ebooks/7500',\n",
       " 'https://www.gutenberg.org/ebooks/7511',\n",
       " 'https://www.gutenberg.org/ebooks/20780',\n",
       " 'https://www.gutenberg.org/ebooks/10142',\n",
       " 'https://www.gutenberg.org/ebooks/9383',\n",
       " 'https://www.gutenberg.org/ebooks/9075',\n",
       " 'https://www.gutenberg.org/ebooks/14128',\n",
       " 'https://www.gutenberg.org/ebooks/14710',\n",
       " 'https://www.gutenberg.org/ebooks/14627',\n",
       " 'https://www.gutenberg.org/ebooks/9859',\n",
       " 'https://www.gutenberg.org/ebooks/9861',\n",
       " 'https://www.gutenberg.org/ebooks/9482',\n",
       " 'https://www.gutenberg.org/ebooks/7888',\n",
       " 'https://www.gutenberg.org/ebooks/22570',\n",
       " 'https://www.gutenberg.org/ebooks/848',\n",
       " 'https://www.gutenberg.org/ebooks/421',\n",
       " 'https://www.gutenberg.org/ebooks/120',\n",
       " 'https://www.gutenberg.org/ebooks/11171',\n",
       " 'https://www.gutenberg.org/ebooks/20916',\n",
       " 'https://www.gutenberg.org/ebooks/24286',\n",
       " 'https://www.gutenberg.org/ebooks/10335',\n",
       " 'https://www.gutenberg.org/ebooks/1867',\n",
       " 'https://www.gutenberg.org/ebooks/2315',\n",
       " 'https://www.gutenberg.org/ebooks/3492',\n",
       " 'https://www.gutenberg.org/ebooks/3149',\n",
       " 'https://www.gutenberg.org/ebooks/10540',\n",
       " 'https://www.gutenberg.org/ebooks/1375',\n",
       " 'https://www.gutenberg.org/ebooks/1278',\n",
       " 'https://www.gutenberg.org/ebooks/1217',\n",
       " 'https://www.gutenberg.org/ebooks/1391',\n",
       " 'https://www.gutenberg.org/ebooks/1868',\n",
       " 'https://www.gutenberg.org/ebooks/498',\n",
       " 'https://www.gutenberg.org/ebooks/1033',\n",
       " 'https://www.gutenberg.org/ebooks/5835',\n",
       " 'https://www.gutenberg.org/ebooks/13506',\n",
       " 'https://www.gutenberg.org/ebooks/19461',\n",
       " 'https://www.gutenberg.org/ebooks/13815',\n",
       " 'https://www.gutenberg.org/ebooks/18531',\n",
       " 'https://www.gutenberg.org/ebooks/902',\n",
       " 'https://www.gutenberg.org/ebooks/22604',\n",
       " 'https://www.gutenberg.org/ebooks/24065',\n",
       " 'https://www.gutenberg.org/ebooks/25545']"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links = []\n",
    "for x, y  in enumerate(driver.find_elements_by_xpath('./html/body/div[1]/div[2]/div[2]/div/ul/li')):\n",
    "    try:\n",
    "        if [i.get_attribute('alt') for i in y.find_elements_by_xpath('./a[@class=\"image\"]/img')][0] == 'BookIcon.png':\n",
    "            links.append(y.find_elements_by_xpath('./a')[0].get_attribute('href'))\n",
    "        else:\n",
    "            continue\n",
    "    except:\n",
    "        continue\n",
    "links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(books) == len(links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.quit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prep URLs for Webscrapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['22234',\n",
       " '2726',\n",
       " '163',\n",
       " '5830',\n",
       " '2786',\n",
       " '3499',\n",
       " '10360',\n",
       " '2788',\n",
       " '514',\n",
       " '7425',\n",
       " '5352',\n",
       " '3806',\n",
       " '8188',\n",
       " '2787',\n",
       " '17337',\n",
       " '21946',\n",
       " '2804',\n",
       " '22022',\n",
       " '3795',\n",
       " '16',\n",
       " '4357',\n",
       " '22566',\n",
       " '517',\n",
       " '518',\n",
       " '961',\n",
       " '519',\n",
       " '520',\n",
       " '25519',\n",
       " '24459',\n",
       " '419',\n",
       " '54',\n",
       " '5660',\n",
       " '22225',\n",
       " '436',\n",
       " '5312',\n",
       " '486',\n",
       " '955',\n",
       " '25581',\n",
       " '485',\n",
       " '957',\n",
       " '4358',\n",
       " '4356',\n",
       " '16259',\n",
       " '956',\n",
       " '960',\n",
       " '21914',\n",
       " '55',\n",
       " '11757',\n",
       " '5420',\n",
       " '14375',\n",
       " '5110',\n",
       " '21015',\n",
       " '5844',\n",
       " '19079',\n",
       " '11915',\n",
       " '12630',\n",
       " '19092',\n",
       " '2493',\n",
       " '5846',\n",
       " '15521',\n",
       " '1825',\n",
       " '14732',\n",
       " '4979',\n",
       " '15168',\n",
       " '13355',\n",
       " '4670',\n",
       " '20877',\n",
       " '17250',\n",
       " '14958',\n",
       " '5791',\n",
       " '4980',\n",
       " '2557',\n",
       " '4698',\n",
       " '460',\n",
       " '5302',\n",
       " '5303',\n",
       " '479',\n",
       " '146',\n",
       " '10466',\n",
       " '384',\n",
       " '5304',\n",
       " '24772',\n",
       " '8574',\n",
       " '113',\n",
       " '5347',\n",
       " '19033',\n",
       " '19778',\n",
       " '17482',\n",
       " '19002',\n",
       " '13',\n",
       " '12',\n",
       " '8994',\n",
       " '8995',\n",
       " '5141',\n",
       " '15798',\n",
       " '4560',\n",
       " '16021',\n",
       " '20795',\n",
       " '16121',\n",
       " '16052',\n",
       " '19859',\n",
       " '17069',\n",
       " '20351',\n",
       " '7865',\n",
       " '5601',\n",
       " '19644',\n",
       " '16540',\n",
       " '17772',\n",
       " '15592',\n",
       " '20425',\n",
       " '19360',\n",
       " '16686',\n",
       " '18077',\n",
       " '18156',\n",
       " '289',\n",
       " '23871',\n",
       " '19648',\n",
       " '23869',\n",
       " '2306',\n",
       " '24430',\n",
       " '9216',\n",
       " '9215',\n",
       " '9219',\n",
       " '9208',\n",
       " '9218',\n",
       " '508',\n",
       " '9255',\n",
       " '9209',\n",
       " '9217',\n",
       " '9202',\n",
       " '9258',\n",
       " '9256',\n",
       " '9213',\n",
       " '9205',\n",
       " '9211',\n",
       " '9212',\n",
       " '9201',\n",
       " '976',\n",
       " '9220',\n",
       " '9257',\n",
       " '9206',\n",
       " '13707',\n",
       " '9210',\n",
       " '9207',\n",
       " '9214',\n",
       " '41',\n",
       " '19293',\n",
       " '25564',\n",
       " '236',\n",
       " '1937',\n",
       " '2781',\n",
       " '557',\n",
       " '2226',\n",
       " '503',\n",
       " '3282',\n",
       " '2435',\n",
       " '7277',\n",
       " '6746',\n",
       " '3454',\n",
       " '3027',\n",
       " '5615',\n",
       " '540',\n",
       " '641',\n",
       " '640',\n",
       " '21935',\n",
       " '21994',\n",
       " '501',\n",
       " '1154',\n",
       " '9080',\n",
       " '19',\n",
       " '225',\n",
       " '697',\n",
       " '18811',\n",
       " '709',\n",
       " '708',\n",
       " '6471',\n",
       " '47',\n",
       " '45',\n",
       " '51',\n",
       " '544',\n",
       " '1354',\n",
       " '5340',\n",
       " '316',\n",
       " '5341',\n",
       " '5343',\n",
       " '3796',\n",
       " '5342',\n",
       " '496',\n",
       " '24053',\n",
       " '1430',\n",
       " '23661',\n",
       " '3536',\n",
       " '778',\n",
       " '4378',\n",
       " '20606',\n",
       " '25496',\n",
       " '836',\n",
       " '1874',\n",
       " '4513',\n",
       " '837',\n",
       " '770',\n",
       " '794',\n",
       " '13912',\n",
       " '5834',\n",
       " '7766',\n",
       " '20997',\n",
       " '14069',\n",
       " '271',\n",
       " '11860',\n",
       " '2770',\n",
       " '5632',\n",
       " '7498',\n",
       " '7434',\n",
       " '6987',\n",
       " '6418',\n",
       " '6380',\n",
       " '10436',\n",
       " '15727',\n",
       " '20781',\n",
       " '7512',\n",
       " '7500',\n",
       " '7511',\n",
       " '20780',\n",
       " '10142',\n",
       " '9383',\n",
       " '9075',\n",
       " '14128',\n",
       " '14710',\n",
       " '14627',\n",
       " '9859',\n",
       " '9861',\n",
       " '9482',\n",
       " '7888',\n",
       " '22570',\n",
       " '848',\n",
       " '421',\n",
       " '120',\n",
       " '11171',\n",
       " '20916',\n",
       " '24286',\n",
       " '10335',\n",
       " '1867',\n",
       " '2315',\n",
       " '3492',\n",
       " '3149',\n",
       " '10540',\n",
       " '1375',\n",
       " '1278',\n",
       " '1217',\n",
       " '1391',\n",
       " '1868',\n",
       " '498',\n",
       " '1033',\n",
       " '5835',\n",
       " '13506',\n",
       " '19461',\n",
       " '13815',\n",
       " '18531',\n",
       " '902',\n",
       " '22604',\n",
       " '24065',\n",
       " '25545']"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links_to_search = []\n",
    "for x in links:\n",
    "    links_to_search.append(x.split('/')[4])\n",
    "links_to_search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Webscrapping for Full Texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "1\n",
      "Text acquired!\n",
      "2\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "3\n",
      "Text acquired!\n",
      "4\n",
      "Text acquired!\n",
      "5\n",
      "Text acquired!\n",
      "6\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "7\n",
      "Text acquired!\n",
      "8\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "9\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "10\n",
      "Text acquired!\n",
      "11\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "12\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "13\n",
      "Text acquired!\n",
      "14\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "15\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "16\n",
      "Text acquired!\n",
      "17\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "18\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "19\n",
      "Text acquired!\n",
      "20\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "21\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "22\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "23\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "24\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "25\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "26\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "27\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "28\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "29\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "30\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "31\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "32\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "33\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "34\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "35\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "36\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "37\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "38\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "39\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "40\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "41\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "42\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "43\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "44\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "45\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "46\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "47\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "48\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "49\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "50\n",
      "Text acquired!\n",
      "51\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "52\n",
      "Text acquired!\n",
      "53\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "54\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "55\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "56\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "57\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "58\n",
      "Text acquired!\n",
      "59\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "60\n",
      "Text acquired!\n",
      "61\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "62\n",
      "Text acquired!\n",
      "63\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "64\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "65\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "66\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "67\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "68\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "69\n",
      "Text acquired!\n",
      "70\n",
      "Text acquired!\n",
      "71\n",
      "Text acquired!\n",
      "72\n",
      "Text acquired!\n",
      "73\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "74\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "75\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "76\n",
      "Text acquired!\n",
      "77\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "78\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "79\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "80\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "81\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "82\n",
      "Text acquired!\n",
      "83\n",
      "Text acquired!\n",
      "84\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "85\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "86\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "87\n",
      "Text acquired!\n",
      "88\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "89\n",
      "Text acquired!\n",
      "90\n",
      "Text acquired!\n",
      "91\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "92\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "93\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "94\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "95\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "96\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "97\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "98\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "99\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "100\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "101\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "102\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "103\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "104\n",
      "Text acquired!\n",
      "105\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "106\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "107\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "108\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "109\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "110\n",
      "Text acquired!\n",
      "111\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "112\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "113\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "114\n",
      "Text acquired!\n",
      "115\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "116\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "117\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "118\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "119\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "120\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "121\n",
      "Text acquired!\n",
      "122\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "123\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "124\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "125\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "126\n",
      "Text acquired!\n",
      "127\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "128\n",
      "Text acquired!\n",
      "129\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "130\n",
      "Text acquired!\n",
      "131\n",
      "Text acquired!\n",
      "132\n",
      "Text acquired!\n",
      "133\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "134\n",
      "Text acquired!\n",
      "135\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "136\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "137\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "138\n",
      "Text acquired!\n",
      "139\n",
      "Text acquired!\n",
      "140\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "141\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "142\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "143\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "144\n",
      "Text acquired!\n",
      "145\n",
      "Text acquired!\n",
      "146\n",
      "Text acquired!\n",
      "147\n",
      "Text acquired!\n",
      "148\n",
      "Text acquired!\n",
      "149\n",
      "Text acquired!\n",
      "150\n",
      "Text acquired!\n",
      "151\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "152\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "153\n",
      "Text acquired!\n",
      "154\n",
      "Text acquired!\n",
      "155\n",
      "Text acquired!\n",
      "156\n",
      "Text acquired!\n",
      "157\n",
      "Text acquired!\n",
      "158\n",
      "Text acquired!\n",
      "159\n",
      "Text acquired!\n",
      "160\n",
      "Text acquired!\n",
      "161\n",
      "Text acquired!\n",
      "162\n",
      "Text acquired!\n",
      "163\n",
      "Text acquired!\n",
      "164\n",
      "Text acquired!\n",
      "165\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "166\n",
      "Text acquired!\n",
      "167\n",
      "Text acquired!\n",
      "168\n",
      "Text acquired!\n",
      "169\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "170\n",
      "Text acquired!\n",
      "171\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "172\n",
      "Text acquired!\n",
      "173\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "174\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "175\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "176\n",
      "Text acquired!\n",
      "177\n",
      "Text acquired!\n",
      "178\n",
      "Text acquired!\n",
      "179\n",
      "Message: Unable to locate element: //pre\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text acquired!\n",
      "180\n",
      "Text acquired!\n",
      "181\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "182\n",
      "Text acquired!\n",
      "183\n",
      "Text acquired!\n",
      "184\n",
      "Text acquired!\n",
      "185\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "186\n",
      "Text acquired!\n",
      "187\n",
      "Text acquired!\n",
      "188\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "189\n",
      "Text acquired!\n",
      "190\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "191\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "192\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "193\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "194\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "195\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "196\n",
      "Text acquired!\n",
      "197\n",
      "Text acquired!\n",
      "198\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "199\n",
      "Text acquired!\n",
      "200\n",
      "Text acquired!\n",
      "201\n",
      "Text acquired!\n",
      "202\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "203\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "204\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "205\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "206\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "207\n",
      "Text acquired!\n",
      "208\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "209\n",
      "Text acquired!\n",
      "210\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "211\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "212\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "213\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "214\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "215\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "216\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "217\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "218\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "219\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "220\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "221\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "222\n",
      "Text acquired!\n",
      "223\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "224\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "225\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "226\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "227\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "228\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "229\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "230\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "231\n",
      "Text acquired!\n",
      "232\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "233\n",
      "Text acquired!\n",
      "234\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "235\n",
      "Text acquired!\n",
      "236\n",
      "Text acquired!\n",
      "237\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "238\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "239\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "240\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "241\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "242\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "243\n",
      "Text acquired!\n",
      "244\n",
      "Text acquired!\n",
      "245\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "246\n",
      "Text acquired!\n",
      "247\n",
      "Text acquired!\n",
      "248\n",
      "Text acquired!\n",
      "249\n",
      "Text acquired!\n",
      "250\n",
      "Text acquired!\n",
      "251\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "252\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "253\n",
      "Text acquired!\n",
      "254\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "255\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "256\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "257\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "258\n",
      "Text acquired!\n",
      "259\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n",
      "260\n",
      "Text acquired!\n",
      "261\n",
      "Message: Unable to locate element: //pre\n",
      "\n",
      "Text acquired!\n"
     ]
    }
   ],
   "source": [
    "texts = []\n",
    "urls_used = []\n",
    "urls_not_used = []\n",
    "for i in range(0, len(links_to_search)):\n",
    "    to_find = links_to_search[i]\n",
    "    print(i)\n",
    "    try:\n",
    "        driver = webdriver.Firefox()\n",
    "        driver.implicitly_wait(5)\n",
    "        driver.get(\"https://www.gutenberg.org/files/\"+to_find+\"/\"+to_find+\"-0.txt\")\n",
    "        text = driver.find_element_by_xpath('//pre')\n",
    "        texts.append(text.text)\n",
    "        driver.quit()\n",
    "        urls_used.append(i)\n",
    "        print('Text acquired!')\n",
    "    except NoSuchElementException as inst:\n",
    "        print(inst)          # __str__ allows args to be printed directly,\n",
    "        driver = webdriver.Firefox()\n",
    "        driver.implicitly_wait(5)\n",
    "        driver.get(\"https://www.gutenberg.org/ebooks/\"+to_find+\".txt.utf-8\")\n",
    "        text = driver.find_element_by_xpath('//pre')\n",
    "        texts.append(text.text)\n",
    "        driver.quit()\n",
    "        urls_used.append(i)\n",
    "        print('Text acquired!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file = open('full_texts.pickle', 'wb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump([books, texts], file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('full_texts.pickle', 'rb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_text_titles, full_text_texts = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "special_cases = [57, 158, 189, 189]\n",
    "for x in special_cases:\n",
    "    full_text_titles.pop(x)\n",
    "    full_text_texts.pop(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "258 258 258\n"
     ]
    }
   ],
   "source": [
    "parse1 = []\n",
    "parse2 = []\n",
    "parse3 = []\n",
    "parse4 = []\n",
    "parse5 = []\n",
    "special_cases = [57, 159, 191, 192]\n",
    "for x in range(0, len(full_text_texts)):\n",
    "#     print('1. removing license agreement at end')\n",
    "    parse1.append(re.sub('\\n', ' ', full_text_texts[x]))\n",
    "#     print('2. removing extras at the end of the text')\n",
    "    if len(parse1[x].split('THE END'))==2:\n",
    "        parse2.append(parse1[x].split('THE END')[0])\n",
    "    elif len(parse1[x].split(\"End of the Project Gutenberg EBook\"))==2:\n",
    "        parse2.append(parse1[x].split(\"End of the Project Gutenberg EBook\")[0])\n",
    "    elif len(parse1[x].split('The End'))==2:\n",
    "        parse2.append(parse1[x].split('The End')[0])\n",
    "    elif len(parse1[x].split('End of Project Gutenberg\\'s'))==2:\n",
    "        parse2.append(parse1[x].split('End of Project Gutenberg\\'s')[0])\n",
    "    elif len(parse1[x].split('END OF THE PROJECT GUTENBERG EBOOK'))==2:\n",
    "        parse2.append(parse1[x].split('END OF THE PROJECT GUTENBERG EBOOK')[0]) \n",
    "    elif len(parse1[x].split('END OF THIS PROJECT GUTENBERG EBOOK'))==2:\n",
    "        parse2.append(parse1[x].split('END OF THIS PROJECT GUTENBERG EBOOK')[0])\n",
    "    elif len(parse1[x].split('The Project Gutenberg Etext'))==4:\n",
    "        parse2.append(parse1[x].split('The Project Gutenberg Etext')[2])\n",
    "    elif len(parse1[x].split('*Project Gutenberg Etext'))==4:\n",
    "        parse2.append(parse1[x].split('*Project Gutenberg Etext')[2])\n",
    "    else:\n",
    "        print('problem at: ', x)\n",
    "#     print('3. removing anything at ebook related at beginning of text')\n",
    "    if len(parse2[x].split('START OF THIS PROJECT GUTENBERG EBOOK'))==2:\n",
    "        parse3.append(parse2[x].split('START OF THIS PROJECT GUTENBERG EBOOK')[1])\n",
    "    elif len(parse2[x].split(\"START OF THE PROJECT GUTENBERG EBOOK\"))==2:\n",
    "        parse3.append(parse2[x].split(\"START OF THE PROJECT GUTENBERG EBOOK\")[1])\n",
    "    elif len(parse2[x].split(\"THIS EBOOK WAS ONE OF PROJECT GUTENBERG'S EARLY FILES PRODUCED AT A TIME WHEN PROOFING METHODS AND TOOLS WERE NOT WELL DEVELOPED.\"))==2:\n",
    "        parse3.append(parse2[x].split(\"THIS EBOOK WAS ONE OF PROJECT GUTENBERG'S EARLY FILES PRODUCED AT A TIME WHEN PROOFING METHODS AND TOOLS WERE NOT WELL DEVELOPED.\")[1])\n",
    "    elif len(parse2[x].split(\"THE SMALL PRINT! FOR PUBLIC DOMAIN ETEXTS\"))==2:\n",
    "        parse3.append(parse2[x].split(\"THE SMALL PRINT! FOR PUBLIC DOMAIN ETEXTS\")[1].split('*END*')[1])\n",
    "    else:\n",
    "        print('problem at: ', x)\n",
    "print(len(parse1), len(parse2), len(parse3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles_from_text = []\n",
    "parse4 = []\n",
    "for x in range(0, len(full_text_texts)):\n",
    "    if len(parse3[x].split('***'))==2:\n",
    "        parse4.append(parse3[x].split('***')[1])\n",
    "    elif len(parse3[x].split('***'))==48:\n",
    "        parse4.append(parse3[x].split('***')[47].split('*')[1])\n",
    "    elif len(parse3[x].split('***'))==3:\n",
    "        if len(parse3[x].split('***')[1].split('Online Distributed Proofreading Team'))==2:\n",
    "            parse4.append(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[1])\n",
    "        elif len(parse3[x].split('***')[1].split('Online Distributed Proofreading Team'))==1:\n",
    "            if len(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[0].split('.zip)'))==2:\n",
    "                parse4.append(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[0].split('.zip)')[1])\n",
    "            elif len(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[0].split('.zip)'))==1:\n",
    "                if len(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[0].split('.zip)')[0].split('.org'))==2:\n",
    "                    parse4.append(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[0].split('.zip)')[0].split('.org')[1])\n",
    "                elif len(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[0].split('.zip)')[0].split('.ac.uk'))==2:\n",
    "                    parse4.append(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[0].split('.zip)')[0].split('.ac.uk')[1])\n",
    "                elif len(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[0].split('.zip)')[0].split('.edu/women/.'))==2:\n",
    "                    parse4.append(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[0].split('.zip)')[0].split('.edu/women/.')[0])\n",
    "                elif len(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[0].split('.zip)')[0].split('     '))==839:\n",
    "                    parse4.append(parse3[x].split('***')[1].split('Online Distributed Proofreading Team')[0].split('.zip)')[0].split('This eBook was')[1])\n",
    "                else:\n",
    "                    parse4.append(parse3[x])\n",
    "            else:\n",
    "                print('review: ', x)\n",
    "        else:\n",
    "            print('look at: ', x)\n",
    "    elif len(parse3[x].split('***'))==7:\n",
    "        parse4.append(parse3[x])\n",
    "    else:\n",
    "        print('problem at: ', x)\n",
    "    titles_from_text.append(parse3[x].split('***')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "from nltk.tokenize import word_tokenize\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'myself', 'wasn', 'does', 'me', 'with', 'your', \"wouldn't\", 'into', 'who', 'has', 'how', 'on', 'in', 'under', 'nor', \"doesn't\", 'will', 'an', 'o', 'other', 'hers', 'him', 'shan', 'our', 'were', 'ourselves', 'himself', 'mustn', 's', 'don', 'very', \"hadn't\", 'whom', 'about', 'when', \"hasn't\", 'doing', 'am', 'themselves', 'being', 'below', 'did', 'i', 'having', 'a', 'if', 'each', 'between', 'these', 'why', 'now', 'had', 've', 'all', 'more', 'aren', 'while', 'down', 'own', 'than', 'by', 'this', 'where', 'few', 'should', 'we', \"won't\", \"mustn't\", 'again', 'such', 'then', 'ain', 'ma', 'so', 'wouldn', 'above', \"didn't\", 'couldn', \"mightn't\", 'his', 'as', 'been', 'isn', 'she', 'both', 'm', 'yours', 'or', 'them', 'too', 'same', 'have', 'off', 'here', 'after', 'and', 'but', 'you', 'further', 'haven', 'he', \"she's\", 'y', 'against', 'during', 'that', 'itself', 'doesn', 'just', 'd', 'theirs', 'over', \"you'll\", 'which', 'up', 'ours', 'my', 'mightn', 'the', \"needn't\", 'out', 'they', \"aren't\", 'for', 'no', 'yourselves', \"it's\", 'only', \"you're\", 're', \"weren't\", 'what', 'their', 'from', 'before', 'it', 'any', \"shouldn't\", \"couldn't\", 'through', 'won', 'hasn', 'its', 'weren', \"wasn't\", 'some', 'until', 'can', 'are', 'to', \"don't\", 'hadn', \"that'll\", 'do', 'because', 'at', 'll', 'needn', \"you've\", 'is', 'was', 'of', 'didn', 'not', \"haven't\", 'herself', 'there', 'shouldn', 'those', \"isn't\", 'yourself', \"should've\", 't', \"you'd\", \"shan't\", 'be', 'her', 'once', 'most'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\eannefawcett\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping corpora\\stopwords.zip.\n"
     ]
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "stop_words=set(nltk.corpus.stopwords.words(\"english\"))\n",
    "print(stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "cleaning\n",
    "- pull out title\n",
    "- pull out author\n",
    "- pull out preface\n",
    "- pull out quote before text\n",
    "- pull out table of contents\n",
    "\n",
    "split data into what's being passed in\n",
    "\n",
    "train test split (for new data exclude test from tokenize) (for smaller dataset include test in tokenizer) \n",
    "(train test validate split (https://datascience.stackexchange.com/questions/15135/train-test-validation-set-splitting-in-sklearn))\n",
    "\n",
    "tokenize\n",
    "\n",
    "fit transform on training, transform only on the test\n",
    "\n",
    "\n",
    "google collab for cloud gpu ($10/month, daily allocation for free)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lexile Levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analytics Checkbox deselected.\n"
     ]
    }
   ],
   "source": [
    "driver = webdriver.Firefox()\n",
    "driver.implicitly_wait(5)\n",
    "driver.get(\"https://hub.lexile.com/find-a-book/search\")\n",
    "analytics = driver.find_element_by_xpath('//label[@for=\"checkbox_1\"]')\n",
    "result = analytics.is_selected()\n",
    "if result:\n",
    "    print('Checkbox already selected')\n",
    "else:\n",
    "    analytics.click()\n",
    "    print('Analytics Checkbox deselected.')\n",
    "save_analytics_button = driver.find_element_by_xpath('//button[text()=\"Save Preferences\"]')\n",
    "save_analytics_button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "books = []\n",
    "authors = []\n",
    "lexiles = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "Finding books\n",
      "Finding authors\n",
      "Finding levels\n",
      "2\n",
      "Finding books\n",
      "Finding authors\n",
      "Finding levels\n",
      "3\n",
      "Finding books\n",
      "Finding authors\n",
      "Finding levels\n",
      "4\n",
      "Finding books\n",
      "Finding authors\n",
      "Finding levels\n"
     ]
    }
   ],
   "source": [
    "for x in range(1, 5):\n",
    "    print(x)\n",
    "    value = full_text_titles[x]\n",
    "    search_bar = driver.find_element_by_xpath('//input[@name=\"quickSearch\"]')\n",
    "    search_bar.send_keys(value)\n",
    "    search_button = driver.find_element_by_xpath('//button[text()=\"Search\"]')\n",
    "    search_button.click()\n",
    "    print('Finding books')\n",
    "    titles = driver.find_elements_by_xpath('//*[@data-testid=\"book-title\"]')\n",
    "    book_titles = []\n",
    "    for result in titles:\n",
    "        book_titles.append(result.text)\n",
    "    print('Finding authors')\n",
    "    authors_of_books = driver.find_elements_by_xpath('//*[@data-testid=\"book-authors\"]')\n",
    "    book_authors = []\n",
    "    for result in authors_of_books:\n",
    "        book_authors.append(result.text)\n",
    "    print('Finding levels')\n",
    "    levels = driver.find_elements_by_class_name('sc-jKmXuR.gfpzur')\n",
    "    lexile_levels = []\n",
    "    for result in levels:\n",
    "        lexile_levels.append(result.text)\n",
    "    books.append(book_titles)\n",
    "    authors.append(book_authors)\n",
    "    lexiles.append(lexile_levels)\n",
    "    new_search = driver.find_element_by_xpath('//button[text()=\"New Search\"]')\n",
    "    new_search.click()\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Eight Cousins',\n",
       "  \"Marcia's Madness\",\n",
       "  'The Bathing Costume',\n",
       "  'Rose in Bloom',\n",
       "  \"Rebecca's Rashness\",\n",
       "  'My Name Is Blessing'],\n",
       " ['Flower Fables',\n",
       "  \"Vietnamese Children's\\nFavorite Stories\",\n",
       "  \"Thai Children's Favorite\\nStories\"],\n",
       " [],\n",
       " ['Jack and Jill',\n",
       "  'Jack and Jill',\n",
       "  'Jack and Jill',\n",
       "  'Jack And Jill',\n",
       "  'Jack and Jill',\n",
       "  'Jack and Jill',\n",
       "  'Jack and Jill',\n",
       "  'Jack and Jill / Jack and\\nJill Go Skating',\n",
       "  'Where Are Jack and\\nJill?',\n",
       "  'Jack and Jill Flip-Side\\nRhymes',\n",
       "  'Jack and Jill and T-Ball\\nBill',\n",
       "  'Jack and Jill and Big\\nDog Bill: A Phonics R…',\n",
       "  'In a Glass Grimmly',\n",
       "  'Cycler',\n",
       "  'Recycler',\n",
       "  'Things Are Looking\\nGrimm, Jill',\n",
       "  'Friendly Faces: People\\nNursery Rhymes',\n",
       "  'Where is Santa?',\n",
       "  \"Baby's First Mother\\nGoose Treasury\",\n",
       "  'Mary Had a Little Jam',\n",
       "  'Over the Candlestick:\\nClassic Nursery Rhym…',\n",
       "  'Soccer Time!',\n",
       "  'The Great Nursery\\nRhyme Disaster',\n",
       "  'I See a Leaf',\n",
       "  'The 100th Day']]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "books"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['by Louisa May Alcott',\n",
       "  'by Lauren\\nBaratz-Logsted, Jackie…',\n",
       "  'by Charlotte Moundlic',\n",
       "  'by Louisa May Alcott',\n",
       "  'by Lauren\\nBaratz-Logsted',\n",
       "  'by Eric Walters'],\n",
       " ['by Louisa May Alcott', 'by Phuoc Thi Minh Tran', 'by Marian D. Toth'],\n",
       " [],\n",
       " ['by Melissa Everett',\n",
       "  'by N/A',\n",
       "  'by Hazel Quintanilla',\n",
       "  'by Anonymous',\n",
       "  'by Sharleen (Illustrator)\\nCollicott',\n",
       "  'by Phoebe Marsh',\n",
       "  'by Liza Woodruff',\n",
       "  'by Wes Magee',\n",
       "  'by Cynthia Rothman',\n",
       "  'by Christopher Harbo',\n",
       "  'by Terry Pierce',\n",
       "  'by Martha Weston',\n",
       "  'by Adam Gidwitz',\n",
       "  'by Lauren McLaughlin',\n",
       "  'by Lauren McLaughlin',\n",
       "  'by Dan Bar-el',\n",
       "  'by Robin Wells',\n",
       "  'by Sourcebooks Inc.',\n",
       "  'by Bruce Lansky',\n",
       "  'by Michael\\nMontgomery',\n",
       "  'by Terry Pierce',\n",
       "  'by Brian Rock',\n",
       "  'by Alayne Pick, Grace\\nMaccarone',\n",
       "  'by Alayne Pick, Grace\\nMaccarone']]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['1150L', '760L', '740L', '1360L', '800L', 'AD630L'],\n",
       " ['1200L', '960L', '880L'],\n",
       " [],\n",
       " ['AD550L',\n",
       "  '380L',\n",
       "  'NP',\n",
       "  '150L',\n",
       "  'NP',\n",
       "  '140L',\n",
       "  'AD290L',\n",
       "  'AD310L',\n",
       "  '90L',\n",
       "  '360L',\n",
       "  '230L',\n",
       "  '190L',\n",
       "  '630L',\n",
       "  'NC720L',\n",
       "  'HL630L',\n",
       "  '760L',\n",
       "  'NP',\n",
       "  '250L',\n",
       "  'NP',\n",
       "  '510L',\n",
       "  '1180L',\n",
       "  '230L',\n",
       "  'AD630L',\n",
       "  '200L',\n",
       "  '280L']]"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lexiles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lexile Level Interpretation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>50th</th>\n",
       "      <th>90th</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>K</td>\n",
       "      <td>BR160L</td>\n",
       "      <td>150L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>165L</td>\n",
       "      <td>570L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>425L</td>\n",
       "      <td>795L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>645L</td>\n",
       "      <td>985L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>850L</td>\n",
       "      <td>1160L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>950L</td>\n",
       "      <td>1260L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>1030L</td>\n",
       "      <td>1340L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>1095L</td>\n",
       "      <td>1410L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>1155L</td>\n",
       "      <td>1470L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>1205L</td>\n",
       "      <td>1520L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>1250L</td>\n",
       "      <td>1570L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>1295L</td>\n",
       "      <td>1610L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>1295L</td>\n",
       "      <td>1610L</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0    50th   90th\n",
       "0           K  BR160L   150L\n",
       "1           1    165L   570L\n",
       "2           2    425L   795L\n",
       "3           3    645L   985L\n",
       "4           4    850L  1160L\n",
       "5           5    950L  1260L\n",
       "6           6   1030L  1340L\n",
       "7           7   1095L  1410L\n",
       "8           8   1155L  1470L\n",
       "9           9   1205L  1520L\n",
       "10         10   1250L  1570L\n",
       "11         11   1295L  1610L\n",
       "12         12   1295L  1610L"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grade_df = pd.read_csv('Lexile levels by grade.csv')\n",
    "grade_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "learn-env",
   "language": "python",
   "name": "learn-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}