2_1_2018_Foong_Education Data Formatter_Ground Truth Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "~010bd468bfc0423fd2\n",
      "~01d435bf3e526173d6\n",
      "~01caece2f244e9af27\n",
      "~01ee2a8c8890151c1f\n",
      "~01b7ba3e7659f8fa15\n",
      "~01c4528d8604af2915\n",
      "~019ace575609b0cfa8\n",
      "~0185c4514e241b889c\n",
      "~018b09bfb8b1ff46d8\n",
      "~01a5bc6d2f8a824f67\n",
      "~0161cb6b6f7775debd\n",
      "~0148c79ceca7c9e77e\n",
      "~013352bffe05d187cf\n",
      "~011e3d50e821b39b77\n",
      "~01e82981266a111ce6\n",
      "~012c51352c78c1b441\n",
      "~01f464d89fa067aec7\n",
      "~01c98c8488669e961b\n",
      "~01b98afa74c41a2292\n",
      "~01fd28834f98d25381\n",
      "~010821a5e7df72b57d\n",
      "~01ffc8bf05fb309b5c\n",
      "~012d44053709e3814f\n",
      "~01ca5d03ca35d74bc5\n",
      "~01f1195835f0077a7d\n",
      "~01f2f34d2a4cf94b78\n",
      "~01dfec01ff12e010c9\n",
      "~0136ebaceb01f3991c\n",
      "~01188050f758172b14\n",
      "~01e2f01b219e0008f4\n",
      "~01cad6188e12352724\n",
      "~01ec5d19ce5c18bbe1\n",
      "~014226e45a3d6482e1\n",
      "~010a31dcb97f9e9df4\n",
      "~01953cec30a7626914\n",
      "~017616cc029be0ac9f\n",
      "~011208a8367d29aa27\n",
      "~0165a72fc6f65b0f0f\n",
      "~010e12df92ef33e697\n",
      "~01340a2f860dc75943\n",
      "~019962ee3660e635f4\n",
      "~01bf6d7e00a8a874b0\n",
      "~01d7f9e241bb79ff02\n",
      "~01f9dd565ab2480e52\n",
      "~018349a31a94bc4246\n",
      "~01fc1b7bfc3ea49959\n",
      "~01350245e6bbbcf1a6\n",
      "~01ebb44d6ddfd1dc36\n",
      "~011fe37e39d22ccfd6\n",
      "~017f2a7ea2084e2be7\n",
      "~016803aad01b303672\n",
      "~012d67c065908ba52a\n",
      "~010752ad4e97d1a328\n",
      "~01de3a7365f1033c7f\n",
      "~01fae97ba8468d397d\n",
      "~01bce2790a51cfc82d\n",
      "~01d72ce144865b6215\n",
      "~016965d3e871f30798\n",
      "~01438236dcd4b44de7\n",
      "~014148543b8d5bce5b\n",
      "~017fad11fd0542bafc\n",
      "~0177631ed8005c9163\n",
      "~0188f06e20d55d1c1e\n",
      "~01e4e7585431fd492e\n",
      "~018e2f701c384f17e6\n",
      "~012462af701de746f9\n",
      "~01fb40e9532f34aed9\n",
      "~017449232369c42a11\n",
      "~01d09a7242d6c15398\n",
      "~01363771393f1a75b1\n",
      "~01adfa6c4baaed436c\n",
      "~01962a4ee413261911\n",
      "~012eb435fcc5a933a5\n",
      "~010f5c5b085044c0c7\n",
      "~01b2362193bd211ec3\n",
      "~01e97c265b49239ac9\n",
      "~01dd2b87a2799e88ad\n",
      "~01feae58f2a64515b3\n",
      "~01055448afab900c11\n",
      "~018e89d7b458becf24\n",
      "~01a8a46d67bf736a03\n",
      "~01fe4a40e62e557941\n",
      "~0145af714e11ab69a9\n",
      "~019a4ec2b687d6f0be\n",
      "~01b7528a54d29a38d8\n",
      "~01ff5989288db5a77b\n",
      "~01e0680211176ad6d8\n",
      "~011468464a83ef3bf1\n",
      "~016a565b4f80e5e5e8\n",
      "~010e253c856cb66d1d\n",
      "~01ddf5c016bb69a3f8\n",
      "~01a645ed618a97378e\n",
      "~01732697c75d3b90c4\n",
      "~01d6b565ddcd7674d1\n",
      "~014df96f1197083b81\n",
      "~01c1cfb06872a47691\n",
      "~01413c6a8db16c7b5d\n",
      "~01928693b1205f9dab\n",
      "~01486074b095e2951d\n",
      "~013e9fb75b55fa8900\n",
      "~01a2b0ea43c965d50a\n",
      "~01e1a0952fc52e110d\n",
      "~01501faab3f9d64d7a\n",
      "~0134d001f1a25b0f23\n",
      "~01991eaab5524e874d\n",
      "~016db3570412c03f90\n",
      "~01f1a169adc080bb55\n",
      "~0147dfdf2845d1fed6\n",
      "~019a6aee43715eb246\n",
      "~01dbab6ecb765f47f1\n",
      "~0134fdac9f99d671c7\n",
      "~01d3effcee4b0723b9\n",
      "~0169b6b6650571cceb\n",
      "~01c151eb125c4d104a\n",
      "~014d8a78cfe7a199e9\n",
      "~01d9600ecd3f983dd6\n",
      "~012c3349217eb4da27\n",
      "~01e837f06e6bdb032d\n",
      "~0150b434ff513baf8f\n",
      "~0109c99edbc420b429\n",
      "~0158c121002263060f\n",
      "~018cf9155e1b6fc121\n",
      "~01c78ccd2b2e6d9a33\n",
      "~0181b8dbafdb641cc3\n",
      "~01e7f3233a5bdeac06\n",
      "~014e213ace6117136b\n",
      "~01b921eb7f33cc15c2\n",
      "~01439cbc6e65cbd3f3\n",
      "~010dc173a514395be8\n",
      "~015d923cb84f27d663\n",
      "~014d64d882124fe0f2\n",
      "~0120c297ebff856da5\n",
      "~010252809a934573bb\n",
      "~01e9a752f0189974ce\n",
      "~01c6548963f1ea9b44\n",
      "~016082fb8c29cc69f3\n",
      "~01ed5015ef4c0c799b\n",
      "~01b5eae23a917f4d44\n",
      "~0110981efa10020b53\n",
      "~0175ce5cd26994f69c\n",
      "~0160eaea83dbf3887a\n",
      "~016e1aae7d2db5d59d\n",
      "~012b56dd5e1fc58843\n",
      "~013695da5c23bdd80c\n",
      "~01c607c5ac2ac29ab9\n",
      "~01b93ea2307b50d547\n",
      "~0109199e2b814654de\n",
      "~01bdfcd6bd24cfb063\n",
      "~01b3413370afd43c38\n",
      "~015ec1f306c84b490e\n",
      "~01591831fbd123ae41\n",
      "~016076b4d51b6c28c0\n",
      "~019c9fbef7bad6c633\n",
      "~0126adbdc4ea155529\n",
      "~014a6f1c006587dbf1\n",
      "~011e9249be2e17c1ce\n",
      "~01d134a33a8e0fbbd6\n",
      "~01f488d3fcdafc170c\n",
      "~01f56e187e5db64b6f\n",
      "~013177343774d50b7a\n",
      "~01c41eb1193b9e03f9\n",
      "~01bec18be5b0a4b60e\n",
      "~018c34c89a8ed63ab3\n",
      "~01a024c8fe4486aab5\n",
      "~0173f8096113577239\n",
      "~01562be50490d19856\n",
      "~017611c48a9306687b\n",
      "~0143f0d22b36f16867\n",
      "~01a9be8fe24e5336f3\n",
      "~01c0d9ef6b640192c7\n",
      "~01ff590eab3f799548\n",
      "~01dd6b1b8fbf75cfc1\n",
      "~013e95c22482d47394\n",
      "~018a6287cfec8042fd\n",
      "~0172915a6abb2f829d\n",
      "~01ad4998d4773b7ecd\n",
      "~01961cc608818152cf\n",
      "~01a6a7e3583d3a2728\n",
      "~01bf3b3e5ee066d63a\n",
      "~01c93e2436362d3c97\n",
      "~016429f93969114a0a\n",
      "~011f64fca893a8c719\n",
      "~01d9c05977ad2a4221\n",
      "~01b5a15e0bfe0a892b\n",
      "~012c3403d923ecc210\n",
      "~011824c9ab47e3d6b8\n",
      "~01d0eec40807588c2f\n",
      "~01a7e4107171e01f86\n",
      "~01b17bc7d8e1340594\n",
      "~016bc23d72e63c5b79\n",
      "~01a652d163fcf93318\n",
      "~01143abe8c1dbb68d3\n",
      "~01d07f65f47de9da36\n",
      "~01906ad3a5729ac861\n",
      "~01e1b5b1f4ddef3f07\n",
      "~014284259e24cb7e71\n",
      "~0192c875e825f9c0e4\n",
      "~01596c4d14eea97ea3\n",
      "~0145829c27a023f388\n",
      "~0122fa0a4ad4827135\n",
      "~01442286de86c0e207\n",
      "This is the user count: 1\n",
      "Found a single ed history\n",
      "Degree: Master of Computer Applications (M.C.A.)\n",
      "School: San Jose City College\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 1\n",
      "This is the user count: 2\n",
      "Found a single ed history\n",
      "Degree: \n",
      "School: Palistine Wheatly High School\n",
      "Comment: \n",
      "----This is the highest education list ['Less Than High School']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 2\n",
      "This is the user count: 3\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: Berklee College of Music\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 3\n",
      "This is the user count: 4\n",
      "Degree: Master of Business Administration (M.B.A.)\n",
      "School: Benedictine University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: University of Chicago\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 4\n",
      "This is the user count: 5\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Science (B.S.)\n",
      "School: University of Florida\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 5\n",
      "This is the user count: 6\n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: Grand Valley State University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: \n",
      "School: Sanilac Clair County Community College\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "Degree: \n",
      "School: Sanilac County Career Center\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Bachelor or Higher', 'Some College or Associate', 'Other']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 6\n",
      "This is the user count: 7\n",
      "Degree: Masters of Education\n",
      "School: Univeristy of Nevada Las Vegas\n",
      "Comment: Emphasis: English\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelors of Arts English Literature\n",
      "School: California State University Fullerton\n",
      "Comment: Major: English literature with emphasis in Romantic literature and creative writing (under full-ride scholarship)\n",
      "Minor: cultural anthropology\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 7\n",
      "This is the user count: 8\n",
      "Degree: \n",
      "School: Carlin Academy of Fine Art\n",
      "Comment: A private art education taught by artist Neilson Carlin, who traces his instructional lineage directly to the academic painting tradition of 19th century France. Curriculum includes classical drawing and oil painting in the tradition of the old masters. www.carlinacademy.com\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Degree: \n",
      "School: Delaware Technical and Community College\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Other', 'Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 8\n",
      "This is the user count: 9\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Science (B.S.)\n",
      "School: San Diego State University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 9\n",
      "This is the user count: 10\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Medicine, Bachelor of Surgery (M.B.B.S.)\n",
      "School: Rajiv Gandhi University of Health Sciences\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 10\n",
      "This is the user count: 11\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Business Administration (B.B.A.)\n",
      "School: Western Michigan University - Haworth College of Business\n",
      "Comment: \n",
      "Ran into some error at user 11\n",
      "This is the user count: 12\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Science (B.S.)\n",
      "School: The Art Institute of Pittsburgh\n",
      "Comment: Coursework in typography, layout design, motion graphics, color theory, graphic design.\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 12\n",
      "This is the user count: 13\n",
      "Found a single ed history\n",
      "Degree: \n",
      "School: Musicians Institute\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 13\n",
      "This is the user count: 14\n",
      "Degree: Master of Arts (M.A.)\n",
      "School: Academy of Art University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Business Administration (B.B.A.)\n",
      "School: Texas A&M Univerisity\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 14\n",
      "This is the user count: 15\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: California State University, Fullerton\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 15\n",
      "This is the user count: 16\n",
      "Degree: Associate's degree\n",
      "School: International Correspondence School\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: High school degree\n",
      "School: Victoria Park High School\n",
      "Comment: \n",
      "----This is the highest education list ['Some College or Associate', 'High School']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 16\n",
      "This is the user count: 17\n",
      "Degree: Master of Arts (M.A.)\n",
      "School: John F. Kennedy University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: University of Georgia\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 17\n",
      "This is the user count: 18\n",
      "Found a single ed history\n",
      "We ran into an error at return edu experience\n",
      "'ascii' codec can't encode character u'\\u2019' in position 64: ordinal not in range(128)\n",
      "----This is the highest education list []----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 18\n",
      "This is the user count: 19\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: Muhlenberg College\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 19\n",
      "This is the user count: 20\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Business Administration (B.B.A.)\n",
      "School: Home schooled\n",
      "Comment: I have been home schooled my whole life and I am currently my own teacher along with tutor and assistance. I would consider myself very well educated. Very open minded, I like to look beyond the limits.\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 20\n",
      "This is the user count: 21\n",
      "Found a single ed history\n",
      "Degree: \n",
      "School: The University of Texas at Austin\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 21\n",
      "This is the user count: 22\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Business Administration (B.B.A.)\n",
      "School: Illinois State University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 22\n",
      "This is the user count: 23\n",
      "Degree: Doctor of Philosophy (Ph.D.)\n",
      "School: Walden University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Master of Public Administration (M.P.A.)\n",
      "School: Walden University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: North Carolina Central University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 23\n",
      "This is the user count: 24\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: University of Alabama-Huntsville\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 24\n",
      "This is the user count: 25\n",
      "Degree: Master of Arts (M.A.)\n",
      "School: Arizona State University\n",
      "Comment: \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: Ithaca College\n",
      "Comment: Double minor in Writing and Politics\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 25\n",
      "This is the user count: 26\n",
      "Found a single ed history\n",
      "Degree: Associate's degree\n",
      "School: Tulsa Community College\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 26\n",
      "This is the user count: 27\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Business Administration (B.B.A.)\n",
      "School: University of Oklahoma\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 27\n",
      "This is the user count: 28\n",
      "Degree: \n",
      "School: Oakland City University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "Degree: \n",
      "School: Miami Dade College\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Some College or Associate', 'Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 28\n",
      "This is the user count: 29\n",
      "Degree: Associate's degree\n",
      "School: Oakland Community College\n",
      "Comment: Graduated Magna Cum Laude, member Phi Theta Kappa Honors Society\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Associate's degree\n",
      "School: Oakland Community College\n",
      "Comment: Graduated Magna Cum Laude, Member Phi Theta Kappa Honors Society\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: \n",
      "School: The Actor's Loft, Royal Oak, MI\n",
      "Comment: Currently studying and working with Emmy award-nominated/winning instructors.\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Some College or Associate', 'Some College or Associate', 'Other']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 29\n",
      "This is the user count: 30\n",
      "Degree: Master of Science (M.S.)\n",
      "School: Midwestern State University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Science (B.S.)\n",
      "School: University of Notre Dame\n",
      "Comment: Magna Cum Laude\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 30\n",
      "This is the user count: 31\n",
      "Degree: Master of Business Administration (M.B.A.)\n",
      "School: Strayer University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Business Administration (B.B.A.)\n",
      "School: Kennesaw State University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 31\n",
      "This is the user count: 32\n",
      "Found a single ed history\n",
      "Degree: High school degree\n",
      "School: Midway High School\n",
      "Comment: \n",
      "----This is the highest education list ['High School']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 32\n",
      "This is the user count: 33\n",
      "Found a single ed history\n",
      "Degree: \n",
      "School: Jamestown Community College\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 33\n",
      "This is the user count: 34\n",
      "Degree: Bachelor of Applied Science (B.A.Sc.)\n",
      "School: The University of Findlay\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Associate's degree\n",
      "School: Owens Community College\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Some College or Associate']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 34\n",
      "This is the user count: 35\n",
      "Found a single ed history\n",
      "We ran into an error at return edu experience\n",
      "'ascii' codec can't encode character u'\\u2019' in position 28: ordinal not in range(128)\n",
      "----This is the highest education list []----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 35\n",
      "This is the user count: 36\n",
      "Degree: Associate of Arts and Sciences (A.A.S.)\n",
      "School: Brown Mackie College\n",
      "Comment: Occupational Therapy Assistant\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: \n",
      "School: Aurora University\n",
      "Comment: Received a certificate in Business\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "Degree: \n",
      "School: United States Air Force Academy\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Some College or Associate', 'Some College or Associate', 'Other']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 36\n",
      "This is the user count: 37\n",
      "Found a single ed history\n",
      "Degree: High school degree\n",
      "School: Loudoun Valley High School\n",
      "Comment: \n",
      "----This is the highest education list ['High School']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 37\n",
      "This is the user count: 38\n",
      "Degree: Bachelor of Business Administration (B.B.A.)\n",
      "School: Indiana Wesleyan University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: \n",
      "School: University of Virginia - Darden Graduate School of Business Administration\n",
      "Comment: Executive Program\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Bachelor or Higher', 'Some College or Associate']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 38\n",
      "This is the user count: 39\n",
      "Found a single ed history\n",
      "Degree: Associate of Arts (A.A.)\n",
      "School: Harold Washington College\n",
      "Comment: I am completing courses at Harold Washington College to obtain my A.A. in Art with a Business Administration minor. I will then be transferring to Clark Atlanta University to receive my B.A. in Art with a Business Administration minor.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 39\n",
      "This is the user count: 40\n",
      "Degree: Master of Arts (M.A.)\n",
      "School: Texas A&M University\n",
      "Comment: Critical Theory Analysis, Rhetorical Analysis, Quantitative Analysis, Political Rhetoric Analysis, Team Analysis, Organizational Communication, \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: McMurry University\n",
      "Comment: Public Speaking, Oral Interpretation, Gender Communication, Mass Communication, Rhetoric, Persuasion, Group Communication, Interpersonal Communication.\n",
      "Minor in Psychology\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 40\n",
      "This is the user count: 41\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Fine Arts (B.F.A.)\n",
      "School: Carnegie Mellon University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 41\n",
      "This is the user count: 42\n",
      "Degree: \n",
      "School: Boston University\n",
      "Comment: Graduate certificate in Digital Product Management which include courses in Product Management with Lean, Agile and System Design Thinking, Platform Strategy for Business, Social Media Marketing, Designing for Experimentation to Enhance Digital Business Innovation, and Business Analytics for Data-Driven Decision Making.\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "Degree: Master of Science (M.S.)\n",
      "School: Missouri State University\n",
      "Comment: The objective of the Department of Defense and Strategic Studies program is to provide professional, graduate-level education in national security policy; foreign policy; arms control; proliferation; international security affairs; defense policy analysis, planning and programs; counter terrorism; regional security issues; weapons of mass destruction; and intelligence analysis.\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: James Madison University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Some College or Associate', 'Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 42\n",
      "This is the user count: 43\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: California Baptist University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 43\n",
      "This is the user count: 44\n",
      "Found a single ed history\n",
      "Degree: Associate's degree\n",
      "School: State University of New York at Stony Brook\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 44\n",
      "This is the user count: 45\n",
      "Found a single ed history\n",
      "Degree: Master of Business Administration (M.B.A.)\n",
      "School: University of Phoenix\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 45\n",
      "This is the user count: 46\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Science (B.S.)\n",
      "School: University of California, Santa Barbara\n",
      "Comment: In my time at UCSB, I focused on developing a versatile skill set that would maximize my value in the technology-driven modern day market. By understanding and applying design principles for visual, as well as sonic (audible) content, I am able to apply creative design skills in a business oriented manner in a software-oriented environment. With my discipline in understanding human-computer relationships, I am able to create the optimal user experience for products of all sorts.\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 46\n",
      "This is the user count: 47\n",
      "Found a single ed history\n",
      "Degree: \n",
      "School: University of Pittsburgh\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 47\n",
      "This is the user count: 48\n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: Metropolitan State University of Denver\n",
      "Comment: I designed my own curriculum, completing coursework in reporting, editing, layout and design, magazine writing, magazine design, photography, public relations, marketing and advertising for a well-rounded experience.\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Associate of Science (A.S.)\n",
      "School: Orange Coast College\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Some College or Associate']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 48\n",
      "This is the user count: 49\n",
      "Found a single ed history\n",
      "Degree: Other\n",
      "School: Digital Media and Digital Arts\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Ambiguous\n",
      "----This is the highest education list ['Other']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 49\n",
      "This is the user count: 50\n",
      "Found a single ed history\n",
      "Degree: \n",
      "School: Metropolitan Community College\n",
      "Comment: I started in Architecture and switched to Video Production.\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 50\n",
      "This is the user count: 51\n",
      "Found a single ed history\n",
      "Degree: Master of Philosophy (M.Phil.)\n",
      "School: West Texas A&M University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 51\n",
      "This is the user count: 52\n",
      "Degree: Associate of Arts (A.A.)\n",
      "School: Fashion Institute of Design & Merchandising\n",
      "Comment: Studied graphic design, apparel design, and product development.\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor's degree\n",
      "School: University of California, Irvine\n",
      "Comment: Received 2 Bachelor degree's.  One in Cultural Anthropology, the study of people and culture with an emphasis on how culture has an impact on consumer decisions.  The other in Cognitive Science, the study of human thought and behavior in relation to psychology and how the human brain works, also, with an emphasis on how these concepts relate to businesses and consumers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Some College or Associate', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 52\n",
      "This is the user count: 53\n",
      "Found a single ed history\n",
      "Degree: \n",
      "School: Ball State University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Person has no degree; checking if they are in a college in the US\n",
      "----This is the highest education list ['Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 53\n",
      "This is the user count: 54\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Engineering (B.Eng.)\n",
      "School: Rungta Engineering College\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 54\n",
      "This is the user count: 55\n",
      "Degree: Bachelor of Science (B.S.)\n",
      "School: Virginia Commonwealth University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: Virginia Commonwealth University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: Virginia Commonwealth University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 55\n",
      "This is the user count: 56\n",
      "Found a single ed history\n",
      "Degree: Associate of Arts and Sciences (A.A.S.)\n",
      "School: Central Georgia Technical College\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Some College or Associate']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 56\n",
      "This is the user count: 57\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Engineering (B.Eng.)\n",
      "School: Ohio State University\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 57\n",
      "This is the user count: 58\n",
      "Found a single ed history\n",
      "Degree: Bachelors\n",
      "School: NYU\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 58\n",
      "This is the user count: 59\n",
      "Found a single ed history\n",
      "Degree: \n",
      "School: Central High School\n",
      "Comment: \n",
      "----This is the highest education list ['Less Than High School']----\n",
      "------This is highest education: None, <type 'str'>-----\n",
      "------This is education id: 5, <type 'str'>------\n",
      "Finished writing data for 59\n",
      "This is the user count: 60\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: Purchase College, State University of New York\n",
      "Comment: The BA program in arts management at Purchase College is designed for those with a passion for the arts with an interest in developing, managing, or producing artists and performers.\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 60\n",
      "This is the user count: 61\n",
      "Degree: Master of Divinity (M.Div.)\n",
      "School: Concordia Seminary, St. Louis\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Bachelor of Arts (B.A.)\n",
      "School: Concordia University, Milwaukee\n",
      "Comment: \n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Bachelor or Higher']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 61\n",
      "This is the user count: 62\n",
      "Degree: Bachelor of Business Administration (B.B.A.)\n",
      "School: Governors State University\n",
      "Comment: Currently a junior pursuing a bachelor's in business admin with a concentration in finance.\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "Degree: Associate's degree\n",
      "School: General Assembly\n",
      "Comment: 10 week online certificate program\n",
      "We did not find the high school; We should be checking the specific degree now!! \n",
      "----This is the highest education list ['Bachelor or Higher', 'Some College or Associate']----\n",
      "------This is highest education: Bachelor or Higher, <type 'str'>-----\n",
      "------This is education id: 0, <type 'str'>------\n",
      "Finished writing data for 62\n",
      "This is the user count: 63\n",
      "Found a single ed history\n",
      "Degree: Bachelor of Applied Science (B.A.Sc.)\n",
      "School: Purchase College, State University of New York\n",
      "Comment: \n"
     ]
    }
   ],
   "source": [
    "import httplib2\n",
    "import oauth2\n",
    "import urllib3\n",
    "import types\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from gender_detector import GenderDetector \n",
    "import psycopg2, psycopg2.extras\n",
    "from causalinference import CausalModel\n",
    "from causalinference.utils import random_data\n",
    "import httplib\n",
    "import base64\n",
    "import json # For Microsoft Face API\n",
    "import urllib as urllib # For Microsoft Face API\n",
    "import time \n",
    "import csv\n",
    "import datetime \n",
    "\n",
    "\n",
    "class UpworkDataFormatter:\n",
    "    \n",
    "    def __init__(self):\n",
    "        # Settings\n",
    "        self.edu_data_file_name = './csv_files/edu_groundtruth_2017_12_12_upwork_analysis_unitedstates_allskills.csv' # Filename for all data\n",
    "        self.edu_data_log_file_name = './log_files/edu_groundtruth_log_upwork_data_analysis_2017_12_12_unitedstates_allskills.txt'\n",
    "        self.ground_truth_file_name = './csv_files/interrater_2017_12_12_upwork_analysis_unitedstates_allskills.csv'\n",
    "        \n",
    "        # Write a log\n",
    "        self.log = open(self.edu_data_log_file_name, 'a')\n",
    "        self.log.write(\"We have started analyzing data!\" + \"\\n\")\n",
    "        self.log.flush()\n",
    "\n",
    "        # Connect to the database \n",
    "        self.conn = psycopg2.connect(\"dbname=eureka01\")\n",
    "        self.cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)\n",
    "        psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json)\n",
    "        \n",
    "        # Initialize values \n",
    "        self.user_count = 1\n",
    "        self.education_levels = [\"Bachelor or Higher\", \"Some College or Associate\",\n",
    "                                 \"High School\", \"Less Than High School\", \"Other\", \"None\"]\n",
    "        self.worker_id_list = self.get_ids_from_dataset(self.ground_truth_file_name)\n",
    "    \n",
    "    def show_education(self):\n",
    "\n",
    "        with open(self.edu_data_file_name, 'w') as csvfile:\n",
    "            fieldnames = ['user_count','education','education_id']\n",
    "            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
    "            writer.writeheader()\n",
    "            \n",
    "            for worker in self.worker_id_list:\n",
    "                self.cur.execute(\"\"\"SELECT detailed_info FROM upwork_unitedstates_allskills_2017_12_12 WHERE user_id = %s;\"\"\", (worker,))\n",
    "                \n",
    "                for user in self.cur:\n",
    "                    try: \n",
    "                        print \"This is the user count: {0}\".format(self.user_count)\n",
    "                        user_count = self.user_count\n",
    "                        education_list = self.return_edu_experience(user)\n",
    "                        highest_education = self.find_highest_education(education_list)\n",
    "                        education_id = str(self.education_levels.index(highest_education))\n",
    "\n",
    "                        print \"----This is the highest education list {0}----\".format(education_list)\n",
    "                        print \"------This is highest education: {0}, {1}-----\".format(highest_education, type(highest_education))\n",
    "                        print \"------This is education id: {0}, {1}------\".format(education_id, type(education_id))\n",
    "\n",
    "                        writer.writerow({'user_count': self.user_count, 'education': highest_education, \n",
    "                                     'education_id': education_id})\n",
    "                        print \"Finished writing data for {0}\".format(self.user_count)\n",
    "\n",
    "                    except:\n",
    "                        print \"Ran into some error at user {0}\".format(self.user_count)\n",
    "\n",
    "                        writer.writerow({'user_count': self.user_count, 'education': \"error\", \n",
    "                                     'education_id': \"error\"})\n",
    "\n",
    "                self.user_count += 1\n",
    "                \n",
    "                    \n",
    "    def find_highest_education(self, edu_list):\n",
    "        for level in self.education_levels:\n",
    "            if level in edu_list:\n",
    "                return level\n",
    "            else:\n",
    "                return \"None\"\n",
    "                \n",
    "    def return_edu_experience(self, user):\n",
    "        all_education = []\n",
    "        try: \n",
    "            ed_history = user[0][\"education\"][\"institution\"]\n",
    "            \n",
    "            if (ed_history == \"\"):\n",
    "                print \"No ed history\"\n",
    "                return all_education\n",
    "            \n",
    "            if (type(ed_history) == dict):\n",
    "                print \"Found a single ed history\"\n",
    "                degree = str(ed_history[\"ed_degree\"])\n",
    "                school = str(ed_history[\"ed_school\"])\n",
    "                comment = str(ed_history[\"ed_comment\"])\n",
    "                \n",
    "                print \"Degree: {0}\".format(degree)\n",
    "                print \"School: {0}\".format(school)\n",
    "                print \"Comment: {0}\".format(comment)\n",
    "                \n",
    "                education = self.check_education(degree, school, comment)\n",
    "                \n",
    "                has_bachelor_or_higher = re.search(\"Professional|Doctorate|Master|Bachelor\", education)\n",
    "                has_some_college = re.search(\"Associate|Certificate|Some College\", education)\n",
    "                has_high_school = re.match(\"High School\", education)\n",
    "                has_less_than_high_school = re.match(\"Less Than High School\", education)\n",
    "                \n",
    "                if (has_high_school): # Check for high school first \n",
    "                    all_education.append(\"High School\")\n",
    "                elif (has_less_than_high_school):\n",
    "                    all_education.append(\"Less Than High School\")\n",
    "                elif (has_bachelor_or_higher):\n",
    "                    all_education.append(\"Bachelor or Higher\")\n",
    "                elif (has_some_college):\n",
    "                    all_education.append(\"Some College or Associate\")\n",
    "                else:\n",
    "                    all_education.append(\"Other\")\n",
    "                \n",
    "                return all_education\n",
    "\n",
    "            elif (type(ed_history) == list):\n",
    "                for item in ed_history:\n",
    "                    degree = str(item[\"ed_degree\"])\n",
    "                    school = str(item[\"ed_school\"])\n",
    "                    comment = str(item[\"ed_comment\"])\n",
    "                    \n",
    "                    print \"Degree: {0}\".format(degree)\n",
    "                    print \"School: {0}\".format(school)\n",
    "                    print \"Comment: {0}\".format(comment)\n",
    "                \n",
    "                    education = self.check_education(degree, school, comment)\n",
    "                    has_bachelor_or_higher = re.search(\"Professional|Doctorate|Master|Bachelor\", education)\n",
    "                    has_some_college = re.search(\"Associate|Certificate|Some College\", education)\n",
    "                    has_less_than_high_school = re.search(\"Less Than High School\", education)\n",
    "                    has_high_school = re.search(\"High School\", education)\n",
    "                    \n",
    "                    if (has_high_school): # Check for high school first \n",
    "                        all_education.append(\"High School\")\n",
    "                    elif (has_less_than_high_school):\n",
    "                        all_education.append(\"Less Than High School\")\n",
    "                    elif (has_bachelor_or_higher):\n",
    "                        all_education.append(\"Bachelor or Higher\")\n",
    "                    elif (has_some_college):\n",
    "                        all_education.append(\"Some College or Associate\")\n",
    "                    else:\n",
    "                        all_education.append(\"Other\")\n",
    "                        \n",
    "                return all_education\n",
    "        \n",
    "        except Exception as error:\n",
    "            print \"We ran into an error at return edu experience\"\n",
    "            print error\n",
    "            return all_education\n",
    "        \n",
    "    def check_education(self, degree, school, comment): # Do a quick check that the user has a high school degree\n",
    "        try:\n",
    "            hs_degree_mentioned = False\n",
    "            diploma_degree_mentioned = False\n",
    "            hs_word_mentioned = False\n",
    "            graduated_mentioned = False\n",
    "            \n",
    "            if degree != \"\":\n",
    "                hs_degree_mentioned = re.search(\"high school\", degree.lower())\n",
    "                diploma_degree_mentioned = re.search(\"diploma\", degree.lower())\n",
    "            \n",
    "            if school != \"\":\n",
    "                hs_word_mentioned = re.search(\"high school\", school.lower())\n",
    "            \n",
    "            if comment != \"\":\n",
    "                graduated_mentioned = re.search(\"graduated\", comment.lower())\n",
    "\n",
    "            if (hs_degree_mentioned or (hs_word_mentioned and (graduated_mentioned or diploma_degree_mentioned))):\n",
    "                return \"High School\"\n",
    "            elif (hs_word_mentioned and degree == \"\"):\n",
    "                return \"Less Than High School\"\n",
    "            elif (self.check_school_by_name(\"high_school\", school)): # Check if the school name is in the database\n",
    "                print \"We found the high school!\"\n",
    "                if (hs_degree_mentioned or diploma_degree_mentioned or graduated_mentioned):\n",
    "                    return \"High School\"\n",
    "                elif (degree == \"\"):\n",
    "                    return \"Less Than High School\"\n",
    "            else:\n",
    "                print \"We did not find the high school; We should be checking the specific degree now!! \"\n",
    "                return self.check_specific_degree(degree, school, comment)\n",
    "        except Exception as error:\n",
    "            print \"Something went wrong\"\n",
    "            print error\n",
    "            return \"Error\"\n",
    "        \n",
    "    def check_specific_degree(self, degree, school, comment): # Do a quick check that the user has a bachelor or associate degree\n",
    "\n",
    "        try:\n",
    "            \n",
    "            # If there is no degree, then check if they are enrolled in a college in the US\n",
    "            if (degree == \"\"):\n",
    "                print \"Person has no degree; checking if they are in a college in the US\"\n",
    "                if (self.check_school_by_name(\"college\", school)):\n",
    "                    return \"Some College\"\n",
    "                else:\n",
    "                    return \"Ambiguous\"\n",
    "            \n",
    "            # Professional degree\n",
    "            if (re.search('d\\.c|d\\.c\\.m|d\\.d\\.s|d\\.m\\.d|ll\\.b|ll\\.m|l\\.l\\.m|l\\.l\\.b|j\\.d\\.|m\\.d\\.|o\\.d\\.|d\\.o\\.|pharm\\.d', degree.lower()) \n",
    "                or re.search('d\\.p\\.m|d\\.p|pod\\.d|m\\.div|m\\.h\\.l|b\\.d|ordination|d\\.v\\.m| law', degree.lower())\n",
    "                or re.search('DC|DCM|DDS|DMD|LLB|LLM|LLB|JD|MD|OD|DO|PharmD|Ed\\.D|EdD|CFA', degree)\n",
    "                or re.search('DPM|DP|PodD|MDiv|MHL|BD|DVM', degree)):\n",
    "                return \"Professional\"\n",
    "\n",
    "            # PhD\n",
    "            elif (re.search('Ph\\.D|PhD', degree)\n",
    "                 or re.search('doctor|licentiate', degree.lower())): # Doctorate placed here in case MDs are also called doctors \n",
    "                return \"Doctorate\"\n",
    "\n",
    "            # Masters\n",
    "            elif (re.search('M\\.B\\.A|MBA|M\\.A|M\\.S|M\\.Sc|M\\.E|MA|MS|MSc|ME|M\\.A\\.Ed|MAEd|MST|M\\.S\\.T|MEd|M\\.Ed|MPA|M\\.P\\.A', degree)\n",
    "                 or re.search('master|engineer\\'s degree', degree.lower())\n",
    "                 or re.search('MFA|M\\.F\\.A|MSC|MCS', degree)):\n",
    "                return \"Master\"\n",
    "\n",
    "            # Bachelors\n",
    "            elif (re.search('B\\.A|B\\.S|B\\.Sc|BA|BS|BSc|BFA|AB|A\\.B|BPharm|B\\.Pharm', degree)\n",
    "                 or re.search('bachelor|bpharm|bachlor|batchelor', degree.lower())):\n",
    "                return \"Bachelor\"\n",
    "\n",
    "            # Associate's Degree\n",
    "            elif (re.search('AA|AS|A\\.A|A\\.S|A\\.A\\.S|AAS', degree)\n",
    "                 or re.search('associate|foundation', degree.lower())):\n",
    "                return \"Associate\"\n",
    "\n",
    "            # Certificate\n",
    "            elif (re.search('certificate', degree.lower())):\n",
    "                return \"Certificate\"\n",
    "            \n",
    "            else:\n",
    "                print \"Ambiguous\" \n",
    "                return \"Ambiguous\"\n",
    "                \n",
    "            \n",
    "        except Exception as error:\n",
    "            print \"We ran into an error specifying the degree\"\n",
    "            print error\n",
    "            return \"Error\"\n",
    "\n",
    "            \n",
    "    def check_school_by_name(self, dataset, name): # Checks if school name is in a dataset of schools \n",
    "        name = name.lower()\n",
    "        name = self.remove_punctuation(name)\n",
    "        private_schools = 'high_schools_in_us.csv'\n",
    "        public_schools = 'high_schools_in_us.csv'\n",
    "        colleges = 'colleges_in_us.csv'\n",
    "        \n",
    "        if (dataset == \"high_school\"): # Check for \"high school\" and against list of private and public high schools\n",
    "            if re.search('highschool', name):\n",
    "                return True\n",
    "            \n",
    "            with open(private_schools, 'r') as f:\n",
    "                private_school_data = csv.DictReader(f)\n",
    "                for row in private_school_data:\n",
    "                    school = row[\"pinst\"].lower()\n",
    "                    school = self.remove_punctuation(school)\n",
    "                    if (name == school):\n",
    "                        return True\n",
    "                    \n",
    "            with open(public_schools, 'r') as f: # Haven't been able to find a suitable dataset \n",
    "                public_school_data = csv.DictReader(f)\n",
    "                for row in public_school_data:\n",
    "                    school = row[\"pinst\"].lower()\n",
    "                    school = self.remove_punctuation(school)\n",
    "                    if (name == school):\n",
    "                        return True\n",
    "            \n",
    "            return False # Return false if we can't find the school name in any of the lists\n",
    "                \n",
    "        elif (dataset == \"college\"): # Check for keywords and against list of colleges\n",
    "            if (re.search('college|university|institute', name)):\n",
    "                return True \n",
    "            with open(colleges, 'r') as f:\n",
    "                college_data = csv.DictReader(f)\n",
    "                for row in college_data:\n",
    "                    school = row[\"INSTNM\"].lower()\n",
    "                    school = self.remove_punctuation(school)\n",
    "                    if (name == school):\n",
    "                        print \"Found a matching college!\"\n",
    "                        print name\n",
    "                        return True\n",
    "            \n",
    "            return False # Return false if we can't find the school name in any of the lists\n",
    "        \n",
    "        else:\n",
    "            return False # Return false if all else fails \n",
    "    \n",
    "    def remove_punctuation(self, word): # Removes all punctuation and blank spaces from a string\n",
    "        modified_word = re.sub(r'[^\\w\\s]','',word)\n",
    "        modified_word = re.sub(' ', '', modified_word)\n",
    "        return modified_word\n",
    "                    \n",
    "    def show_degree(self, user):\n",
    "        try: \n",
    "            ed_history = user[0][\"education\"][\"institution\"]\n",
    "            if(ed_history == \"\"):\n",
    "                return \"No edu listed\"\n",
    "            elif(type(ed_history) == dict):\n",
    "                return ed_history[\"ed_degree\"]\n",
    "            elif(type(ed_history) == list):\n",
    "                degree_list = []\n",
    "                for ed_experience in user[0][\"education\"][\"institution\"]:\n",
    "                    if (ed_experience[\"ed_degree\"] == \"\"):\n",
    "                        degree_list.append(\"none\")\n",
    "                    elif (ed_experience[\"ed_degree\"] != \"\"):\n",
    "                        degree_list.append(ed_experience[\"ed_degree\"])\n",
    "                return degree_list\n",
    "            else:\n",
    "                return \"Error: Some other error\"\n",
    "                \n",
    "        except:\n",
    "            return \"Error: No education\"\n",
    "    \n",
    "    def get_ids_from_dataset(self, spreadsheet):\n",
    "        id_list = []\n",
    "        df = pd.read_csv(spreadsheet)\n",
    "        for row in df['worker_id']:\n",
    "            print row\n",
    "            id_list.append(row)\n",
    "        return id_list\n",
    "                    \n",
    "myObject = UpworkDataFormatter()\n",
    "\n",
    "myObject.show_education()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}