TDA_NoiseInjection-MovieReviewsData-prototype8.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Textual Data Amplification (TDA) Experiments - Prototype 7\n",
    "\n",
    "## Paraphrase Generation - Noise Injection\n",
    "\n",
    "## Movie Reviews Data\n",
    "\n",
    "## Sentiment polarity prediction task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Python Basic Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python:  3.6.5 (v3.6.5:f59c0932b4, Mar 28 2018, 03:03:55) \n",
      "[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] True\n",
      "NumPy:  1.16.2 True\n",
      "Pandas:  0.24.2\n",
      "Matplotlib:  3.0.3\n",
      "NLTK:  3.4\n",
      "IPython:  7.4.0\n",
      "skikit-learn 0.20.3\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import nltk\n",
    "import IPython\n",
    "import sklearn\n",
    "\n",
    "# Versions of the packages\n",
    "print('Python: ', sys.version, float(str(sys.version).split('.')[0])+float(str(sys.version).split('.')[1])/10  >= 2.6)\n",
    "print('NumPy: ', np.__version__, float(np.__version__.split('.')[0])+float(np.__version__.split('.')[1])/10 >= 1.6 )\n",
    "print('Pandas: ', pd.__version__)\n",
    "print('Matplotlib: ', matplotlib.__version__)\n",
    "print('NLTK: ', nltk.__version__)\n",
    "print('IPython: ', IPython.__version__)\n",
    "print('skikit-learn', sklearn.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data\n",
    "\n",
    "#### Movie Review Data\n",
    "http://www.cs.cornell.edu/people/pabo/movie-review-data/\n",
    "\n",
    "#### Download link\n",
    "http://www.cs.cornell.edu/people/pabo/movie-review-data/review_polarity.tar.gz\n",
    "\n",
    "#### Data files structure\n",
    "\n",
    "Create a directory for the data (DATA) then negative (pos) and positive reviews (neg) sub directories.\n",
    "\n",
    "You should also create two augmented data directories (noiseinjection_aug_neg and noiseinjection_aug_pos)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Data Augmentation Object\n",
    "### Generic Class Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import re\n",
    "from nltk import wordpunct_tokenize\n",
    "from nltk import sent_tokenize\n",
    "from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction\n",
    "from os import listdir\n",
    "import random\n",
    "import timeit\n",
    "import numpy as np\n",
    "from scipy.stats import norm\n",
    "\n",
    "class TextDataAugmentation(object):\n",
    "    \n",
    "    # Constructor \n",
    "    def __init__(self, data_dir, \n",
    "                 neg_rev, pos_rev, aug_neg_rev, aug_pos_rev,\n",
    "                 paraphr_nbr_max=5, paratext_nbr_max=5\n",
    "                ): \n",
    "        self.DATA_DIR = data_dir\n",
    "        self.NEG_REVIEWS = neg_rev\n",
    "        self.POS_REVIEWS = pos_rev\n",
    "        self.AUGMENTED_NEG_REVIEWS = aug_neg_rev\n",
    "        self.AUGMENTED_POS_REVIEWS = aug_pos_rev\n",
    "        self.name = \"TextDataAugmentation\" \n",
    "        self.encoding = 'utf-8'\n",
    "        self.SENTENCE_LENGTH_MAX = 1000\n",
    "        self.PARAPHRASES_NUMBER_MAX = paraphr_nbr_max\n",
    "        self.PARATEXTS_NUMBER_MAX = paratext_nbr_max\n",
    "        self.PARATEXT_ITERATION_LIMIT = 0.50\n",
    "        self.TRACE_RANDOM_KEY = False\n",
    "        self.TRACE_COMBINATIONS = False\n",
    "\n",
    "    # To check if this object is TextDataAugmentation class \n",
    "    def getName(self): \n",
    "        return self.name\n",
    " \n",
    "    # To check if this object is TextDataAugmentation class \n",
    "    def isTextDataAugmentation(self): \n",
    "        return True\n",
    "\n",
    "    # load doc into memory\n",
    "    def load_doc(self, file_path_with_name):\n",
    "        # open the file as read only\n",
    "        with open(file_path_with_name, 'r', encoding=self.encoding) as input_file:\n",
    "            # read all text\n",
    "            text = input_file.read()\n",
    "        return text\n",
    "\n",
    "    # save doc to file\n",
    "    def save_doc(self,file_name_with_path,text):\n",
    "        # open the file as write only\n",
    "        with open(file_name_with_path, 'w', encoding=self.encoding) as ouput_file:\n",
    "            # write text\n",
    "            ouput_file.write(text)\n",
    "\n",
    "    def tokenizeTextString(self, text_string):\n",
    "        return wordpunct_tokenize(text_string.replace('_',' ').replace('(',' ').replace(')',' ').replace(\"'\",' ').replace('\"','').replace('/',' ').replace(\"\\\\\", \" \").replace('  ',' '))\n",
    "\n",
    "    def standardize_text(self, raw_input_text):\n",
    "        raw_input_text = raw_input_text.lower().replace(\" n't\",\"n't\").replace(\" 's\",\"'s\")\n",
    "        return ' '.join(self.tokenizeTextString(raw_input_text.lower()))\n",
    "    \n",
    "    def sentence_length(self, sentence):\n",
    "        return len(sentence.split(\" \"))\n",
    "\n",
    "    def get_heuristic_weigth(self, original_sentence, new_paraphrase):\n",
    "        return 1\n",
    "\n",
    "    def add_new_paraphrase(self, paraphrases, original_sentence, new_paraphrase):\n",
    "        new_paraphrase_dict = dict()\n",
    "        if  new_paraphrase not in paraphrases.keys():\n",
    "            new_paraphrase_dict[new_paraphrase] = self.get_heuristic_weigth(original_sentence,new_paraphrase)\n",
    "            return new_paraphrase_dict\n",
    "        return new_paraphrase_dict\n",
    "\n",
    "    def splitted(self, document):\n",
    "        return '\\n' in document\n",
    "\n",
    "    def sentences_splitting(self, document):\n",
    "        if self.splitted(document):\n",
    "            # returns a list of sentences already splitted\n",
    "            # and marked by a \\n tag\n",
    "            return document.split('\\n')\n",
    "        else:\n",
    "            return sent_tokenize(document)\n",
    "\n",
    "    def evaluate_combinations_number(self, paratext_dict):\n",
    "        combinations_nbr = 1\n",
    "        for sentence_index in paratext_dict.keys():\n",
    "            combinations_nbr *= len(paratext_dict[sentence_index])\n",
    "        return combinations_nbr\n",
    "\n",
    "    # Normalizing the replacement weights list in order to get probabilities \n",
    "    def normalize_weights_list(self, weights_list):\n",
    "        weights_sum = sum(weights_list)\n",
    "        # Avoid division by 0 \n",
    "        if weights_sum == 0:\n",
    "            weights_sum = 1\n",
    "        return [a_weight/weights_sum for a_weight in weights_list]\n",
    "\n",
    "    def get_an_integer_normaly_draw_from_n_range(n):\n",
    "        if n > 0:\n",
    "            mean = n/2.0\n",
    "            std = n/5.0\n",
    "            draw_number = list(norm.ppf(np.random.random(1), loc=mean, scale=std).astype(int))[0]\n",
    "            while not (draw_number >= 1) and (draw_number <= n):\n",
    "                draw_number = list(norm.ppf(np.random.random(1), loc=mean, scale=std).astype(int))[0]\n",
    "        else:\n",
    "            draw_number = 0\n",
    "        return draw_number \n",
    "\n",
    "    # draw a random number of replacements among all possible replacements \n",
    "    # distribution: 'left_skewed', 'right_skewed', 'uniform'\n",
    "    def get_variants_using_random_distribution(self, paratext_dict,distribution='uniform'):\n",
    "        variants_index = [n for n in range(1,len(paratext_dict)+1) if len(paratext_dict[str(n)])>1]\n",
    "        if len(variants_index) > 0:\n",
    "            variants_quantity = self.get_an_integer_normaly_draw_from_n_range(len(variants_index))\n",
    "            if distribution == 'left_skewed':\n",
    "                distribution_weights = [2*n for n in range(1,len(variants_index)+1)]\n",
    "            elif distribution == 'right_skewed':\n",
    "                distribution_weights = [2/n for n in range(1,len(variants_index)+1)]\n",
    "            else: # uniform\n",
    "                distribution_weights = [1 for n in range(1,len(variants_index)+1)]\n",
    "            normalized_distribution_weights = self.normalize_weights_list(distribution_weights)\n",
    "            if self.TRACE_RANDOM_KEY:\n",
    "                print(\"len(variants_index):\",len(variants_index),\"variants_quantity:\",variants_quantity)\n",
    "            if variants_quantity > len(variants_index):\n",
    "                variants_quantity = len(variants_index)\n",
    "                if self.TRACE_RANDOM_KEY:\n",
    "                    print(\"New variants_quantity:\",variants_quantity)\n",
    "            randomized_variants = np.random.choice(variants_index,\\\n",
    "                                                   size=variants_quantity,\\\n",
    "                                                   replace=False,\\\n",
    "                                                   p=normalized_distribution_weights)\n",
    "            if self.TRACE_RANDOM_KEY:\n",
    "                print(\"*** variants_quantity: \", variants_quantity)\n",
    "                print(\"*** randomized_variants: \",randomized_variants)\n",
    "                print(\"distribution_weights: \",distribution_weights)\n",
    "                print(\"normalized_distribution_weights: \",normalized_distribution_weights)\n",
    "                print(\"variants_quantity:\",variants_quantity,\"len(paratext_dict):\",len(paratext_dict),\"len(variants_index)\",len(variants_index))\n",
    "        else:\n",
    "            randomized_variants = []\n",
    "        return list(randomized_variants)\n",
    "\n",
    "    # Choose randomly one variant for a part among all possible variants based on heuristic weights\n",
    "    def get_one_variant_using_heuristic_weights(self, paraphrase_dict,part_index):\n",
    "        # Choose the replacement among all the possible replacements based on heuristic weights\n",
    "        variants = list(range(0,len(paraphrase_dict[str(part_index)])))\n",
    "        variants_heuristic_weights = list(paraphrase_dict[str(part_index)].values())\n",
    "        if len(variants_heuristic_weights)> 1:\n",
    "            a_variant = random.choices(population=variants,\\\n",
    "                                       weights=variants_heuristic_weights,k=1)[0]\n",
    "        else:\n",
    "            a_variant = variants[0]\n",
    "        return a_variant\n",
    "\n",
    "    def generate_sampling_random_key(self, paratext_dict,distribution='uniform'):\n",
    "        random_part_key = \"\"\n",
    "        # Choose randomly the variants based on a distribution\n",
    "        variants_indexes = self.get_variants_using_random_distribution(paratext_dict,distribution=distribution)\n",
    "        if self.TRACE_RANDOM_KEY:\n",
    "            print(\"variants_indexes: \",variants_indexes)\n",
    "        for part_index in range(1,len(paratext_dict)+1):\n",
    "            if part_index in variants_indexes:\n",
    "                if self.TRACE_RANDOM_KEY:\n",
    "                    print(\"part_index found: \",part_index)\n",
    "                # Choose randomly one variant for a precise part among all possible variants based on heuristic weights\n",
    "                a_variant = self.get_one_variant_using_heuristic_weights(paratext_dict,part_index)\n",
    "                if self.TRACE_RANDOM_KEY:\n",
    "                    print(\"a_variant: \",a_variant)\n",
    "                random_part_key = random_part_key + str(part_index) + \"-\" + str(a_variant) + \"_\"\n",
    "            else:\n",
    "                random_part_key = random_part_key + str(part_index) + \"-0\" + \"_\"\n",
    "        return random_part_key[:-1]\n",
    "\n",
    "    def generate_new_unique_key(self, paratext_dict,keys_memory,distribution=\"uniform\"):\n",
    "        random_key = self.generate_sampling_random_key(paratext_dict,distribution)\n",
    "        while random_key in keys_memory:\n",
    "            random_key = self.generate_sampling_random_key(paratext_dict,distribution)\n",
    "        keys_memory.add(random_key)\n",
    "        return (keys_memory,random_key)\n",
    "\n",
    "    def generate_one_random_paratext(self, paratext_dict, random_key):\n",
    "        paratext = []\n",
    "        for index,key in enumerate(random_key.split(\"_\")):\n",
    "            sub_keys = key.split(\"-\")\n",
    "            paraphrase = list(paratext_dict[str(index+1)].keys())[int(sub_keys[1])]\n",
    "            paratext.append(paraphrase)\n",
    "        return paratext\n",
    "\n",
    "    def generate_paraphrases(new_sentence):\n",
    "        return new_sentence\n",
    "    \n",
    "    def create_paratext_dict(self, sentences_list):\n",
    "        sentences_number = len(sentences_list)\n",
    "        paratext_dict = {}\n",
    "        for sentence_index in range(sentences_number):\n",
    "            new_sentence = ' '.join(re.findall(r\"[a-zA-Z'-]+|[.,;!?\\'\\’]+\",sentences_list[sentence_index].lower()))\n",
    "            paraphrases_dict = {new_sentence:1}\n",
    "            if len(new_sentence) > 0:\n",
    "                new_paraphrases_dict = self.generate_paraphrases(new_sentence)\n",
    "                paraphrases_dict.update(new_paraphrases_dict)\n",
    "            paratext_dict[str(sentence_index+1)] = paraphrases_dict\n",
    "        return paratext_dict\n",
    "\n",
    "    def generate_sampling(self, original_document, augmentation_factor):\n",
    "        splitted_document = self.sentences_splitting(original_document)\n",
    "        paratext_dict = self.create_paratext_dict(splitted_document)\n",
    "        print(\"Paratext Dict created for the document\")\n",
    "        combinations_number_max = self.evaluate_combinations_number(paratext_dict)\n",
    "        if augmentation_factor > combinations_number_max:\n",
    "            print(\"*** WARNING! Desired paratexts_sample number: \" + str(augmentation_factor) + \" exceeds Combinations Number Max, which is: \", combinations_number_max)\n",
    "            augmentation_factor = combinations_number_max\n",
    "        if augmentation_factor > self.PARATEXTS_NUMBER_MAX:\n",
    "            print(\"*** WARNING! Desired paratexts_sample number: \" + str(augmentation_factor) + \" exceeds PARATEXTS_NUMBER_MAX, which is: \", PARATEXTS_NUMBER_MAX)\n",
    "            augmentation_factor = self.PARATEXTS_NUMBER_MAX\n",
    "        paratexts_sample = []\n",
    "        keys_memory = set()\n",
    "        while len(paratexts_sample) < augmentation_factor:\n",
    "            if len(keys_memory) > combinations_number_max * self.PARATEXT_ITERATION_LIMIT:\n",
    "                break\n",
    "            keys_memory, random_key = self.generate_new_unique_key(paratext_dict,keys_memory,distribution='uniform')\n",
    "            new_paratext = self.generate_one_random_paratext(paratext_dict,random_key)\n",
    "            paratexts_sample.append(new_paratext)\n",
    "        print(\"Text Data Augmentation done for the document\")\n",
    "        return paratexts_sample\n",
    "\n",
    "    def get_datafile_id(filename):\n",
    "        return filename.split(\"_\")[-2][-3:]\n",
    "        \n",
    "    def augment_text_data(self, augmentation_factor=5):\n",
    "        is_train = True\n",
    "        # WARNING - many lines of code below are data source specific !\n",
    "        for sub_directory in [self.NEG_REVIEWS,self.POS_REVIEWS]:\n",
    "            data_directory = data_dir + sub_directory\n",
    "            for filename in listdir(data_directory):\n",
    "                start_time = timeit.default_timer()\n",
    "                # skip files that do not have the right extension\n",
    "                # WARNING - data source specific\n",
    "                if not filename.endswith(\".txt\"):\n",
    "                    continue\n",
    "                # Processing training set, so skip any reviews in the test set\n",
    "                # which start with 'cv9' - WARNING - data source specific\n",
    "                if is_train and filename.startswith('cv9'):\n",
    "                    continue\n",
    "                # Processing test set, so skip any reviews in the training set\n",
    "                # which not start with 'cv9' - WARNING - data source specific\n",
    "                if not is_train and not filename.startswith('cv9'):\n",
    "                    continue\n",
    "                # create the full path of the file to open\n",
    "                file_path = data_directory + filename\n",
    "                # load the original document\n",
    "                original_text = self.load_doc(file_path)\n",
    "                print(\"Augmenting...\", file_path)\n",
    "                paratexts = self.generate_sampling(original_text,augmentation_factor=augmentation_factor)\n",
    "                for index,paraphrases in enumerate(paratexts):\n",
    "                    new_text = \" \\n\".join(paraphrases)\n",
    "                    new_filename = \"ampl_\" + filename.split(\".\")[0] + \"_\" + str(index)+ '.txt'\n",
    "                    if sub_directory == self.NEG_REVIEWS:\n",
    "                        new_file_path = self.DATA_DIR+self.AUGMENTED_NEG_REVIEWS+new_filename \n",
    "                        self.save_doc(new_file_path,new_text)\n",
    "                    else:\n",
    "                        new_file_path = self.DATA_DIR+self.AUGMENTED_POS_REVIEWS+new_filename\n",
    "                        self.save_doc(new_file_path,new_text)\n",
    "                    if (index % 10) == 0:\n",
    "                        print(\"Saved:\",new_file_path)\n",
    "                        end_time = timeit.default_timer()\n",
    "                        print(\"Elapsed time: {}\".format(end_time - start_time))\n",
    "                        print(\"-----\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Noise Injection TDA Object\n",
    "### Sub Class Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "import random\n",
    "import string\n",
    "\n",
    "# Inherited or Sub class (TextDataAugmentation) \n",
    "class NoiseInjection(TextDataAugmentation): \n",
    "    # Based on work done in collaboration with Simon ROQUETTE at Polytechnique Montreal\n",
    "    # https://github.com/simonroquette/CORAP\n",
    "\n",
    "    # Constructor \n",
    "    def __init__(self,data_dir, neg_rev, pos_rev,aug_neg_rev, aug_pos_rev, \n",
    "                 paraphr_nbr_max=5, paratext_nbr_max=5,\n",
    "                 noise_percent=0.1,error_table=\"no_errors_table\"): \n",
    "        self.NOISE_PERCENT_PER_LONG_WORD = noise_percent\n",
    "        self.MAX_ITERATIONS = 1000\n",
    "        self.MAX_WORD_LENGTH = 50\n",
    "        self.TRACE = False\n",
    "        #TODO Should space be part of our alphabet ??? probably if want deal spaced out words\n",
    "        self.ALPHABET = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .,:;'*!?`$%&(){}[]-/\\@_#\" \n",
    "        # ERRORS_TABLE and ERRORS_DICT should be build at the beginning, only one time !\n",
    "        # Don't put the same char in two equivalence tables. It will be overwritten.\n",
    "        # Indeed, equivalence is a transitive property, if A looks like B and B looks like C, A looks like C\n",
    "        # And therefore they should be in the same table\n",
    "        # TYPICAL OCR (Optical Character Recognition) ERRORS\n",
    "        self.OCR_ERRORS_TABLE = [['i', 'j', 'l'],\n",
    "                                 ['n', 'r'],\n",
    "                                 ['m', 'nn', 'rn', 'nr'],\n",
    "                                 ['mm', 'nnm', 'mnn', 'nnn', 'nnnn', 'rnm', 'nrm', 'mrn'],\n",
    "                                 ['u', 'v', ],\n",
    "                                 ['w', 'vv'],\n",
    "                                 ['o', 'a'],\n",
    "                                 #['s', 'z'],  # s and z are more for handwriting\n",
    "                                 #['x', 'ae', 'oe', 'oc'],  # Maybe too much\n",
    "                                 ['g', 'q'],\n",
    "                                 #['k', 'le', 'lR'], # Maybe too much\n",
    "                                 ['c', 'e'],\n",
    "                                 ['h', 'b']\n",
    "                                 #['d', 't']  # Maybe too much\n",
    "                                ]\n",
    "        # QWERTY KEYBOARD ERRORS (based on key physical proximity)\n",
    "        self.KEYBOARD_ERRORS_TABLE = [\n",
    "            ['q','1','2','3','w','s','a'],\n",
    "            ['w','2','3','4','e','d','s','a','q'],\n",
    "            ['e','3','4','5','r','f','d','s','w'],\n",
    "            ['e','3','4','5','r','f','d','s','w'],\n",
    "            ['r','4','5','6','t','g','f','d','e'],\n",
    "            ['t','5','6','7','y','h','g','f','r'],\n",
    "            ['y','6','7','8','u','j','h','g','t'],\n",
    "            ['u','7','8','9','i','k','j','h','y'],\n",
    "            ['i','8','9','0','o','l','k','j','u'],\n",
    "            ['o','9','0','-','p',';','l','k','i'],\n",
    "            ['p','0','-','=','[',\"'\",';','l','o'],\n",
    "            ['a','q','w','s','x','z','\\\\'],\n",
    "            ['s','q','w','e','d','c','x','z','a'],\n",
    "            ['d','w','e','r','f','v','c','x','s'],\n",
    "            ['f','e','r','t','g','b','v','c','d'],\n",
    "            ['f','e','r','t','g','b','v','c','d'],\n",
    "            ['g','r','t','y','h','n','b','v','f'],\n",
    "            ['h','t','y','u','j','m','n','b','g'],\n",
    "            ['j','y','u','i','k',',','m','n','h'],\n",
    "            ['k','u','i','o','l','.',',','m','j'],\n",
    "            ['l','i','o','p',';','/','.',',','k'],\n",
    "            ['z','a','s','x',' ','\\\\'],\n",
    "            ['x','a','s','d','c',' ','z'],\n",
    "            ['c','s','d','f','v',' ','x'],\n",
    "            ['v','d','f','g','b',' ','c'],\n",
    "            ['b','f','g','h','n',' ','v'],\n",
    "            ['n','g','h','j','m',' ','b'],\n",
    "            ['m','h','j','k',',',' ','n'],\n",
    "        ]\n",
    "        # Above and below someone could add new specific ERRORS_TABLE if needed\n",
    "        if error_table == \"keyboard\":\n",
    "            self.ERRORS_TABLE = self.KEYBOARD_ERRORS_TABLE\n",
    "        elif error_table == \"ocr\":\n",
    "            self.ERRORS_TABLE = self.OCR_ERRORS_TABLE\n",
    "        elif error_table == \"no_errors_table\":\n",
    "            self.ERRORS_TABLE = []\n",
    "            \n",
    "        # Build dictionnary once, because it is faster when looking for equivalences\n",
    "        ERRORS_DICT = dict()\n",
    "        for line in range(len(self.ERRORS_TABLE)):\n",
    "            for entry in range(len(self.ERRORS_TABLE[line])):\n",
    "                if len(self.ERRORS_TABLE[line][entry]) <= 2:  # We don't care entries that are bigger than 2\n",
    "                    copy = list(self.ERRORS_TABLE[line])\n",
    "                    del copy[entry]\n",
    "                    ERRORS_DICT[self.ERRORS_TABLE[line][entry]] = copy\n",
    "        # MANUALLY ADD SOME ERRORS HERE THAT ARE ONLY ONE WAY: ERRORS_DICT[to_add] = [add1, add2]\n",
    "        self.ERRORS_DICT = ERRORS_DICT\n",
    "        super(NoiseInjection, self).__init__(data_dir, \n",
    "                                             neg_rev, pos_rev,\n",
    "                                             aug_neg_rev, aug_pos_rev,\n",
    "                                             paraphr_nbr_max, paratext_nbr_max\n",
    "                                            )\n",
    "\n",
    "    # To check if this object is Backtranslation class \n",
    "    def isNoiseInjection(self): \n",
    "        return True\n",
    "\n",
    "    def hasnum(self,word):\n",
    "        for c_i in word:\n",
    "            if c_i.isdigit():\n",
    "                return True\n",
    "        return False\n",
    "\n",
    "    def noise_char(self,w):\n",
    "        # only one error per word\n",
    "        # only random operations, no informed errors\n",
    "        # in order to make ERRORS_TABLE more probable\n",
    "        # noise_op = random.choice(['DELETE', 'INSERT', 'REPLACE', 'ERRORS_TABLE', 'ERRORS_TABLE', 'ERRORS_TABLE']) \n",
    "        if self.ERRORS_TABLE == []:\n",
    "            # ONLY DELETE, INSERT or REPLACE, NO ERRORS TABLE\n",
    "            noise_op = random.choice(['DELETE', 'INSERT', 'REPLACE'])\n",
    "        else:\n",
    "            noise_op = random.choice(['DELETE', 'INSERT', 'REPLACE', 'ERRORS_TABLE'])\n",
    "\n",
    "        # Events with no noise here : - numbers\n",
    "        # - ponctuation like \"!\" \"??\" or \"),\"\n",
    "        if self.hasnum(w) or ((not w.isalpha()) and len(w) < 3):\n",
    "            w = w\n",
    "        else:\n",
    "            if noise_op == \"DELETE\" and len(w) > 1: # Words of length 1 don't overgo deletion...\n",
    "                    idx = random.randint(0, len(w) - 1)\n",
    "                    w = w[:idx] + w[idx + 1:]\n",
    "\n",
    "            if noise_op == \"INSERT\":\n",
    "                ins_idx = random.randint(0, len(w) - 1)\n",
    "                ins_char_idx = np.random.randint(0, len(string.ascii_lowercase))\n",
    "                ins_char = list(string.ascii_lowercase)[ins_char_idx]\n",
    "                w = w[:ins_idx] + ins_char + w[ins_idx:]\n",
    "\n",
    "            if noise_op == \"REPLACE\":\n",
    "                target_idx = random.randint(0, len(w) - 1)\n",
    "                rep_char_idx = np.random.randint(0, len(string.ascii_lowercase))\n",
    "                rep_char = list(string.ascii_lowercase)[rep_char_idx]\n",
    "                w = w[:target_idx] + rep_char + w[target_idx + 1:]\n",
    "\n",
    "            if noise_op == \"ERRORS_TABLE\":\n",
    "                # Choose randomly the character to replace\n",
    "                # if fusion of two character is more probable should be modified...\n",
    "                choices = list(range(len(w)))\n",
    "                added = ''\n",
    "                while len(choices) > 0 and added == '':\n",
    "                    idx = random.randint(0, len(choices)-1)\n",
    "                    i = choices[idx]\n",
    "                    if i == len(w) - 1:\n",
    "                        if (i > 0) and w[i - 1] + w[i] in self.ERRORS_DICT:\n",
    "                            added = random.choice(self.ERRORS_DICT[w[i - 1] + w[i]])\n",
    "                            w = w[:i - 1] + added\n",
    "                        elif w[i] in self.ERRORS_DICT:\n",
    "                            added = random.choice(self.ERRORS_DICT[w[i]])\n",
    "                            w = w[:i] + added\n",
    "                        else:\n",
    "                            del choices[idx]\n",
    "                    elif i == len(w) - 2:\n",
    "                        if w[i] + w[i + 1] in self.ERRORS_DICT:\n",
    "                            added = random.choice(self.ERRORS_DICT[w[i] + w[i + 1]])\n",
    "                            w = w[:i] + added\n",
    "                        elif (i > 0) and w[i - 1] + w[i] in self.ERRORS_DICT:\n",
    "                            added = random.choice(self.ERRORS_DICT[w[i - 1] + w[i]])\n",
    "                            w = w[:i - 1] + added + w[i + 1]\n",
    "                        elif w[i] in self.ERRORS_DICT:\n",
    "                            added = random.choice(self.ERRORS_DICT[w[i]])\n",
    "                            w = w[:i] + added + w[i + 1:]\n",
    "                        else:\n",
    "                            del choices[idx]\n",
    "                    else:\n",
    "                        if w[i] + w[i + 1] in self.ERRORS_DICT:\n",
    "                            added = random.choice(self.ERRORS_DICT[w[i] + w[i + 1]])\n",
    "                            w = w[:i] + added + w[i + 2:]\n",
    "                        elif (i > 0) and w[i - 1] + w[i] in self.ERRORS_DICT:\n",
    "                            added = random.choice(self.ERRORS_DICT[w[i - 1] + w[i]])\n",
    "                            w = w[:i - 1] + added + w[i + 1:]\n",
    "                        elif w[i] in self.ERRORS_DICT:\n",
    "                            added = random.choice(self.ERRORS_DICT[w[i]])\n",
    "                            w = w[:i] + added + w[i + 1:]\n",
    "                        else:\n",
    "                            del choices[idx]\n",
    "        return w\n",
    "\n",
    "    def inject_noise(self, source_sentence):\n",
    "        tokens_list= self.tokenizeTextString(source_sentence)\n",
    "        replacements = []\n",
    "        for token in tokens_list:\n",
    "            if len(token) > 3:\n",
    "                if random.uniform(0, 1) <= self.NOISE_PERCENT_PER_LONG_WORD  :\n",
    "                    token = self.noise_char(token)\n",
    "            replacements.append(token)\n",
    "        return \" \".join(replacements) \n",
    "\n",
    "    def generate_paraphrases(self, source_sentence):\n",
    "        paraphrases = {}\n",
    "        iterations = 0\n",
    "        while (len(paraphrases) < self.PARAPHRASES_NUMBER_MAX) and (iterations < self.MAX_ITERATIONS) :\n",
    "            noisy_sentence = self.inject_noise(source_sentence)\n",
    "            if len(paraphrases) <= self.PARAPHRASES_NUMBER_MAX:\n",
    "                paraphrases.update(self.add_new_paraphrase(paraphrases,source_sentence,noisy_sentence))\n",
    "            iterations += 1\n",
    "        return paraphrases       "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Local data directories setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_dir = \"DATA/\"\n",
    "neg_rev = \"neg/\"\n",
    "pos_rev = \"pos/\"\n",
    "aug_neg_rev = \"noiseinjection_aug_neg/\"\n",
    "aug_pos_rev = \"noiseinjection_aug_pos/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment - Creation of TDA Noise Injection object "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paraphr_nbr_max = 5\n",
    "paratext_nbr_max = 5\n",
    "noise_percent = 0.1\n",
    "errors_table = 'no_errors_table'\n",
    "my_tda = NoiseInjection(data_dir, neg_rev, pos_rev, aug_neg_rev, aug_pos_rev, \n",
    "                        paraphr_nbr_max, paratext_nbr_max, noise_percent, errors_table)\n",
    "my_tda.isNoiseInjection()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_tda.isTextDataAugmentation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'TextDataAugmentation'"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_tda.name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment - word level noise injection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "augmentation\n",
      "augmenhation\n",
      "augomentation\n",
      "atgmentation\n"
     ]
    }
   ],
   "source": [
    "word = 'augmentation'\n",
    "print(word)\n",
    "w = my_tda.noise_char(word)\n",
    "print(w)\n",
    "w = my_tda.noise_char(word)\n",
    "print(w)\n",
    "w = my_tda.noise_char(word)\n",
    "print(w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment - sentence level noise injection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_tda.NOISE_PERCENT_PER_LONG_WORD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "original_text = \"to be or not to be that is the question\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'to be or not to be hat is the question'"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_tda.inject_noise(original_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment - paraphrases generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'to be or not to be that is the question': 1,\n",
       " 'to be or not to be thadt is the question': 1,\n",
       " 'to be or not to be thdat is the question': 1,\n",
       " 'to be or not to be tha is the question': 1,\n",
       " 'to be or not to be that is the queswtion': 1}"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_tda.generate_paraphrases(original_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment - Text Data Augmentation by extensive paratexts generation using Noise injection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "my_tda.augment_text_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "    Augmenting... DATA/neg/cv676_22202.txt\n",
    "    Paratext Dict created for the document\n",
    "    Text Data Augmentation done for the document\n",
    "    Saved: DATA/noiseinjection_aug_neg/ampl_cv676_22202_0.txt\n",
    "    Elapsed time: 0.054947464959695935\n",
    "    -----\n",
    "    Augmenting... DATA/neg/cv839_22807.txt\n",
    "    Paratext Dict created for the document\n",
    "    Text Data Augmentation done for the document\n",
    "    Saved: DATA/noiseinjection_aug_neg/ampl_cv839_22807_0.txt\n",
    "    Elapsed time: 0.034176019951701164\n",
    "    -----\n",
    "    Augmenting... DATA/neg/cv155_7845.txt\n",
    "    Paratext Dict created for the document\n",
    "    Text Data Augmentation done for the document\n",
    "    Saved: DATA/noiseinjection_aug_neg/ampl_cv155_7845_0.txt\n",
    "    Elapsed time: 0.011110538151115179\n",
    "    ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}