Language Identification from Very Short Strings - Load Model from Weights.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Abdul Wahab</h2>\n",
    "<h3>Natural Language Identification (Embedded Devices) - Using Deep Neural Network</h3>\n",
    "\n",
    "<p>\n",
    "In this project, I pulled text data from TED Talks in 63 languages.\n",
    "I converted the text into its binary reperesentation of 4 byte for each letter, utf-8 encoding. \n",
    "Using Tensorflow, I trained a simple deep neural network to classify input language. I acheived 91% accuracy with mostly spoken 17 languages and 80% accuracy with all 56 languages.\n",
    "</p>\n",
    "\n",
    "<p> \n",
    "Dataset: https://www.kaggle.com/wahabjawed/text-dataset-for-63-langauges\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Required libraries\n",
    "\n",
    "%config IPCompleter.greedy=True\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.enable_v2_behavior()\n",
    "tf.get_logger().setLevel('ERROR')\n",
    "from tensorflow.compat.v2.keras.models import Sequential\n",
    "from tensorflow.compat.v2.keras.layers import Dense,Dropout\n",
    "from tensorflow.compat.v2.keras import initializers, optimizers\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "from unidecode import unidecode\n",
    "from array import array\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Configuration</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Map language index to natural language\n",
    "\n",
    "labels_extended = { \n",
    "          0: ['Vietnamese','vi'], 1:['Albanian','sq'], 2:['Arabic','ar'],\n",
    "          3: ['Armenian','hy'], 4: ['Azerbaijani','az'], \n",
    "          5: ['Belarusian','be'],6: ['Bengali','bn'], \n",
    "          7: ['Bosnian','bs'], 8: ['Bulgarian','bg'], \n",
    "          9: ['Burmese','my'], 10: ['Catalan', 'ca'],\n",
    "          11: ['Chinese Simplified','zh-cn'], 12: ['Chinese Traditional','zh-tw'],\n",
    "          13: ['Chinese Yue','zh'], 14: ['Croatian','hr'],\n",
    "          15: ['Czech','cs'], 16: ['Danish','da'],\n",
    "          17: ['Dutch','nl'], 18: ['English','en'],\n",
    "          19: ['Esperanto','eo'], 20: ['Estonian','et'],\n",
    "          21: ['Finnish','fi'], 22:['French','fr'],\n",
    "          23: ['Galician','gl'], 24: ['Georgian','ka'], \n",
    "          25: ['German','de'],26: ['Urdu','ur'],\n",
    "          27: ['Gujarati','gu'], 28: ['Hebrew','he'], \n",
    "          29: ['Hindi','hi'], 30: ['Hungarian', 'hu'],\n",
    "          31: ['Indonesian','id'], 32: ['Italian','it'],\n",
    "          33: ['Japanese','ja'], 34: ['Korean','ko'],\n",
    "          35: ['Latvian','lv'], 36: ['Lithuanian','lt'],\n",
    "          37: ['Macedonian','mk'], 38: ['Malay','ms'],\n",
    "          39: ['Marathi','mr'], 40: ['Mongolian','mn'],\n",
    "          41: ['Norwegian','nb'], 42: ['Persian','bg'],\n",
    "          43: ['Polish','pl'], 44: ['Portuguese','pt'],\n",
    "          45: ['Romanian','ro'],46: ['Russian','ru'], \n",
    "          47: ['Serbian','sr'], 48: ['Slovak','sk'], \n",
    "          49: ['Slovenian','sl'], 50: ['Spanish', 'es'],\n",
    "          51: ['Swedish','sv'], 52: ['Tamil','ta'],\n",
    "          53: ['Thai','th'], 54: ['Turkish','tr'],\n",
    "          55: ['Ukrainian','uk']\n",
    "          }\n",
    "\n",
    "\n",
    "\n",
    "labels_standard = { \n",
    "        0: ['Indonesian','id'], 1:['English','en'], 2:['German','de'],\n",
    "        3: ['Turkish','tr'],4:['Hindi','hn'],\n",
    "        5: ['Spanish','es'],6: ['Bengali','bn'], \n",
    "        7: ['French','fr'], 8: ['Italian','it'], \n",
    "        9: ['Dutch','nl'], 10: ['Portuguese', 'pt'],\n",
    "        11: ['Swedish','sv'], 12: ['Russian','ru'],\n",
    "        13: ['Czech','cs'], 14: ['Arabic','ar'],\n",
    "        15: ['Chinese Traditional','zh-cn'],16: ['Persian','fa']\n",
    "}\n",
    "\n",
    "\n",
    "#['STANDARD','EXTENDED']\n",
    "# STANDARD supports 16 languages\n",
    "# EXTENDED supports 56 languages\n",
    "\n",
    "TYPE = 'STANDARD'\n",
    "\n",
    "\n",
    "\n",
    "# assign number of languages to process\n",
    "\n",
    "if(TYPE =='STANDARD'):\n",
    "    LABEL = labels_standard\n",
    "else:\n",
    "    LABEL = labels_extended\n",
    "\n",
    "\n",
    "# regular expression pattern used to filter out data\n",
    "\n",
    "pattern = r'[^\\w\\s]+|[0-9]'\n",
    "\n",
    "# Max length of input text\n",
    "MAX_INPUT_LENGTH = 13\n",
    "\n",
    "#MAX data length for each language to balnace the dataset\n",
    "MAX_LENGTH_DATA = 300000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Helper Functions</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper Functions\n",
    "\n",
    "def clean_sentences(sentences):\n",
    "    '''\n",
    "    Goal: Filter out non predictive text about speaker using regular expression pattern\n",
    "    \n",
    "    @param sentences: (list) sentences is a list of strings, where each string is a sentence.\n",
    "                       Note: The raw language_transcription should be tokenized by sentence prior\n",
    "                       to being passed into this function.\n",
    "    '''\n",
    "    return re.sub(pattern,'',sentences)\n",
    "\n",
    "def convertTextToBinary(word):\n",
    "    word_vec = []\n",
    "    vec = ''\n",
    "    n = len(word)\n",
    "    for i in range(n):\n",
    "        current_letter = word[i]\n",
    "        ind = ord(current_letter)\n",
    "        placeholder = bin(ind)[2:].zfill(32)\n",
    "        vec = vec + placeholder\n",
    "    vec = vec.zfill(32*MAX_INPUT_LENGTH)\n",
    "    for digit in vec:\n",
    "        word_vec.append(int(digit))\n",
    "    return word_vec\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Deep Neural Network - Helper Function</h2>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createModelStandard():\n",
    "    initializer = initializers.he_uniform()\n",
    "    model = Sequential()\n",
    "    model.add(Dense(416, activation='relu', kernel_initializer=initializer, input_dim=MAX_INPUT_LENGTH*32))\n",
    "    model.add(Dense(512, activation='relu', kernel_initializer=initializer))\n",
    "    model.add(Dense(128, activation='relu', kernel_initializer=initializer))\n",
    "    model.add(Dropout(0.15))\n",
    "    model.add(Dense(len(LABEL), activation='softmax'))\n",
    "    model.summary()\n",
    "    \n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer=tf.keras.optimizers.Adam(1e-3), metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "\n",
    "def createModelExtended():\n",
    "    initializer = initializers.he_uniform()\n",
    "    model = Sequential()\n",
    "    model.add(Dense(416, activation='relu', kernel_initializer=initializer, input_dim=MAX_INPUT_LENGTH*32))\n",
    "    model.add(Dense(1024, activation='relu', kernel_initializer=initializer))\n",
    "    model.add(Dense(256, activation='relu', kernel_initializer=initializer))\n",
    "    model.add(Dropout(0.15))\n",
    "    model.add(Dense(len(LABEL), activation='softmax'))\n",
    "    model.summary()\n",
    "    \n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer=tf.keras.optimizers.Adam(1e-3), metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "def loadWeights():\n",
    "    model.load_weights(f'weights/{TYPE}/weights_{TYPE}.chk')\n",
    "    \n",
    "def detectLanguage(text, model):\n",
    "    #test for results\n",
    "\n",
    "    if len(text) > MAX_INPUT_LENGTH:\n",
    "        text = text[:MAX_INPUT_LENGTH]\n",
    "\n",
    "    text = clean_sentences(text)\n",
    "    word_vec = convertTextToBinary(text)\n",
    "    word_vec =np.array(word_vec,dtype='float32')\n",
    "    word_vec = np.reshape(word_vec, (1,word_vec.shape[0]))\n",
    "\n",
    "\n",
    "    output = model.predict(word_vec)\n",
    "    \n",
    "    digit = np.argmax(output[0])\n",
    "    \n",
    "   \n",
    "\n",
    "    print(f\"the language for input {text}: {LABEL[digit][0]}\")\n",
    "    \n",
    "    for i in range(len(LABEL)):\n",
    "        lang = LABEL[i][0]\n",
    "        score = output[0][i]\n",
    "        print(lang + ': ' + str(round(100*score, 2)) + '%')\n",
    "    print('\\n')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Deep Neural Network - Load Weights From Disk</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_12 (Dense)             (None, 416)               173472    \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 512)               213504    \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 17)                2193      \n",
      "=================================================================\n",
      "Total params: 454,833\n",
      "Trainable params: 454,833\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# create model\n",
    "\n",
    "if(TYPE =='STANDARD'):\n",
    "    model = createModelStandard()\n",
    "else:\n",
    "    model = createModelExtended()\n",
    "\n",
    "# load weights\n",
    "\n",
    "loadWeights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: AutoGraph could not transform <function Model.make_predict_function.<locals>.predict_function at 0x15ca23a70> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: 'arguments' object has no attribute 'posonlyargs'\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
      "the language for input father: English\n",
      "Indonesian: 0.0%\n",
      "English: 99.91%\n",
      "German: 0.0%\n",
      "Turkish: 0.0%\n",
      "Hindi: 0.01%\n",
      "Spanish: 0.0%\n",
      "Bengali: 0.0%\n",
      "French: 0.01%\n",
      "Italian: 0.0%\n",
      "Dutch: 0.07%\n",
      "Portuguese: 0.0%\n",
      "Swedish: 0.01%\n",
      "Russian: 0.0%\n",
      "Czech: 0.0%\n",
      "Arabic: 0.0%\n",
      "Chinese Traditional: 0.0%\n",
      "Persian: 0.0%\n",
      "\n",
      "\n",
      "the language for input মনবত: Bengali\n",
      "Indonesian: 0.0%\n",
      "English: 0.0%\n",
      "German: 0.0%\n",
      "Turkish: 0.0%\n",
      "Hindi: 0.0%\n",
      "Spanish: 0.0%\n",
      "Bengali: 100.0%\n",
      "French: 0.0%\n",
      "Italian: 0.0%\n",
      "Dutch: 0.0%\n",
      "Portuguese: 0.0%\n",
      "Swedish: 0.0%\n",
      "Russian: 0.0%\n",
      "Czech: 0.0%\n",
      "Arabic: 0.0%\n",
      "Chinese Traditional: 0.0%\n",
      "Persian: 0.0%\n",
      "\n",
      "\n",
      "the language for input بچے: Persian\n",
      "Indonesian: 0.0%\n",
      "English: 0.0%\n",
      "German: 0.0%\n",
      "Turkish: 0.0%\n",
      "Hindi: 0.0%\n",
      "Spanish: 0.0%\n",
      "Bengali: 0.0%\n",
      "French: 0.0%\n",
      "Italian: 0.0%\n",
      "Dutch: 0.0%\n",
      "Portuguese: 0.0%\n",
      "Swedish: 0.0%\n",
      "Russian: 0.0%\n",
      "Czech: 0.0%\n",
      "Arabic: 0.0%\n",
      "Chinese Traditional: 0.0%\n",
      "Persian: 100.0%\n",
      "\n",
      "\n",
      "the language for input الأطفال: Arabic\n",
      "Indonesian: 0.0%\n",
      "English: 0.0%\n",
      "German: 0.0%\n",
      "Turkish: 0.0%\n",
      "Hindi: 0.0%\n",
      "Spanish: 0.0%\n",
      "Bengali: 0.0%\n",
      "French: 0.0%\n",
      "Italian: 0.0%\n",
      "Dutch: 0.0%\n",
      "Portuguese: 0.0%\n",
      "Swedish: 0.0%\n",
      "Russian: 0.0%\n",
      "Czech: 0.0%\n",
      "Arabic: 100.0%\n",
      "Chinese Traditional: 0.0%\n",
      "Persian: 0.0%\n",
      "\n",
      "\n",
      "the language for input إنسانية: Arabic\n",
      "Indonesian: 0.0%\n",
      "English: 0.0%\n",
      "German: 0.0%\n",
      "Turkish: 0.0%\n",
      "Hindi: 0.0%\n",
      "Spanish: 0.0%\n",
      "Bengali: 0.0%\n",
      "French: 0.0%\n",
      "Italian: 0.0%\n",
      "Dutch: 0.0%\n",
      "Portuguese: 0.0%\n",
      "Swedish: 0.0%\n",
      "Russian: 0.0%\n",
      "Czech: 0.0%\n",
      "Arabic: 100.0%\n",
      "Chinese Traditional: 0.0%\n",
      "Persian: 0.0%\n",
      "\n",
      "\n",
      "the language for input mänskligheten: Swedish\n",
      "Indonesian: 0.0%\n",
      "English: 0.0%\n",
      "German: 0.0%\n",
      "Turkish: 0.0%\n",
      "Hindi: 0.0%\n",
      "Spanish: 0.0%\n",
      "Bengali: 0.0%\n",
      "French: 0.0%\n",
      "Italian: 0.0%\n",
      "Dutch: 0.0%\n",
      "Portuguese: 0.0%\n",
      "Swedish: 100.0%\n",
      "Russian: 0.0%\n",
      "Czech: 0.0%\n",
      "Arabic: 0.0%\n",
      "Chinese Traditional: 0.0%\n",
      "Persian: 0.0%\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#test for results\n",
    "\n",
    "text_arr = ['father','মানবতা','بچے','الأطفال','إنسانية','mänskligheten']\n",
    "\n",
    "for text in text_arr:\n",
    "    detectLanguage(text, model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}