Scripts/Word_Level_Text_Generation_First_Time.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 2,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Word-Level Text Generation First Time",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "<table align=\"center\">\n",
        "  <td align=\"center\"><a target=\"_blank\" href=\"https://colab.research.google.com/github/dhairya0907/Creepypasta-Text-Generator/blob/main/Scripts/Word_Level_Text_Generation_First_Time.ipynb\">\n",
        "        <img src=\"https://i.ibb.co/2P3SLwK/colab.png\"  style=\"padding-bottom:5px;\" />Run in Google Colab</a></td>\n",
        "  <td align=\"center\"><a target=\"_blank\" href=\"https://github.com/dhairya0907/Creepypasta-Text-Generator/blob/main/Scripts/Word_Level_Text_Generation_First_Time.ipynb\">\n",
        "        <img src=\"https://i.ibb.co/xfJbPmL/github.png\"  height=\"70px\" style=\"padding-bottom:5px;\"  />View Source on GitHub</a></td>\n",
        "</table>"
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "%%capture\n",
        "!pip install wandb -qqq"
      ],
      "outputs": [],
      "metadata": {
        "id": "fqAW5GhyxSMv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "%load_ext tensorboard"
      ],
      "outputs": [],
      "metadata": {
        "id": "GA5spwlc11j-"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "import tensorflow.compat.v2 as tf\n",
        "from tensorflow.python.framework.ops import disable_eager_execution\n",
        "\n",
        "tf.enable_v2_behavior()\n",
        "disable_eager_execution()"
      ],
      "outputs": [],
      "metadata": {
        "id": "4yCshBeIczYw"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "import os\n",
        "import sys\n",
        "import time\n",
        "import wandb\n",
        "import keras\n",
        "import string\n",
        "import datetime\n",
        "import numpy as np\n",
        "from tqdm import tqdm\n",
        "\n",
        "from wandb import AlertLevel\n",
        "from google.colab import drive\n",
        "from wandb.keras import WandbCallback\n",
        "from keras.layers import LSTM, Dense, Embedding"
      ],
      "outputs": [],
      "metadata": {
        "id": "XfBenkXZOA0F"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "drive.mount('/content/drive')"
      ],
      "outputs": [],
      "metadata": {
        "id": "wZ8Fw4AgP8cr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "wandb.login()"
      ],
      "outputs": [],
      "metadata": {
        "id": "L-mgWm4QxZyx"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "Epochs = 50\n",
        "Batch = 256\n",
        "sequence_len = 20\n",
        "diversity_random = 2 \n",
        "predict_normal_len = 100\n",
        "predict_random_len = 100\n",
        "vocab_len_highest = 0 # len of the highest vocab in all the data\n",
        "main_folder_name = \"\" # Folder name where the file is stored\n",
        "file_string = \"had , ? wine\" # Any two words from the text file\n",
        "file_name  = \"\" # File name on which you want traning to be done"
      ],
      "outputs": [],
      "metadata": {
        "id": "15jTjc94aDpF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "folder_file = file_name.split(\"/\")"
      ],
      "outputs": [],
      "metadata": {
        "id": "p_e4JTd0Ahs2"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "f = open(\"/content/drive/My Drive/path_to_file\", \"r\")\n",
        "text = f.read()\n",
        "f.close()"
      ],
      "outputs": [],
      "metadata": {
        "id": "DaitU8fsOJ8z"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "text = text.lower()\n",
        "text = text.split()\n",
        "vocab = set(text)\n",
        "len(vocab)"
      ],
      "outputs": [],
      "metadata": {
        "id": "z9NV1frkPEBO"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "# Optional wandb\n",
        "\n",
        "experiment_name = \"\"\n",
        "\n",
        "wandb.init(project=\"\", \n",
        "           name=file_name, \n",
        "           sync_tensorboard=True,\n",
        "           group=experiment_name,\n",
        "           notes=\"\",\n",
        "           tags=[\"\", \"\", \"\",\"\",\"\"],)\n",
        "\n",
        "artifact = wandb.Artifact('', type='dataset')\n",
        "artifact.add_file(\"/content/drive/My Drive/path_to_file\")\n",
        "wandb.log_artifact(artifact)\n",
        "\n",
        "wandb.config.update({\"epochs\": Epochs, \n",
        "                     \"batch_size\": Batch,\n",
        "                     \"architecture\": \"RNN\",\n",
        "                     \"dataset\": file_name,\n",
        "                     \"vocab_Orignal\": len(vocab),\n",
        "                     \"vocab_Max\": vocab_len_highest,\n",
        "                     \"activation\": \"softmax\",\n",
        "                     \"optimizer\": \"Adam\",\n",
        "                     \"loss\": \"sparse_categorical_crossentropy\",\n",
        "                     \"metric\": \"accuracy\",\n",
        "                     })\n",
        "\n",
        "\n",
        "WANDB_DISABLE_CODE= \"true\""
      ],
      "outputs": [],
      "metadata": {
        "id": "IforLt-GxdXS"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "print(vocab)"
      ],
      "outputs": [],
      "metadata": {
        "id": "qNmZ62UI7BWp"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "index_to_word = {i: word for i, word in enumerate(vocab)}\n",
        "word_to_index = {word: i for i, word in index_to_word.items()}"
      ],
      "outputs": [],
      "metadata": {
        "id": "SmxxibEzQM45"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "def normalize(s):\n",
        "    chars = string.punctuation\n",
        "    for c in tqdm(chars):\n",
        "        s = s.replace(c,\" \"+c+\" \")\n",
        "    s = s.lower()\n",
        "    return s"
      ],
      "outputs": [],
      "metadata": {
        "id": "EStWSwj0QksG"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "def accept(words):\n",
        "    for word in tqdm(words):\n",
        "        if(not word in vocab):\n",
        "            return False\n",
        "    return True"
      ],
      "outputs": [],
      "metadata": {
        "id": "eg7vyv9JQrKk"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "def text_to_index(s):\n",
        "    s = normalize(s)\n",
        "    s_words = s.split()\n",
        "    if (not accept(s_words)):\n",
        "        print(\"Error\")\n",
        "        return \"\"\n",
        "    return np.array([word_to_index[word] for word in s_words])"
      ],
      "outputs": [],
      "metadata": {
        "id": "H_299ZFHQvvh"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "def word_list_to_index(s_words):\n",
        "    if (not accept(s_words)):\n",
        "        print(\"Error\")\n",
        "        return \"\"\n",
        "    return np.array([word_to_index[word] for word in s_words])"
      ],
      "outputs": [],
      "metadata": {
        "id": "-0baBE_JQwtl"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "def index_to_text(inds):\n",
        "    s = \"\"\n",
        "    for i in tqdm(range(inds.shape[0])):\n",
        "        s += index_to_word[inds[i]] + \" \"\n",
        "    return s"
      ],
      "outputs": [],
      "metadata": {
        "id": "-de_hTvcQy3g"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "def predict(s, numWords):\n",
        "  s_return = s\n",
        "  s_ind = text_to_index(s)\n",
        "  model_new.reset_states()\n",
        "  \n",
        "  for i in tqdm(range(s_ind.shape[0])):\n",
        "    pred = model_new.predict_on_batch(np.array(s_ind[i]).reshape(1,1))\n",
        "    \n",
        "  for i in tqdm(range(numWords)):\n",
        "    next_word_ind = np.argmax(pred)\n",
        "    s_return += \" \" + index_to_word[next_word_ind]\n",
        "    pred = model_new.predict_on_batch(np.array(next_word_ind).reshape(1,1))\n",
        "\n",
        "\n",
        "  filename = \"/content/drive/My Drive/path_to_store_all_stories\"\n",
        "  if not os.path.exists(os.path.dirname(filename)):\n",
        "    try:\n",
        "        os.makedirs(os.path.dirname(filename))\n",
        "    except OSError as exc:\n",
        "        if exc.errno != errno.EEXIST:\n",
        "            raise\n",
        "\n",
        "    f = open(filename, \"a\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"Number of words : \\n\" + str(numWords))\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"Starting String : \\n\" + str(s))\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"Story : \\n\")\n",
        "    f.write(s_return)\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.close()\n",
        "  \n",
        "  return s_return"
      ],
      "outputs": [],
      "metadata": {
        "id": "Krr1NxwmmoaM"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "def predictRandom(s, numWords, conf = 1):\n",
        "  s_return = s\n",
        "  s_ind = text_to_index(s)\n",
        "  model_new.reset_states()\n",
        "  \n",
        "  for i in tqdm(range(s_ind.shape[0])):\n",
        "    pred = model_new.predict_on_batch(np.array(s_ind[i]).reshape(1,1))\n",
        "    \n",
        "  for i in tqdm(range(numWords)):\n",
        "    pred_new = np.power(pred[0], conf)\n",
        "    pred_new = pred_new / np.sum(pred_new)\n",
        "    \n",
        "    next_word_ind = np.random.choice(np.arange(vocab_len_highest), p = pred_new)\n",
        "    \n",
        "    s_return += \" \" + index_to_word[next_word_ind]\n",
        "    pred = model_new.predict_on_batch(np.array(next_word_ind).reshape(1,1))\n",
        "    \n",
        "    \n",
        "  filename = \"/content/drive/My Drive/path_to_store_all_stories\"\n",
        "  if not os.path.exists(os.path.dirname(filename)):\n",
        "    try:\n",
        "        os.makedirs(os.path.dirname(filename))\n",
        "    except OSError as exc:\n",
        "        if exc.errno != errno.EEXIST:\n",
        "            raise\n",
        "\n",
        "    f = open(filename, \"a\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"Number of words : \\n\" + str(numWords))\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"Starting String : \\n\" + str(s))\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"Story : \\n\")\n",
        "    f.write(s_return)\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    f.close()\n",
        "  \n",
        "  return s_return"
      ],
      "outputs": [],
      "metadata": {
        "id": "84wYF_6UmujK"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "normalize(\" irfvjj; dd:kDJJRRJ\")"
      ],
      "outputs": [],
      "metadata": {
        "id": "V5WE3Fs7kUkX"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "index_to_text(text_to_index(file_string))"
      ],
      "outputs": [],
      "metadata": {
        "id": "xfWKENsYRRb4"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "text_ind = word_list_to_index(text)"
      ],
      "outputs": [],
      "metadata": {
        "id": "MrAWYkPoRTZm"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "train_x = []\n",
        "train_y = []\n",
        "\n",
        "for i in tqdm(range(text_ind.shape[0] - sequence_len - 1)):\n",
        "    train_x.append(text_ind[i:i+sequence_len])\n",
        "    train_y.append(text_ind[i+sequence_len])\n",
        "\n",
        "train_x = np.array(train_x)\n",
        "train_y = np.array(train_y)"
      ],
      "outputs": [],
      "metadata": {
        "id": "qtx3_hdNRVRA"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "train_x.shape"
      ],
      "outputs": [],
      "metadata": {
        "id": "SbVVW-WjRWF-"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "train_y.shape"
      ],
      "outputs": [],
      "metadata": {
        "id": "mmgKuVnQRYKf"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "index_to_text(train_x[46])\n",
        "index_to_text(np.array([train_y[45]]))"
      ],
      "outputs": [],
      "metadata": {
        "id": "MqvaLEcVRZ0K"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "model = keras.Sequential()\n",
        "model.add(Embedding(vocab_len_highest, 30))\n",
        "model.add(LSTM(64, return_sequences = True))\n",
        "model.add(LSTM(128, return_sequences = False))\n",
        "model.add(Dense(vocab_len_highest, activation = \"softmax\"))"
      ],
      "outputs": [],
      "metadata": {
        "id": "ORZBvqNRRpTa"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "model.summary()"
      ],
      "outputs": [],
      "metadata": {
        "id": "1_b4wGR_RrWj"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "model.compile(optimizer=keras.optimizers.Adam(), \n",
        "              loss = keras.losses.sparse_categorical_crossentropy, \n",
        "              metrics=['accuracy'])"
      ],
      "outputs": [],
      "metadata": {
        "id": "TFwXt5fzRtgP"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "wandb.config.update({\"epochs\": Epochs, \n",
        "                     \"batch_size\": Batch,\n",
        "                     \"architecture\": \"RNN\",\n",
        "                     \"dataset\": file_name,\n",
        "                     \"vocab_Orignal\": len(vocab),\n",
        "                     \"vocab_Max\": vocab_len_highest,\n",
        "                     \"activation\": \"softmax\",\n",
        "                     \"optimizer\": \"Adam\",\n",
        "                     \"loss\": \"sparse_categorical_crossentropy\",\n",
        "                     \"metric\": \"accuracy\",\n",
        "                     \"layers\" : len(model.layers),\n",
        "                     })"
      ],
      "outputs": [],
      "metadata": {
        "id": "utI0FaV4-4fs"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "filepath=\"/content/drive/My Drive/\"+os.path.splitext(folder_file[2])[0]+\"-second-{epoch:02d}-{loss:.4f}.hdf5\" # Path to save the model checkpoints\n",
        "checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')\n",
        "\n",
        "\n",
        "log_dir = \"logs/fit/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
        "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
        "\n",
        "callbacks_list = [checkpoint, WandbCallback(), tensorboard_callback]"
      ],
      "outputs": [],
      "metadata": {
        "id": "9uBdsWJkRvg-"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "start_time = time.time()\n",
        "\n",
        "model.fit(train_x, train_y, batch_size = Batch, epochs = Epochs, callbacks = callbacks_list)\n",
        "\n",
        "print(\"--- %s seconds ---\" % (time.time() - start_time))"
      ],
      "outputs": [],
      "metadata": {
        "id": "4vocaSUoSG5m"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "time_in_min = (time.time() - start_time)/60\n",
        "print(time_in_min)"
      ],
      "outputs": [],
      "metadata": {
        "id": "o1pJVzhrgmWv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "time_in_min = round(time_in_min, 2)\n",
        "print(time_in_min)"
      ],
      "outputs": [],
      "metadata": {
        "id": "IFSY6lU1jfjR"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "time_in_hr = round(time_in_min, 2)/60\n",
        "print(time_in_hr)"
      ],
      "outputs": [],
      "metadata": {
        "id": "LXi5OU72j5v2"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "time_in_hr = round(time_in_hr, 2)\n",
        "print(time_in_hr)"
      ],
      "outputs": [],
      "metadata": {
        "id": "xsvqRDeSkDZB"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "wandb.config.update({\"fitting_time\" : str(time_in_hr)+\"h\"}) # Optional"
      ],
      "outputs": [],
      "metadata": {
        "id": "3Lf-vvHlEyC0"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "#!kill 3639\n",
        "%tensorboard --logdir logs/fit"
      ],
      "outputs": [],
      "metadata": {
        "id": "0VCIy3na2RQ0"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "model.save(\"/content/drive/My Drive/\"+os.path.splitext(folder_file[2])[0]+\"_final_model.hdf5\")"
      ],
      "outputs": [],
      "metadata": {
        "id": "SKvuzzRAQ6uO"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "model.save(\"/content/drive/My Drive/Machine Learning Research/output/model/Algorithmia/\"+os.path.splitext(folder_file[2])[0]+\"_algorithmia_model.h5\")"
      ],
      "outputs": [],
      "metadata": {
        "id": "1VcxJ9o_tBAh"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "model_new = keras.Sequential()\n",
        "model_new.add(Embedding(vocab_len_highest, 30, batch_input_shape = (1, 1)))\n",
        "model_new.add(LSTM(64, return_sequences = True, stateful = True))\n",
        "model_new.add(LSTM(128, return_sequences = False, stateful = True))\n",
        "model_new.add(Dense(vocab_len_highest, activation = \"softmax\"))"
      ],
      "outputs": [],
      "metadata": {
        "id": "7qaUQm7dUn6w"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "model_new.set_weights(model.get_weights())"
      ],
      "outputs": [],
      "metadata": {
        "id": "UuqC-39aUxWo"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "predict(\"\", predict_normal_len)"
      ],
      "outputs": [],
      "metadata": {
        "id": "7Gdo5x9gU024"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "predictRandom(\"\", predict_random_len, diversity_random)"
      ],
      "outputs": [],
      "metadata": {
        "id": "GXLf1MDOs9jD"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "# Optional\n",
        "\n",
        "trained_model_artifact = wandb.Artifact(\n",
        "            os.path.splitext(folder_file[2])[0]+\"_algorithmia_model.h5\", type=\"model\",\n",
        "            description=\"\",)\n",
        "\n",
        "\n",
        "trained_model_artifact.add_file(\"/content/drive/My Drive/Machine Learning Research/output/model/Algorithmia/\"+os.path.splitext(folder_file[2])[0]+\"_algorithmia_model.h5\")\n",
        "wandb.log_artifact(trained_model_artifact)"
      ],
      "outputs": [],
      "metadata": {
        "id": "BcpSqRfEQfFE"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "# Optional\n",
        "wandb.alert(\n",
        "      title='Run  '+main_folder_name+'  Finished',\n",
        "      text=f'The Run of file {file_name} is completed in {(time.time() - start_time)} seconds.',\n",
        "      level=AlertLevel.INFO,\n",
        "      wait_duration= datetime.timedelta(minutes=0)\n",
        "    )"
      ],
      "outputs": [],
      "metadata": {
        "id": "qDJWjsY-IApd"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "source": [
        "# Optional\n",
        "\n",
        "wandb.finish()"
      ],
      "outputs": [],
      "metadata": {
        "id": "IoRa7gluJwPs"
      }
    }
  ]
}