session_2/Makerspace Session 2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d716f5fb",
   "metadata": {},
   "source": [
    "# AI Makerspace Session 2\n",
    "_Hosted by:_ Enrique Noriega-Atala\n",
    "_Date:_ 10/11/24"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77da551a",
   "metadata": {},
   "source": [
    "This notebook is an implementation of the following tutorial: https://www.datacamp.com/tutorial/flan-t5-tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0b3e266b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: nltk in ./.local/lib/python3.8/site-packages (3.9.1)\n",
      "Requirement already satisfied: datasets in ./.local/lib/python3.8/site-packages (3.0.1)\n",
      "Requirement already satisfied: transformers in ./.local/lib/python3.8/site-packages (4.45.2)\n",
      "Requirement already satisfied: tokenizers in ./.local/lib/python3.8/site-packages (0.20.1)\n",
      "Requirement already satisfied: evaluate in ./.local/lib/python3.8/site-packages (0.4.3)\n",
      "Requirement already satisfied: rouge_score in ./.local/lib/python3.8/site-packages (0.1.2)\n",
      "Requirement already satisfied: sentencepiece in ./.local/lib/python3.8/site-packages (0.2.0)\n",
      "Requirement already satisfied: huggingface_hub in ./.local/lib/python3.8/site-packages (0.25.2)\n",
      "Requirement already satisfied: accelerate in ./.local/lib/python3.8/site-packages (1.0.0)\n",
      "Requirement already satisfied: joblib in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from nltk) (1.1.0)\n",
      "Requirement already satisfied: click in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from nltk) (8.0.3)\n",
      "Requirement already satisfied: regex>=2021.8.3 in ./.local/lib/python3.8/site-packages (from nltk) (2024.9.11)\n",
      "Requirement already satisfied: tqdm in ./.local/lib/python3.8/site-packages (from nltk) (4.66.5)\n",
      "Requirement already satisfied: aiohttp in ./.local/lib/python3.8/site-packages (from datasets) (3.10.10)\n",
      "Requirement already satisfied: pandas in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from datasets) (1.4.0)\n",
      "Requirement already satisfied: xxhash in ./.local/lib/python3.8/site-packages (from datasets) (3.5.0)\n",
      "Requirement already satisfied: filelock in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from datasets) (3.4.2)\n",
      "Requirement already satisfied: requests>=2.32.2 in ./.local/lib/python3.8/site-packages (from datasets) (2.32.3)\n",
      "Requirement already satisfied: packaging in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from datasets) (21.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from datasets) (6.0)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in ./.local/lib/python3.8/site-packages (from datasets) (0.3.8)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from datasets) (1.22.2)\n",
      "Requirement already satisfied: fsspec[http]<=2024.6.1,>=2023.1.0 in ./.local/lib/python3.8/site-packages (from datasets) (2024.2.0)\n",
      "Requirement already satisfied: pyarrow>=15.0.0 in ./.local/lib/python3.8/site-packages (from datasets) (17.0.0)\n",
      "Requirement already satisfied: multiprocess in ./.local/lib/python3.8/site-packages (from datasets) (0.70.16)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in ./.local/lib/python3.8/site-packages (from transformers) (0.4.5)\n",
      "Requirement already satisfied: absl-py in ./.local/lib/python3.8/site-packages (from rouge_score) (2.1.0)\n",
      "Requirement already satisfied: six>=1.14.0 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from rouge_score) (1.16.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./.local/lib/python3.8/site-packages (from huggingface_hub) (4.9.0)\n",
      "Requirement already satisfied: torch>=1.10.0 in ./.local/lib/python3.8/site-packages (from accelerate) (2.4.1+cu124)\n",
      "Requirement already satisfied: psutil in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from accelerate) (5.9.0)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets) (2.4.3)\n",
      "Requirement already satisfied: yarl<2.0,>=1.12.0 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets) (1.14.0)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.3)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets) (1.4.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from aiohttp->datasets) (21.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets) (6.1.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from packaging->datasets) (3.0.7)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (2.0.11)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (2021.10.8)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (3.3)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (1.26.8)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.6.0.99 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (11.6.0.99)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.4.99 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (12.4.99)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.4.99 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (12.4.99)\n",
      "Requirement already satisfied: triton==3.0.0 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (3.0.0)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.4.2.65 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (12.4.2.65)\n",
      "Requirement already satisfied: sympy in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (1.12)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (2.20.5)\n",
      "Requirement already satisfied: jinja2 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (3.0.3)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.3.0.142 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (12.3.0.142)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12==12.4.99 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (12.4.99)\n",
      "Requirement already satisfied: networkx in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (3.0)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.4.99 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (12.4.99)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (9.1.0.70)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.4.99 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (12.4.99)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.2.0.44 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (11.2.0.44)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.5.119 in ./.local/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (10.3.5.119)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from pandas->datasets) (2021.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: propcache>=0.2.0 in ./.local/lib/python3.8/site-packages (from yarl<2.0,>=1.12.0->aiohttp->datasets) (0.2.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/ohpc/pub/apps/python/3.8.12/lib/python3.8/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.0.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in ./.local/lib/python3.8/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
      "\u001b[33mWARNING: You are using pip version 22.0.3; however, version 24.2 is available.\n",
      "You should consider upgrading via the '/opt/ohpc/pub/apps/python/3.8.12/bin/python3.8 -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install nltk datasets transformers tokenizers evaluate rouge_score sentencepiece huggingface_hub accelerate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5cb1267b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import evaluate\n",
    "import numpy as np\n",
    "from datasets import load_dataset\n",
    "from transformers import T5Tokenizer, DataCollatorForSeq2Seq\n",
    "from transformers import T5ForConditionalGeneration, Seq2SeqTrainingArguments, Seq2SeqTrainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "993989d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n"
     ]
    }
   ],
   "source": [
    "MODEL_NAME = \"google/flan-t5-base\"\n",
    "\n",
    "tokenizer = T5Tokenizer.from_pretrained(MODEL_NAME)\n",
    "model = T5ForConditionalGeneration.from_pretrained(MODEL_NAME)\n",
    "data_collator = DataCollatorForSeq2Seq(tokenizer, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a991f24a",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_NAME= \"yahoo_answers_qa\"\n",
    "yahoo_answers_qa = load_dataset(DATA_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c85afcb7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'question', 'answer', 'nbestanswers', 'main_category'],\n",
       "        num_rows: 87362\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yahoo_answers_qa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "41f61a7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'question', 'answer', 'nbestanswers', 'main_category'],\n",
       "        num_rows: 61153\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['id', 'question', 'answer', 'nbestanswers', 'main_category'],\n",
       "        num_rows: 26209\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yahoo_answers_qa = yahoo_answers_qa['train'].train_test_split(test_size=0.3)\n",
    "yahoo_answers_qa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5001e187",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['What is the measurement of the yeast in a yeast packet?'],\n",
       " ['I believe it is slightly less than 1 table spoon but I exchange 1 tablespoon for the packet.  The only possible difference is that the rise might occur slightly faster but not that noticable.'])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = yahoo_answers_qa['test'].select([1256])\n",
    "d['question'], d['answer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "338f553b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[863,\n",
       "  1525,\n",
       "  48,\n",
       "  822,\n",
       "  10,\n",
       "  363,\n",
       "  19,\n",
       "  8,\n",
       "  9753,\n",
       "  13,\n",
       "  8,\n",
       "  17937,\n",
       "  16,\n",
       "  3,\n",
       "  9,\n",
       "  17937,\n",
       "  13531,\n",
       "  58,\n",
       "  1]]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We prefix our tasks with \"answer the question\"\n",
    "prefix = \"Please answer this question: \"\n",
    "\n",
    "def preprocess_function(examples):\n",
    "    \"\"\"Add prefix to the sentences, tokenize the text, and set the labels\"\"\"\n",
    "   # The \"inputs\" are the tokenized answer:\n",
    "    \n",
    "    inputs = [prefix + doc for doc in examples[\"question\"]]\n",
    "    model_inputs = tokenizer(inputs, max_length=128, truncation=True)\n",
    "  \n",
    "    # The \"labels\" are the tokenized outputs:\n",
    "    labels = tokenizer(text_target=examples[\"answer\"], \n",
    "                      max_length=512,         \n",
    "                      truncation=True)\n",
    "    \n",
    "    model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
    "    return model_inputs\n",
    "\n",
    "x = preprocess_function(yahoo_answers_qa['test'].select([1256]))\n",
    "x['input_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9749e248",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0841faf0105a416daaef1febd81a4767",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/61153 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/u6/enoriega/.local/lib/python3.8/site-packages/transformers/models/t5/tokenization_t5.py:289: UserWarning: This sequence already has </s>. In future versions this behavior may lead to duplicated eos tokens being added.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c277569d256a46009a60c61943af9dfc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/26209 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Map the preprocessing function across our dataset\n",
    "tokenized_dataset = yahoo_answers_qa.map(preprocess_function, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8ac1a8a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "nltk.download(\"punkt\", quiet=True)\n",
    "metric = evaluate.load(\"rouge\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d9f5e316",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(eval_preds):\n",
    "    generations, groundtruth = eval_preds\n",
    "\n",
    "    # decode preds and labels\n",
    "    groundtruth = np.where(groundtruth != -100, groundtruth, tokenizer.pad_token_id)\n",
    "    decoded_generations = tokenizer.batch_decode(generations, skip_special_tokens=True)\n",
    "    decoded_groundtruth = tokenizer.batch_decode(groundtruth, skip_special_tokens=True)\n",
    "\n",
    "    # rougeLSum expects newline after each sentence\n",
    "    decoded_preds = [\"\\n\".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_generations]\n",
    "    decoded_labels = [\"\\n\".join(nltk.sent_tokenize(label.strip())) for label in decoded_groundtruth]\n",
    "\n",
    "    result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)\n",
    "  \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4fbf0574",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/u6/enoriega/.local/lib/python3.8/site-packages/transformers/training_args.py:1545: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Global Parameters\n",
    "L_RATE = 3e-4\n",
    "BATCH_SIZE = 8\n",
    "PER_DEVICE_EVAL_BATCH = 4\n",
    "WEIGHT_DECAY = 0.01\n",
    "SAVE_TOTAL_LIM = 3\n",
    "NUM_EPOCHS = 3\n",
    "\n",
    "\n",
    "# Set up training arguments\n",
    "training_args = Seq2SeqTrainingArguments(\n",
    "   output_dir=\"./results\",\n",
    "   evaluation_strategy=\"epoch\",\n",
    "   learning_rate=L_RATE,\n",
    "   per_device_train_batch_size=BATCH_SIZE,\n",
    "   per_device_eval_batch_size=PER_DEVICE_EVAL_BATCH,\n",
    "   weight_decay=WEIGHT_DECAY,\n",
    "   save_total_limit=SAVE_TOTAL_LIM,\n",
    "   num_train_epochs=NUM_EPOCHS,\n",
    "   predict_with_generate=True,\n",
    "   push_to_hub=False\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f21c55fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Detected kernel version 3.10.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n"
     ]
    }
   ],
   "source": [
    "trainer = Seq2SeqTrainer(\n",
    "   model=model,\n",
    "   args=training_args,\n",
    "   train_dataset=tokenized_dataset[\"train\"],\n",
    "   eval_dataset=tokenized_dataset[\"test\"],\n",
    "   tokenizer=tokenizer,\n",
    "   data_collator=data_collator,\n",
    "   compute_metrics=compute_metrics\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26c2c13b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='6674' max='22935' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [ 6674/22935 27:51 < 1:07:53, 3.99 it/s, Epoch 0.87/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4af13777",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}