sentiment_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb6ef129-d9d0-4090-ab93-93f5530f4f9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "!tar -xf input.zip\n",
    "!tar -xf inference_data.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a28b59e1-688b-4c47-b676-84c8a23b1122",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import (\n",
    "    T5Tokenizer,\n",
    "    T5ForConditionalGeneration,\n",
    "    TrainingArguments,\n",
    "    Trainer\n",
    ")\n",
    "from datasets import load_dataset\n",
    "\n",
    "MODEL = 't5-small'\n",
    "BATCH_SIZE = 48\n",
    "NUM_PROCS = 16\n",
    "EPOCHS = 10\n",
    "OUT_DIR = 'results_t5small'\n",
    "MAX_LENGTH = 256\n",
    "\n",
    "dataset_train = load_dataset(\n",
    "    'csv', \n",
    "    data_files='input/train.csv',\n",
    "    split='train',\n",
    "    nrows=20000\n",
    ")\n",
    "dataset_valid = load_dataset(\n",
    "    'csv', \n",
    "    data_files='input/valid.csv',\n",
    "    split='train',\n",
    "    nrows=5000\n",
    ")\n",
    "print(len(dataset_train))\n",
    "print(len(dataset_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d581de4-7105-4483-9d3c-91425669913d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tokenizer = T5Tokenizer.from_pretrained(MODEL)\n",
    "def preprocess_function(examples):\n",
    "    inputs = [f\"assign tag: {title} {body}\" for (title, body) in zip(examples['Title'], examples['Body'])]\n",
    "    model_inputs = tokenizer(\n",
    "        inputs,\n",
    "        max_length=MAX_LENGTH,\n",
    "        truncation=True,\n",
    "        padding='max_length'\n",
    "    )\n",
    "    cleaned_tag = [' '.join(''.join(tag.split('<')).split('>')[:-1]) for tag in examples['Tags']]\n",
    "    with tokenizer.as_target_tokenizer():\n",
    "        labels = tokenizer(\n",
    "            cleaned_tag,\n",
    "            max_length=MAX_LENGTH,\n",
    "            truncation=True,\n",
    "            padding='max_length'\n",
    "        )\n",
    " \n",
    "    model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
    "    return model_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29066b81-a20b-4bfa-87ac-6132bed02b62",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tokenized_train = dataset_train.map(\n",
    "    preprocess_function,\n",
    "    batched=True,\n",
    "    num_proc=NUM_PROCS\n",
    ")\n",
    "tokenized_valid = dataset_valid.map(\n",
    "    preprocess_function,\n",
    "    batched=True,\n",
    "    num_proc=NUM_PROCS\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ed9662f-75ac-48f6-922a-76cdd93e9ef7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "trainable_params=[\n",
    "\n",
    "    'decoder.block.1.layer.2.DenseReluDense.wi.weight',\n",
    "    'decoder.block.1.layer.2.DenseReluDense.wo.weight',\n",
    "    'decoder.block.1.layer.2.layer_norm.weight',\n",
    "\n",
    "    'decoder.block.2.layer.2.DenseReluDense.wi.weight',\n",
    "    'decoder.block.2.layer.2.DenseReluDense.wo.weight',\n",
    "    'decoder.block.2.layer.2.layer_norm.weight',\n",
    "    \n",
    "    'decoder.block.3.layer.2.DenseReluDense.wi.weight',\n",
    "    'decoder.block.3.layer.2.DenseReluDense.wo.weight',\n",
    "    'decoder.block.3.layer.2.layer_norm.weight',\n",
    "   \n",
    "    'decoder.block.4.layer.2.DenseReluDense.wi.weight',\n",
    "    'decoder.block.4.layer.2.DenseReluDense.wo.weight',\n",
    "    'decoder.block.4.layer.2.layer_norm.weight',\n",
    "    \n",
    "    'decoder.block.5.layer.0.SelfAttention.q.weight',\n",
    "    'decoder.block.5.layer.0.SelfAttention.k.weight',\n",
    "    'decoder.block.5.layer.0.SelfAttention.v.weight',\n",
    "    'decoder.block.5.layer.0.SelfAttention.o.weight',\n",
    "    'decoder.block.5.layer.0.layer_norm.weight',\n",
    "    'decoder.block.5.layer.2.DenseReluDense.wi.weight',\n",
    "    'decoder.block.5.layer.2.DenseReluDense.wo.weight',\n",
    "    'decoder.block.5.layer.2.layer_norm.weight',\n",
    "    'decoder.final_layer_norm.weight'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0f0dfcc-4ee5-498e-87f5-89e528cd6884",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = T5ForConditionalGeneration.from_pretrained(MODEL)\n",
    "for name, param in model.named_parameters():\n",
    "    if name not in trainable_params: # choose whatever you like here\n",
    "        param.requires_grad = False\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(f\"Device is {device}\")\n",
    "model.to(device)\n",
    "total_params = sum(p.numel() for p in model.parameters())\n",
    "print(f\"{total_params:,} total parameters.\")\n",
    "total_trainable_params = sum(\n",
    "    p.numel() for p in model.parameters() if p.requires_grad)\n",
    "print(f\"{total_trainable_params:,} training parameters.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "545dc208-8245-4e9d-ba2d-7fec7732d8b1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "training_args=TrainingArguments(\n",
    "    output_dir=OUT_DIR,\n",
    "    num_train_epochs=EPOCHS,\n",
    "    per_device_train_batch_size=BATCH_SIZE,\n",
    "    per_device_eval_batch_size=BATCH_SIZE,\n",
    "    warmup_steps=500,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir=OUT_DIR,\n",
    "    logging_steps=10,\n",
    "    evaluation_strategy='steps',\n",
    "    save_steps=500,\n",
    "    eval_steps=500,\n",
    "    load_best_model_at_end=True,\n",
    "    save_total_limit=5,\n",
    "    report_to='tensorboard',\n",
    "    learning_rate=0.0001,\n",
    "    fp16=True,\n",
    "    dataloader_num_workers=4\n",
    ")\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_train,\n",
    "    eval_dataset=tokenized_valid\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68819bc2-1e48-4da3-a655-bb10789abcd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc75529c-c93d-479a-abc0-48675e1e6270",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import T5ForConditionalGeneration, T5Tokenizer\n",
    "\n",
    "model_path = './results_t5small/checkpoint-4000'\n",
    "model = T5ForConditionalGeneration.from_pretrained(model_path)\n",
    "#tokenizer = T5Tokenizer.from_pretrained('results_t5small')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cee51e18-6347-4624-a91a-157a6c0e162d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def do_correction(text, model, tokenizer):\n",
    "    input_text = f\"assign tag: {text}\"\n",
    "    inputs= tokenizer.encode(\n",
    "        input_text,\n",
    "        return_tensors='pt',\n",
    "        max_length=256,\n",
    "        padding='max_length',\n",
    "        truncation=True\n",
    "    )\n",
    "    corrected_ids = model.generate(\n",
    "        inputs,\n",
    "        max_length=256,\n",
    "        num_beams=5, # `num_beams=1` indicated temperature sampling.\n",
    "        early_stopping=True\n",
    "    )\n",
    "    corrected_sentence = tokenizer.decode(\n",
    "        corrected_ids[0],\n",
    "        skip_special_tokens=True\n",
    "    )\n",
    "    return corrected_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ec23dcc-93a0-4557-b047-acd2fd2a3b08",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "for file in os.listdir('inference_data/'):\n",
    "    f = open(f\"inference_data/{file}\", 'r')\n",
    "    sentence = f.read()\n",
    "    corrected_sentence = do_correction(sentence, model, tokenizer)\n",
    "    print(f\"QUERY: {sentence}\\nTAGS: {corrected_sentence}\")\n",
    "    print('-'*80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07288047-b81f-4b8f-afae-8a4a972f20d2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}