3_Training_Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9381fe14-9ae7-4578-9131-1a3b53aae83e",
   "metadata": {},
   "source": [
    "# End to end process for adding both entity ruler and word vectors to an NER model\n",
    "1. Document cleaning and splitting the corpus into test and train sets\n",
    "2. Build word vectors\n",
    "3. Build training data with entity ruler and split into train and validation data\n",
    "4. Add word vectors to model, run"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2bc8e48-cec1-49a1-9004-af973c9319be",
   "metadata": {},
   "source": [
    "## Notebook 3\n",
    "- Load training data\n",
    "- Process to create training entities\n",
    "    - Build entity ruler patterns\n",
    "    - Build training data\n",
    "- Convert to spaCy 3 format\n",
    "    - Save training and validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ba836013-2b32-42d0-938f-373ed7e3111e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#import, save data\n",
    "import json\n",
    "\n",
    "#build entity ruler\n",
    "import spacy\n",
    "\n",
    "#convert to training format\n",
    "from spacy.tokens import DocBin\n",
    "\n",
    "import srsly\n",
    "import typer\n",
    "import warnings\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30d33ff2-7174-4b9d-8345-68c24dd133f9",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "676652ac-9407-4eb5-a65d-05d6618377e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import training data\n",
    "def load_data(file):\n",
    "    with open(file, 'r', encoding='utf-8') as f:\n",
    "        data = json.load(f)\n",
    "    return(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a6ac241a-8655-4633-abd5-b71bb5aa18b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "files = load_data('data/sw_train_ner.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a0d3ed36-f373-47aa-82da-ead867923ecf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Corran asked', 'necessarily Luke realm possibility either', 'use dueling blasters ranged weapons', 'Usually Deputy Director led morning meetings', 'Wedge gave light slap arm', 'turned Leia', 'Yeah', 'finally caught', 'Kyp put shuttle Corran retrieved extra tool kit pulled shuttle left', '']\n"
     ]
    }
   ],
   "source": [
    "print(files[5:15])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e43100be-01bd-4aa7-97ab-e690143b7920",
   "metadata": {},
   "source": [
    "### Create entity ruler and add pipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5cbb4b2f-d4a0-4ffc-b0a6-3f1f2f868f0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_md')\n",
    "#nlp.add_pipe('ner')\n",
    "ruler = nlp.add_pipe('entity_ruler', before='ner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6788e8f9-f80e-4cc8-ac31-4649831f8cdc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['tok2vec', 'tagger', 'parser', 'attribute_ruler', 'lemmatizer', 'entity_ruler', 'ner']\n"
     ]
    }
   ],
   "source": [
    "print(nlp.pipe_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e92877a5-54c4-4464-b6ff-7285f5d3d966",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Lists of entities and patterns\n",
    "patterns = [\n",
    "            {'label':'PERSON', 'pattern':'Jaina'},\n",
    "            {'label':'PERSON', 'pattern':'Wedge'},\n",
    "            {'label':'PERSON', 'pattern':'Wedge Antilles'},\n",
    "            {'label':'PERSON', 'pattern':'Corran'},\n",
    "            {'label':'PERSON', 'pattern':'Corran Horn'},\n",
    "            {'label':'PERSON', 'pattern':'Iella'},\n",
    "            {'label':'PERSON', 'pattern':'Iella Wessiri'},\n",
    "            {'label':'PERSON', 'pattern':'Jacen'},\n",
    "            {'label':'PERSON', 'pattern':\"Brianna\"},\n",
    "            {'label':'PERSON', 'pattern':\"Sirren Choth\"},\n",
    "            {'label':'PERSON', 'pattern':\"Sirren\"},\n",
    "            {'label':'PERSON', 'pattern':\"Sir ren\"},\n",
    "            {'label':'PERSON', 'pattern':'Anakin'},\n",
    "            {'label':'PERSON', 'pattern':'Cami'},\n",
    "            {'label':'PERSON', 'pattern':'Camie'},\n",
    "            {'label':'PERSON', 'pattern':'Valah'},\n",
    "            {'label':'PERSON', 'pattern':'Leia'},\n",
    "            {'label':'PERSON', 'pattern':'Salta'},\n",
    "            {'label':'PERSON', 'pattern':'Han'},\n",
    "            {'label':'PERSON', 'pattern':'Kyp'},\n",
    "            {'label':'PERSON', 'pattern':'Mirax'},\n",
    "            {'label':'PERSON', 'pattern':'Cal'},\n",
    "            {'label':'PERSON', 'pattern':'Hobbie'},\n",
    "            {'label':'PERSON', 'pattern':'Mourtos'},\n",
    "            {'label':'PERSON', 'pattern':'Winter'},\n",
    "            {'label':'PERSON', 'pattern':'Booster'},\n",
    "            {'label':'PERSON', 'pattern':'Visalia'},\n",
    "            {'label':'PERSON', 'pattern':\"Borsk Feylya\"},\n",
    "            {'label':'TITLE', 'pattern':\"Director\"},\n",
    "            {'label':'TITLE', 'pattern':\"Assistant Director\"},\n",
    "            {'label':'TITLE', 'pattern':\"Deputy Assistant Director\"},\n",
    "            {'label':'TITLE', 'pattern':\"Deputy\"},\n",
    "            {'label':'TITLE', 'pattern':\"Lieutenant\"},\n",
    "            {'label':'TITLE', 'pattern':\"General\"},\n",
    "            {'label':'TITLE', 'pattern':\"Councilor\"},\n",
    "            {'label':'TITLE', 'pattern':\"Moff\"},\n",
    "            {'label':'TITLE', 'pattern':\"Commander\"},\n",
    "            {'label':'TITLE', 'pattern':\"Captain\"},\n",
    "            {'label':'TITLE', 'pattern':\"Chief\"},\n",
    "            {'label':'TITLE', 'pattern':\"Assistant Chief\"},\n",
    "            {'label':'TITLE', 'pattern':\"AD\"},\n",
    "            {'label':'TITLE', 'pattern':\"Master\"},\n",
    "            {'label':'TITLE', 'pattern':\"Master Jedi\"},\n",
    "            {'label':'LOC', 'pattern':'Coruscant'},\n",
    "            {'label':'LOC', 'pattern':'Alderaan'},\n",
    "            {'label':'LOC', 'pattern':'Yavin'},\n",
    "            {'label':'LOC', 'pattern':'Jedah City'},\n",
    "            {'label':'LOC', 'pattern':'Tatooine'},\n",
    "            {'label':'LOC', 'pattern':'Formuth'},\n",
    "            {'label':'ORG', 'pattern':'NRDI'},\n",
    "            {'label':'ORG', 'pattern':'New Republic Directorate of Intelligence'},\n",
    "            {'label':'ORG', 'pattern':'Empire'},\n",
    "            {'label':'ORG', 'pattern':'Vir Shat'},\n",
    "            {'label':'ORG', 'pattern':'Vir Azmun'},\n",
    "            {'label':'ORG', 'pattern':'Sith'},\n",
    "            {'label':'ORG', 'pattern':'Jedi'},\n",
    "            {'label':'ORG', 'pattern':'Jedi Order'},\n",
    "            {'label':'PERSON', 'pattern':\"Vir Sun\"},\n",
    "            {'label':'LOC', 'pattern':'Charapath'},\n",
    "            {'label':'LOC', 'pattern':'Xalos'},\n",
    "            {'label':'LOC', 'pattern':'Dagobah'},\n",
    "            {'label':'LOC', 'pattern':'Hoth'},\n",
    "            {'label':'LOC', 'pattern':'Bakura'},\n",
    "            {'label':'LOC', 'pattern':'Scarif'},\n",
    "            {'label':'PERSON', 'pattern':'Tycho'},\n",
    "            {'label':'FORCE', 'pattern':'Force'},\n",
    "            {'label':'SPECIES', 'pattern':'Duros'},\n",
    "            {'label':'SPECIES', 'pattern':'Yuzong Vong'},\n",
    "            {'label':'SPECIES', 'pattern':'Utapaun'},\n",
    "            {'label':'SPECIES', 'pattern':'Bothan'},\n",
    "            {'label':'SPECIES', 'pattern':\"Twilek\"},\n",
    "            {'label':'SPECIES', 'pattern':\"Elomin\"},\n",
    "            {'label':'PRODUCT', 'pattern':'Holobook'},\n",
    "            {'label':'PRODUCT', 'pattern':'HoloBook'},\n",
    "            {'label':'PRODUCT', 'pattern':'HoloNet'},\n",
    "            {'label':'FORM', 'pattern':'Soresu'},\n",
    "            {'label':'FORM', 'pattern':'Makashi'},\n",
    "            {'label':'DROID', 'pattern':'R2'},\n",
    "            {'label':'DROID', 'pattern':'Whistler'},\n",
    "            {'label':'DROID', 'pattern':'3PO'},\n",
    "            {'label':'DROID', 'pattern':'Threepio'},\n",
    "            {'label':'ORG', 'pattern':\"Vir Teng\"},\n",
    "            {'label':'PERSON', 'pattern':'Dom'},\n",
    "            {'label':'SHIP', 'pattern':'Falcon'},\n",
    "            {'label':'SHIP', 'pattern':'X Wing'},\n",
    "            {'label':'SHIP', 'pattern':'Y Wings'},\n",
    "            {'label':'SHIP', 'pattern':'TIE fighter'},\n",
    "            {'label':'SHIP', 'pattern':'Errant Venture'},\n",
    "            {'label':'SHIP', 'pattern':'Pulsar Skate'},\n",
    "            {'label':'PERSON', 'pattern':'Melki'},\n",
    "            {'label':'FORM', 'pattern':'Form IV'},\n",
    "            {'label':'FORM', 'pattern':'Form II'},\n",
    "            {'label':'FORM', 'pattern':'Form III'}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5a3986d2-b307-4559-9c95-e9654b512379",
   "metadata": {},
   "outputs": [],
   "source": [
    "ruler.add_patterns(patterns)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5cc8c60-f18c-4c2c-9b06-c0a073f63787",
   "metadata": {},
   "source": [
    "### Put into spaCy format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bad28af3-49d2-4862-b308-72d99fcc6243",
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_DATA = []\n",
    "for sentence in files:\n",
    "    doc = nlp(sentence)\n",
    "    entities = []\n",
    "    for ent in doc.ents:\n",
    "        entities.append([ent.start_char, ent.end_char, ent.label_])\n",
    "    TRAIN_DATA.append([sentence, {'entities':entities}])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "818a9bb0-6ebe-4d7a-a965-95a4d829f7de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['Corran asked', {'entities': [[0, 6, 'PERSON']]}], ['necessarily Luke realm possibility either', {'entities': [[12, 16, 'PERSON']]}], ['use dueling blasters ranged weapons', {'entities': []}], ['Usually Deputy Director led morning meetings', {'entities': [[8, 14, 'TITLE'], [15, 23, 'TITLE'], [28, 35, 'TIME']]}], ['Wedge gave light slap arm', {'entities': [[0, 5, 'PERSON']]}], ['turned Leia', {'entities': [[7, 11, 'PERSON']]}], ['Yeah', {'entities': []}], ['finally caught', {'entities': []}], ['Kyp put shuttle Corran retrieved extra tool kit pulled shuttle left', {'entities': [[0, 3, 'PERSON'], [16, 22, 'PERSON']]}], ['', {'entities': []}]]\n"
     ]
    }
   ],
   "source": [
    "print(TRAIN_DATA[5:15])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "720b7421-dbb9-4d37-949f-ab31150a5e39",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16560\n"
     ]
    }
   ],
   "source": [
    "print(len(TRAIN_DATA))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c310e62-0ba8-4ebc-92f7-2fa0fa723e56",
   "metadata": {},
   "source": [
    "### Save training data as json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "58342850-5833-4a03-bfb7-340ae8bcacdd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_data(file, data):\n",
    "    with open(file, 'w', encoding='utf-8') as f:\n",
    "        json.dump(data, f, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "45fc953e-f49b-4807-bf58-3e5cc9d5d2d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_data('./data/sw_training_data.json', TRAIN_DATA[:8280])\n",
    "save_data('./data/sw_valid_data.json', TRAIN_DATA[8280:])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d741f80-3bc9-448f-a64e-457f63cbdcf9",
   "metadata": {},
   "source": [
    "### Covert to spaCy 3 format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d952c911-fb10-40f5-bd83-a89f5d8776b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(file):\n",
    "    with open(file, 'r', encoding='utf-8') as f:\n",
    "        data = json.load(f)\n",
    "    return(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e2727a94-ef38-4394-84c9-653ed31fb3a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = load_data('./data/sw_training_data.json')\n",
    "valid_data = load_data('./data/sw_valid_data.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "efbd0909-007a-4e58-9568-15817e6c956b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert(lang:str, TRAIN_DATA, output_path: Path):\n",
    "    nlp = spacy.blank(lang)\n",
    "    db = DocBin()\n",
    "    for text, annot in TRAIN_DATA:\n",
    "        doc = nlp.make_doc(text)\n",
    "        ents = []\n",
    "        for start, end, label in annot['entities']:\n",
    "            span = doc.char_span(start, end, label=label)\n",
    "            if span is None:\n",
    "                msg = f\"Skipping entity [{start}, {end}, {label}] in the following text because the character span '{doc.text[start:end]}' does not align with token boundaries:\\n\\n{repr(text)}\\n\"\n",
    "                warnings.warn(msg)\n",
    "            else:\n",
    "                ents.append(span)\n",
    "        doc.ents = ents\n",
    "        db.add(doc)\n",
    "    db.to_disk(output_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c81c1c03-2d38-47be-b9f2-c6957b4b8f09",
   "metadata": {},
   "outputs": [],
   "source": [
    "convert('en', train_data, 'data/train.spacy')\n",
    "convert('en', valid_data, 'data/valid.spacy')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}