T4-1-2-NER-OASIS-metadata-records.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process OASIS records\n",
    "Vocabulary-based Named Entity Recognition (NER) applied to a set of XML OASIS abstracts obtained from ADS. Detecting temporal phrases and object/monument types."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "# install required dependencies\n",
    "%pip install --upgrade pip\n",
    "%pip install spacy\n",
    "%pip install ipywidgets\n",
    "%sx python -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reference required modules\n",
    "#from IPython.display import display, HTML\n",
    "from slugify import slugify # for creating valid filenames from identifiers\n",
    "import spacy # for NER processing\n",
    "from spacy.tokens import Doc # for NER results\n",
    "from lxml import etree as ET # for parsing input records from XML file\n",
    "from datetime import datetime as DT # for timestamps\n",
    "from html import escape # for writing escaped text within HTML \n",
    "import pandas as pd  # for DataFrame\n",
    "import os\n",
    "from rematch2 import PeriodoRuler, GeoNamesRuler, VocabularyRuler, NegationRuler, DocSummary, StringCleaning, child_span_remover\n",
    "from rematch2.spacypatterns import patterns_en_ATTRIBUTE_RULES # rules to override POS tags in some cases\n",
    "import warnings\n",
    "from pprint import pprint\n",
    "# suppress user warnings during execution\n",
    "warnings.filterwarnings(action='ignore', category=UserWarning)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "# parse and extract a list of OASIS abstract records from source XML file \n",
    "# returns [{\"id\", \"text\"}, {\"id\", \"text\"}, ...] for subsequent processing\n",
    "def get_records_from_xml_file(file_path: str=\"\") -> list:\n",
    "    records = []\n",
    "    try:\n",
    "        # read XML file\n",
    "        tree = ET.parse(file_path)\n",
    "        root = tree.getroot()\n",
    "    except:\n",
    "        print(f\"Could not read from {file_path}\")\n",
    "        return []\n",
    "\n",
    "    # find rows to be processed in the XML file\n",
    "    rows = tree.xpath(\"/table/rows/row\")\n",
    "\n",
    "    for row in rows:\n",
    "        # find abstract(s) in the current item\n",
    "        abstracts = row.xpath(\"value[@columnNumber='1']/text()\")\n",
    "       \n",
    "        # if multiple abstracts, get first one\n",
    "        if (len(abstracts) > 0):\n",
    "            abstract = abstracts[0]\n",
    "        else:\n",
    "            abstract = \"\"\n",
    "\n",
    "         # find identifier(s) in the current item\n",
    "        identifiers = row.xpath(\"value[@columnNumber='0']/text()\")\n",
    "\n",
    "        # if multiple identifiers, get first one (remove URL prefix if present)\n",
    "        if (len(identifiers) > 0):\n",
    "            identifier = identifiers[0]\n",
    "            identifier = identifier.replace(\n",
    "                \"https://archaeologydataservice.ac.uk/archsearch/record?titleId=\", \"\")\n",
    "        else:\n",
    "            identifier = \"\"\n",
    "\n",
    "        ## create new (cleaned) record and add it\n",
    "        #record = {}\n",
    "        #record[\"id\"] = str(identifier).strip()\n",
    "        #record[\"text\"] = str(abstract).strip()\n",
    "        records.append({\n",
    "            \"id\": str(identifier).strip(),\n",
    "            \"text\": str(abstract).strip()\n",
    "        })\n",
    "\n",
    "    # finally, return the extracted list\n",
    "    return records\n",
    "\n",
    "\n",
    "# parse and extract a list of OASIS abstract records from source CSV file \n",
    "# returns [{\"id\", \"text\"}, {\"id\", \"text\"}, ...] for subsequent processing\n",
    "def get_records_from_csv_file(file_path: str=\"\") -> list:\n",
    "    records = []\n",
    "    \n",
    "    # read the CSV file to a DataFrame\n",
    "    df = pd.read_csv(file_path, skip_blank_lines=True)\n",
    "    # set any NaN values to blank string\n",
    "    df.fillna(\"\", inplace=True)\n",
    "    # convert the data to a dict structure\n",
    "    items = df.to_dict(orient=\"records\") \n",
    "    \n",
    "    records = list(map(lambda item: { \n",
    "            \"id\": str(item.get(\"file\", \"\")).strip(), \n",
    "            \"title\": str(item.get(\"title\", \"\")).strip(), \n",
    "            \"text\": str(item.get(\"abstract\")).strip() \n",
    "        }, items))\n",
    "    \n",
    "    return records\n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "metadata": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing record 1 of 75 [ID: archael547-079-116-ceolwulf.pdf]\n",
      "processing record 2 of 75 [ID: archael547-005-040-breeze.pdf]\n",
      "processing record 3 of 75 [ID: archael547-041-077-jackson.pdf]\n",
      "processing record 4 of 75 [ID: archael547-001-003-croom.pdf]\n",
      "processing record 5 of 75 [ID: archael547-117-154-swann.pdf]\n",
      "processing record 6 of 75 [ID: archael547-223-254-custom_house.pdf]\n",
      "processing record 7 of 75 [ID: archael547-255-270-quilter.pdf]\n",
      "processing record 8 of 75 [ID: archael547-155-199-nolan.pdf]\n",
      "processing record 9 of 75 [ID: archael547-201-221-newman.pdf]\n",
      "processing record 10 of 75 [ID: archael547-271-305-wear_glass.pdf]\n",
      "processing record 11 of 75 [ID: 2022_96_001_012_Cooper_Garton.pdf]\n",
      "processing record 12 of 75 [ID: 2022_96_013-068_Huxley.pdf]\n",
      "processing record 13 of 75 [ID: 2022_96_069-078_Simmonds.pdf]\n",
      "processing record 14 of 75 [ID: 2022_96_079-094_Browning_et_al.pdf]\n",
      "processing record 15 of 75 [ID: 2022_96_095_120_Scott.pdf]\n",
      "processing record 16 of 75 [ID: 2022_96_121-136_Aldred.pdf]\n",
      "processing record 17 of 75 [ID: 2022_96_137-156_Postles.pdf]\n",
      "processing record 18 of 75 [ID: surreyac103_001-021_graham.pdf]\n",
      "processing record 19 of 75 [ID: surreyac103_063-090_lambert.pdf]\n",
      "processing record 20 of 75 [ID: surreyac103_091-172_haslam.pdf]\n",
      "processing record 21 of 75 [ID: surreyac103_173-183_price.pdf]\n",
      "processing record 22 of 75 [ID: surreyac103_185-266_saxby.pdf]\n",
      "processing record 23 of 75 [ID: surreyac103_267-296_butler.pdf]\n",
      "processing record 24 of 75 [ID: surreyac103_297-305_english.pdf]\n",
      "processing record 25 of 75 [ID: surreyac103_307-321_nelson.pdf]\n",
      "processing record 26 of 75 [ID: surreyac103_323-329_english.pdf]\n",
      "processing record 27 of 75 [ID: NAS_20_1985_67-86_Jackson.pdf]\n",
      "processing record 28 of 75 [ID: NAS_20_1985_87-112_Taylor.pdf]\n",
      "processing record 29 of 75 [ID: NAS_20_1985_113-138_Shaw.pdf]\n",
      "processing record 30 of 75 [ID: archael522-001-027-heslop.pdf]\n",
      "processing record 31 of 75 [ID: archael522-029-041-monaghan.pdf]\n",
      "processing record 32 of 75 [ID: archael522-043-066-snape.pdf]\n",
      "processing record 33 of 75 [ID: archael522-067-077-whitworth.pdf]\n",
      "processing record 34 of 75 [ID: archael522-079-084-richardson.pdf]\n",
      "processing record 35 of 75 [ID: archael522-085-151-fraser.pdf]\n",
      "processing record 36 of 75 [ID: archael522-153-184-heslop.pdf]\n",
      "processing record 37 of 75 [ID: archael522-185-217-ryder.pdf]\n",
      "processing record 38 of 75 [ID: archael522-219-233-goodrick.pdf]\n",
      "processing record 39 of 75 [ID: archael522-235-253-linsley.pdf]\n",
      "processing record 40 of 75 [ID: archael522-273-276-notes.pdf]\n",
      "processing record 41 of 75 [ID: 120_001_030.pdf]\n",
      "processing record 42 of 75 [ID: 120_031_097.pdf]\n",
      "processing record 43 of 75 [ID: 120_098_160.pdf]\n",
      "processing record 44 of 75 [ID: 120_161_200.pdf]\n",
      "processing record 45 of 75 [ID: 120_201_214.pdf]\n",
      "processing record 46 of 75 [ID: 120_215_235.pdf]\n",
      "processing record 47 of 75 [ID: 120_236_241.pdf]\n",
      "processing record 48 of 75 [ID: 120_242_254.pdf]\n",
      "processing record 49 of 75 [ID: 078_047_054.pdf]\n",
      "processing record 50 of 75 [ID: 078_174_204.pdf]\n",
      "processing record 51 of 75 [ID: 078_216_226.pdf]\n",
      "processing record 52 of 75 [ID: 078_233_250.pdf]\n",
      "processing record 53 of 75 [ID: 078_264_270.pdf]\n",
      "processing record 54 of 75 [ID: 078_391_396.pdf]\n",
      "processing record 55 of 75 [ID: DAJ_v106_1986_005-017.pdf]\n",
      "processing record 56 of 75 [ID: DAJ_v106_1986_018-100.pdf]\n",
      "processing record 57 of 75 [ID: DAJ_v086_1966_031-053.pdf]\n",
      "processing record 58 of 75 [ID: DAJ_v086_1966_054-069.pdf]\n",
      "processing record 59 of 75 [ID: DAJ_v086_1966_093-098.pdf]\n",
      "processing record 60 of 75 [ID: DAJ_v086_1966_099-101.pdf]\n",
      "processing record 61 of 75 [ID: DAJ_v086_1966_102-102.pdf]\n",
      "processing record 62 of 75 [ID: DAJ_v086_1966_103-103.pdf]\n",
      "processing record 63 of 75 [ID: DAJ_v086_1966_106-111.pdf]\n",
      "processing record 64 of 75 [ID: DAJ_v023_1901_040-047.pdf]\n",
      "processing record 65 of 75 [ID: DAJ_v023_1901_090-098.pdf]\n",
      "processing record 66 of 75 [ID: DAJ_v023_1901_099-104.pdf]\n",
      "processing record 67 of 75 [ID: DAJ_v023_1901_105-107.pdf]\n",
      "processing record 68 of 75 [ID: DAJ_v023_1901_108-114.pdf]\n",
      "processing record 69 of 75 [ID: SAC_v115pdfa.pdf]\n",
      "processing record 70 of 75 [ID: SAC118_Garton.pdf]\n",
      "processing record 71 of 75 [ID: SAC118_Bedwin.pdf]\n",
      "processing record 72 of 75 [ID: SAC118_Barr-Hamilton.pdf]\n",
      "processing record 73 of 75 [ID: SAC118_Redknapp_and_Millett.pdf]\n",
      "processing record 74 of 75 [ID: SAC118_Stevens.pdf]\n",
      "processing record 75 of 75 [ID: SAC118_Freke2.pdf]\n"
     ]
    }
   ],
   "source": [
    "# using predefined spaCy pipeline (English), no NER\n",
    "#nlp = spacy.load(\"en_core_web_sm\", disable = ['ner'])\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "# adding custom rules to override default POS tagging for specific cases\n",
    "# NOTE: adding rules to existing attribute_ruler component didn't seem to\n",
    "# work, so inserting another one directly after it and adding rules to that\n",
    "#nlp.get_pipe(\"attribute_ruler\").add_patterns(patterns_en_ATTRIBUTE_RULES)\n",
    "ar = nlp.add_pipe(\"attribute_ruler\", name=\"custom_attribute_ruler\", after=\"attribute_ruler\")\n",
    "ar.add_patterns(patterns_en_ATTRIBUTE_RULES)\n",
    "#pprint(nlp.get_pipe(\"attribute_ruler2\").patterns[-5:])\n",
    "\n",
    "# using HE Cultural Periods authority\n",
    "periodo_authority_id = \"p0kh9ds\" \n",
    "\n",
    "# add rematch2 NER components (usually to the end of the pipeline)\n",
    "nlp.add_pipe(\"yearspan_ruler\", last=True)    \n",
    "nlp.add_pipe(\"periodo_ruler\", last=True, config={\"periodo_authority_id\": periodo_authority_id}) \n",
    "nlp.add_pipe(\"fish_archobjects_ruler\", last=True)\n",
    "nlp.add_pipe(\"fish_monument_types_ruler\", last=True)  \n",
    "nlp.add_pipe(\"fish_supplementary_ruler\", last=True) \n",
    "nlp.add_pipe(\"geonames_ruler\", last=True, config={\"country_codes\": [\"GB\"]}) \n",
    "nlp.add_pipe(\"negation_ruler\", last=True) \n",
    "nlp.add_pipe(\"child_span_remover\", last=True) \n",
    "\n",
    "# process ADS CSV report examples\n",
    "#input_directory = \"./data/ner-input/oasis-report-metadata\"\n",
    "#input_file_name = \"report_metadata.csv\"\n",
    "#output_directory = f\"./data/ner-output/ner-output-oasis-report-metadata\"\n",
    "\n",
    "# process ADS CSV journal examples\n",
    "input_directory = \"./data/ner-input/ads-journal-metadata\"\n",
    "input_file_name = \"journal_metadata.csv\"\n",
    "output_directory = \"./data/ner-output/ner-output-ads-journal-metadata\"\n",
    "\n",
    "# process ADS XML metadata examples\n",
    "#input_directory = \"./data/ner-input/oasis-descr-examples\"\n",
    "#input_file_name = \"oasis_descr_examples.xml\"\n",
    "#output_directory = f\"./data/ner-output/ner-output-oasis-descr-examples\"\n",
    "\n",
    "# timestamp for use in directory names\n",
    "yyyymmdd = DT.now().strftime('%Y%m%d')\n",
    "\n",
    "# create output file path if it does not already exist\n",
    "output_directory = f\"{output_directory}-{yyyymmdd}\"\n",
    "if not os.path.exists(output_directory):\n",
    "    os.makedirs(output_directory)\n",
    "\n",
    "input_records = []\n",
    "input_file = os.path.join(input_directory, input_file_name)\n",
    "if input_file.lower().endswith(\".xml\"):\n",
    "    input_records = get_records_from_xml_file(input_file)\n",
    "elif input_file.lower().endswith(\".csv\"):\n",
    "    input_records = get_records_from_csv_file(input_file)\n",
    "\n",
    "record_count = len(input_records)\n",
    "\n",
    "metadata = {\n",
    "    \"identifier\": \"\",\n",
    "    \"title\": \"vocabulary-based NER results\",\n",
    "    \"description\": \"vocabulary-based NER annotation on text abstracts\",\n",
    "    \"creator\": \"T4-1-2-NER-OASIS-metadata-records.ipynb\",\n",
    "    \"created\": DT.now().strftime('%Y-%m-%dT%H:%M:%SZ'),\n",
    "    \"periodo_authority_id\": periodo_authority_id,\n",
    "    \"ner_pipeline\": nlp.pipe_names,\n",
    "    \"input_file_name\": input_file_name,\n",
    "    \"input_record_count\": record_count        \n",
    "}\n",
    "\n",
    "current_record = 0\n",
    "for record in input_records or []:\n",
    "    current_record += 1\n",
    "\n",
    "    # get ID from the record\n",
    "    identifier = record.get(\"id\", \"\").strip()   \n",
    "    metadata[\"identifier\"] = identifier\n",
    "\n",
    "    # print progress indicator\n",
    "    print(f\"processing record {current_record} of {record_count} [ID: {identifier}]\")\n",
    "        \n",
    "    # Combine title and main text from the record   \n",
    "    # input_text = \"\\n\".join([record.get(\"title\", \"\") + \".\", record.get(\"text\", \"\")])\n",
    "    # 04/10/24 just process main text don't include title\n",
    "    input_text = record.get(\"text\", \"\")\n",
    "\n",
    "    # normalise input text prior to NER processing\n",
    "    cleaned = StringCleaning.normalize_text(input_text)\n",
    "    if(len(cleaned) > 0):\n",
    "        # perform annotation on (normalized) text\n",
    "        doc = nlp(cleaned)\n",
    "\n",
    "        # (optionally) add any identified place entities to the custom spans        \n",
    "        for ent in filter(lambda e: e.label_ == \"GPE\", doc.ents):\n",
    "            doc.spans[\"rematch\"].append(ent)            \n",
    "\n",
    "        summary = DocSummary(doc, metadata=metadata)\n",
    "       \n",
    "        # build output file names incorporating record identifiers\n",
    "        # slugify identifiers in case of bad characters for file names\n",
    "        html_file_name = os.path.join(output_directory, f\"ner-output-{slugify(identifier)}.html\") \n",
    "        text_file_name = os.path.join(output_directory, f\"ner-output-{slugify(identifier)}.txt\")\n",
    "        json_file_name = os.path.join(output_directory, f\"ner-output-{slugify(identifier)}.json\")\n",
    "        \n",
    "        # write results report to HTML, TEXT and JSON files    \n",
    "        report = summary.report(format=\"html\")      \n",
    "        with open(html_file_name, \"w\") as file:\n",
    "            file.write(report)\n",
    "\n",
    "        report = summary.report(format=\"text\")\n",
    "        with open(text_file_name, \"w\") as file:\n",
    "            file.write(report)\n",
    "                \n",
    "        report = summary.report(format=\"json\")\n",
    "        with open(json_file_name, \"w\") as file:\n",
    "            file.write(report) \n",
    "        \n",
    "        # temp interrupt after a few records (while testing)\n",
    "        #if current_record == 10:\n",
    "            #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "metadata": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfrom IPython.display import display, HTML\\n\\n# build list of results\\ndef result_link(record):\\n    identifier = record[\"id\"] \\n    file_path=f\"https://html-preview.github.io/?url=https://github.com/cbinding/rematch2/blob/main/data/output/{slugify(identifier)}.html\"\\n    return f\"<li><a href=\\'{file_path}\\'>{identifier}</a></li>\" \\nresults = list(map(result_link, input_records or []))\\nresults.sort()\\n#display(HTML(\"<ul>\" + \"\".join(results) + \"</ul>\"))\\nwith open(\"./data/output/results.md\", \"w\") as file:\\n    file.write(\"<ul>\" + \"\".join(results) + \"</ul>\")\\n'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "# build list of results\n",
    "def result_link(record):\n",
    "    identifier = record[\"id\"] \n",
    "    file_path=f\"https://html-preview.github.io/?url=https://github.com/cbinding/rematch2/blob/main/data/output/{slugify(identifier)}.html\"\n",
    "    return f\"<li><a href='{file_path}'>{identifier}</a></li>\" \n",
    "results = list(map(result_link, input_records or []))\n",
    "results.sort()\n",
    "#display(HTML(\"<ul>\" + \"\".join(results) + \"</ul>\"))\n",
    "with open(\"./data/output/results.md\", \"w\") as file:\n",
    "    file.write(\"<ul>\" + \"\".join(results) + \"</ul>\")\n",
    "\"\"\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}