T4-1-2-NER-OASIS-reports-full-text.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running ATRIUM NER pipeline on full text extracted from OASIS PDF reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "import warnings\n",
    "# suppress user warnings during execution\n",
    "warnings.filterwarnings(action='ignore', category=UserWarning)\n",
    "\n",
    "# load required dependencies\n",
    "%pip install --upgrade pip\n",
    "%pip install spacy\n",
    "%pip install ipywidgets\n",
    "%sx python -m spacy download en_core_web_sm\n",
    "\n",
    "import spacy # for NER processing\n",
    "from spacy.tokens import Doc # for NER results\n",
    "from datetime import datetime as DT # for timestamps\n",
    "import os\n",
    "from slugify import slugify # for valid filenames from identifiers\n",
    "from rematch2 import PeriodoRuler, VocabularyRuler, NegationRuler, DocSummary, StringCleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing 'NAS_20_1985_87-112_Taylor_new.txt'\n",
      "processing 'archael547-005-040-breeze_new.txt'\n",
      "processing 'archael547-079-116-ceolwulf_new.txt'\n",
      "processing 'surreyac103_063-090_lambert_new.txt'\n"
     ]
    }
   ],
   "source": [
    "# using predefined spaCy pipeline (English)\n",
    "nlp = spacy.load(\"en_core_web_sm\", disable = ['ner'])\n",
    "\n",
    "# using HE Periods list\n",
    "periodo_authority_id = \"p0kh9ds\" \n",
    "\n",
    "# add rematch2 NER component(s) to the end of the pipeline\n",
    "nlp.add_pipe(\"yearspan_ruler\", last=True)    \n",
    "nlp.add_pipe(\"periodo_ruler\", last=True, config={\"periodo_authority_id\": periodo_authority_id}) \n",
    "nlp.add_pipe(\"fish_archobjects_ruler\", last=True)\n",
    "nlp.add_pipe(\"fish_monument_types_ruler\", last=True)  \n",
    "nlp.add_pipe(\"fish_supplementary_ruler\", last=True) \n",
    "nlp.add_pipe(\"negation_ruler\", last=True) \n",
    "nlp.add_pipe(\"child_span_remover\", last=True) \n",
    "\n",
    "input_directory = \"./data/journals_july_2024/text extraction - new\"\n",
    "\n",
    "# subset of files to process\n",
    "file_names = [\n",
    "    \"archael547-079-116-ceolwulf_new.txt\",\n",
    "    \"archael547-005-040-breeze_new.txt\",\n",
    "    \"2022_96_013-068_huxley_new.txt\",\n",
    "    \"2022_96_001_012_cooper_garton_new.txt\",\n",
    "    \"surreyac103_063-090_lambert_new.txt\",\n",
    "    \"nas_20_1985_67-86_jackson_new.txt\",\n",
    "    \"nas_20_1985_87-112_taylor_new.txt\",\n",
    "    \"daj_v023_1901_040-047_new.txt\",\n",
    "    \"daj_v086_1966_031-053_new.txt\",\n",
    "    \"120_031_097_new.txt\"\n",
    "]\n",
    "\n",
    "# create output file path if it does not already exist\n",
    "yyyymmdd = DT.now().strftime('%Y%m%d')        \n",
    "output_directory = os.path.join(input_directory, f\"ner-output-{yyyymmdd}\")\n",
    "if not os.path.exists(output_directory):\n",
    "    os.makedirs(output_directory)\n",
    "\n",
    "counter = 0\n",
    "for entry in os.scandir(input_directory):        \n",
    "    counter += 1\n",
    "    # temp break for testing\n",
    "    #if counter > 7: \n",
    "       # break\n",
    "    \n",
    "    #if entry.is_file() and entry.name.endswith(\".txt\"): \n",
    "    if entry.is_file() and entry.name.lower() in file_names:    \n",
    "        # print progress indicator\n",
    "        input_file_name = entry.name        \n",
    "        print(f\"processing '{input_file_name}'\")\n",
    "\n",
    "        # read text contents of input file\n",
    "        input_file_text = \"\"\n",
    "        with open(entry.path) as input_file:\n",
    "            input_file_text = input_file.read()\n",
    "\n",
    "        # set up metadata to include in output\n",
    "        metadata = {\n",
    "            \"identifier\": input_file_name,\n",
    "            \"title\": \"vocabulary-based NER results\",\n",
    "            \"description\": \"vocabulary-based NER annotation on report full-text\",\n",
    "            \"creator\": \"T4-1-2-NER-OASIS-reports-full-text.ipynb\",\n",
    "            \"created\": DT.now().strftime('%Y-%m-%dT%H:%M:%SZ'),\n",
    "            \"periodo_authority_id\": periodo_authority_id,\n",
    "            \"ner_pipeline\": nlp.pipe_names,\n",
    "            \"input_file_name\": input_file_name,\n",
    "            \"input_record_count\": 1\n",
    "        }\n",
    "\n",
    "        # normalise input text prior to annotation\n",
    "        clean_file_text = StringCleaning.normalize_text(input_file_text)\n",
    "\n",
    "        # perform annotation on cleaned text    \n",
    "        doc = nlp(clean_file_text)\n",
    "        summary = DocSummary(doc, metadata=metadata)\n",
    "\n",
    "        # write results to text files\n",
    "        html_file_name = os.path.join(output_directory, f\"ner-output-{slugify(input_file_name)}.html\") \n",
    "        text_file_name = os.path.join(output_directory, f\"ner-output-{slugify(input_file_name)}.txt\")\n",
    "        json_file_name = os.path.join(output_directory, f\"ner-output-{slugify(input_file_name)}.json\")\n",
    "        \n",
    "        # note last run took 21 mins for 2 files  \n",
    "        report = summary.report(format=\"html\")      \n",
    "        with open(html_file_name, \"w\") as file:\n",
    "            file.write(report)\n",
    "\n",
    "        report = summary.report(format=\"text\")\n",
    "        with open(text_file_name, \"w\") as file:\n",
    "            file.write(report)\n",
    "             \n",
    "        report = summary.report(format=\"json\")\n",
    "        with open(json_file_name, \"w\") as file:\n",
    "            file.write(report) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}