using_rematch2_components.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "import warnings\n",
    "# suppress user warnings during execution\n",
    "warnings.filterwarnings(action='ignore', category=UserWarning)\n",
    "\n",
    "%pip install --upgrade pip\n",
    "%pip install spacy\n",
    "%pip install ipywidgets\n",
    "#%pip install -U jupyter\n",
    "%sx python -m spacy download de_core_news_sm\n",
    "%sx python -m spacy download en_core_web_sm\n",
    "%sx python -m spacy download es_core_news_sm\n",
    "%sx python -m spacy download fr_core_news_sm\n",
    "%sx python -m spacy download it_core_news_sm\n",
    "%sx python -m spacy download nl_core_news_sm\n",
    "%sx python -m spacy download nb_core_news_sm\n",
    "%sx python -m spacy download sv_core_news_sm\n",
    "\n",
    "# suppress user warnings during execution\n",
    "warnings.filterwarnings(action='ignore', category=UserWarning)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introduction\n",
    "rematch2 performs specialised NER focussed on vocabularies and temporal entities, and implements a series of spaCy pipeline components to identify various entity types in free text. These components are used by the TemporalAnnotator class but may also be used directly for more fine-grained control if required. This notebook describes each component with working examples of usage. The following example python script uses the ``periodo_ruler`` component to identify named periods (terms from a specified Perio.do authority) in a given text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"spans\" style=\"line-height: 2.5; direction: ltr\">\n",
       " Eight trenches were excavated across the 0.93 ha Site . Previous archaeological works immediately to the north of the Site in the \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    early\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: moccasin; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        YEARSPAN\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    1990s\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       ", including field walking and targeted excavation , recorded worked flint and a pit possibly dated to the \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Late\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Iron\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Age\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       ", while other investigations in the wider area have identified remains of \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Late\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Bronze\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Age\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       ", \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Early\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Iron\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Age\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "and Saxon date . Evidence for truncation of the deposit sequence was noted across the development area , with the Site area potentially having been levelled and drained prior to the establishment of turf pitches , with topsoil present only as a very thin turf horizon overlying sterile subsoil and brickearth deposits . The evaluation identified a single pit of \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Late\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Bronze\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Age\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "to \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Early\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Iron\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Age\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "date . No other archaeological features were located during the works . \n",
       " </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Using a rematch2 component\n",
    "import spacy\n",
    "from spacy import displacy\n",
    "from rematch2 import DocSummary, PeriodoRuler, YearSpanRuler, child_span_remover\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "# use a predefined spaCy pipeline, disabling the default NER component\n",
    "nlp = spacy.load(\"en_core_web_sm\", disable = ['ner'])\n",
    "# add rematch2 periodo_ruler component to the end of the pipeline\n",
    "# looking for terms from Perio.do authority \"p0kh9ds\" (http://purl.org/heritagedata/schemes/eh_period)\n",
    "nlp.add_pipe(\"yearspan_ruler\", last=True)\n",
    "nlp.add_pipe(\"periodo_ruler\", last=True, config={\n",
    "             \"periodo_authority_id\": \"p0kh9ds\"})\n",
    "nlp.add_pipe(\"child_span_remover\", last=True)\n",
    "\n",
    "# processing example text using the modified pipeline\n",
    "# (text from https://doi.org/10.5284/1100106)\n",
    "txt = '''\n",
    "Eight trenches were excavated across the 0.93ha Site. Previous archaeological works immediately to the north of the \\\n",
    "Site in the early 1990s, including field walking and targeted excavation, recorded worked flint and a pit possibly dated to \\\n",
    "the Late Iron Age, while other investigations in the wider area have identified remains of Late Bronze Age, Early Iron Age and \\\n",
    "Saxon date. Evidence for truncation of the deposit sequence was noted across the development area, with the Site area potentially \\\n",
    "having been levelled and drained prior to the establishment of turf pitches, with topsoil present only as a very thin turf horizon overlying \\\n",
    "sterile subsoil and brickearth deposits. The evaluation identified a single pit of Late Bronze Age to Early Iron Age date. No other \\\n",
    "archaeological features were located during the works.\n",
    "'''\n",
    "doc = nlp(txt)\n",
    "\n",
    "# display highlighted entities in context\n",
    "html = DocSummary(doc).doctext(format=\"html\")\n",
    "display(HTML(html))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The pipeline components contain language-specific patterns allowing you to perform similar processing in other languages. The following example uses the \"periodo_ruler\" component with French NER pipeline and a specific Perio.do authority file identifier (see http://n2t.net/ark:/99152/p02chr4 - \"PACTOLS chronology periods used in DOLIA data, 2021\")."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"spans\" style=\"line-height: 2.5; direction: ltr\">Quelques éléments lithiques \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Würm\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    IV\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "dispersés sont des indicateurs de la période \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    néolithique\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "d' environ \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    fin\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: moccasin; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        YEARSPAN\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    11000\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    -\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    début\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    10000\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    av\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    JC\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       ". Un ensemble de fosses , circonscrit sur une surface de 35 m2 , a livré du mobilier céramique du \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    premier\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    âge\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    du\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Fer\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "( \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Hallstatt\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "C ) et quelques éléments lithiques . Un bâtiment sur poteaux se situe à une distance de 100 m vers l’ ouest . Pour la période de \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    La\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Tène\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    finale\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "et gallo-romaine , l’ ensemble des vestiges fossoyés et bâtis sont concentrés sur une parcelle à la croisée de deux chemins actuels présents sur le cadastre de 1807 . L’ élément structurant majeur est le fossé F6 qui traverse perpendiculairement la parcelle , large de 3 m pour une profondeur de 1,80 m sous la terre arable . Seul un angle de fossé , de nature différente et de plus petite taille , semble participer à cette même organisation du paysage . Un bâtiment de plan rectangulaire , 13 x 9 m , sur fondations de schiste dont deux angles ont été découverts , est orienté de façon identique au fossé F6 . Il est très bien fondé sur une profondeur de 0,70 m avec de gros blocs de schiste . Si quelques rares tessons du Haut-Empire ont été trouvés dans la fondation du bâtiment , le mobilier céramique du colmatage de la zone humide située à 10 m au sud-ouest , est compris \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    entre\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: moccasin; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        YEARSPAN\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    le\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    milieu\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    du\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Ier\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    siècle\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "et le \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    début\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: moccasin; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        YEARSPAN\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    du\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    IIe\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    siècle\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    de\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    notre\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    ère\n",
       "    \n",
       "<span style=\"background: moccasin; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       ". Situé à 45 m plus au sud et parallèle au bâtiment , un fossé rectiligne de 3 m de large pour 1,80 m de profondeur sous la terre arable , scinde l’ espace en deux . Son creusement en V à été comblé en deux temps . La première phase est une sédimentation naturelle qui a piégé quelques tessons protohistoriques et des scories ferreuses dont un culot de forge . Le colmatage supérieur est composé de matériaux issus de la démolition avec de très nombreuses tuiles , des blocs de schiste brut ainsi que quelques tessons de céramique gallo-romaine . De nombreux trous de poteaux et fosses se situent entre ces deux structures majeures . Un angle de fossé dessinant l’ amorce d’ un enclos se développe au sud . Du mobilier \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    La\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Tène\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    finale\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "a également été trouvé dans une fosse située dans cette espace . Un réseau fossoyé se développe à l’ est , très érodé du côté nord . Des fragments de céramique possiblement \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    haut\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: yellow; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        PERIOD\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Moyen\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Âge\n",
       "    \n",
       "<span style=\"background: yellow; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "ont été trouvés dans son comblement </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Using a rematch2 component in another language\n",
    "import spacy\n",
    "#from spacy import displacy\n",
    "from rematch2 import DocSummary, PeriodoRuler, YearSpanRuler, child_span_remover\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "# use predefined (French) spaCy pipeline, disabling the default NER component\n",
    "nlp = spacy.load(\"fr_core_news_sm\", disable = ['ner'])\n",
    "# add rematch2 component(s) to the end of the pipeline\n",
    "nlp.add_pipe(\"yearspan_ruler\", last=True)\n",
    "nlp.add_pipe(\"periodo_ruler\", last=True, config={\"periodo_authority_id\": \"p02chr4\"})\n",
    "nlp.add_pipe(\"child_span_remover\", last=True)\n",
    "# process example text using the modified pipeline \n",
    "doc = nlp(\"Quelques éléments lithiques Würm IV dispersés sont des indicateurs de la période néolithique d'environ fin 11000 - début 10000 av JC. Un ensemble de fosses, circonscrit sur une surface de 35 m2, a livré du mobilier céramique du premier âge du Fer (Hallstatt C) et quelques éléments lithiques. Un bâtiment sur poteaux se situe à une distance de 100 m vers l’ouest. Pour la période de La Tène finale et gallo-romaine, l’ensemble des vestiges fossoyés et bâtis sont concentrés sur une parcelle à la croisée de deux chemins actuels présents sur le cadastre de 1807. L’élément structurant majeur est le fossé F6 qui traverse perpendiculairement la parcelle, large de 3 m pour une profondeur de 1,80 m sous la terre arable. Seul un angle de fossé, de nature différente et de plus petite taille, semble participer à cette même organisation du paysage. Un bâtiment de plan rectangulaire, 13 x 9 m, sur fondations de schiste dont deux angles ont été découverts, est orienté de façon identique au fossé F6. Il est très bien fondé sur une profondeur de 0,70 m avec de gros blocs de schiste. Si quelques rares tessons du Haut-Empire ont été trouvés dans la fondation du bâtiment, le mobilier céramique du colmatage de la zone humide située à 10 m au sud-ouest, est compris entre le milieu du Ier siècle et le début du IIe siècle de notre ère. Situé à 45 m plus au sud et parallèle au bâtiment, un fossé rectiligne de 3 m de large pour 1,80 m de profondeur sous la terre arable, scinde l’espace en deux. Son creusement en V à été comblé en deux temps. La première phase est une sédimentation naturelle qui a piégé quelques tessons protohistoriques et des scories ferreuses dont un culot de forge. Le colmatage supérieur est composé de matériaux issus de la démolition avec de très nombreuses tuiles, des blocs de schiste brut ainsi que quelques tessons de céramique gallo-romaine. De nombreux trous de poteaux et fosses se situent entre ces deux structures majeures. Un angle de fossé dessinant l’amorce d’un enclos se développe au sud. Du mobilier La Tène finale a également été trouvé dans une fosse située dans cette espace. Un réseau fossoyé se développe à l’est, très érodé du côté nord. Des fragments de céramique possiblement haut Moyen Âge ont été trouvés dans son comblement\")\n",
    "# display highlighted entities in context\n",
    "html = DocSummary(doc).doctext(format=\"html\")\n",
    "display(HTML(html))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"spans\" style=\"line-height: 2.5; direction: ltr\">\n",
       " This collection comprises \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    site\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "data ( images , a report , a project database and GIS data ) from an archaeological excavation undertaken by Cotswold Archaeology between January and February 2020 at Lydney B Phase III , Archers Walk , Lydney , Gloucestershire . An area of 0.6 ha was excavated within this phase ( Phase III ) of a wider development area . \n",
       " Aside from three residual flints , none closely datable , the earliest remains comprised a small \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    assemblage\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "of Roman pottery and ceramic \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    building\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "material , also residual and most likely derived from a Roman \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    farmstead\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "found immediately to the north within the Phase II excavation area . A single \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    sherd\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "of Anglo - Saxon grass - tempered pottery was also residual . \n",
       " The earliest \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    features\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       ", which accounted for the majority of the remains on \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    site\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       ", relate to medieval agricultural activity focused within a large \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    enclosure\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       ". There was little to suggest domestic occupation within the \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    site\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       ": the pottery \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    assemblage\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "was modest and well abraded , whilst charred \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    plant\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    remains\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "were sparse , and , as with some metallurgical \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    residues\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       ", point to waste disposal rather than the locations of processing or consumption . A focus of occupation within the Rodley Manor \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    site\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       ", on higher ground 160 m to the north - west , seems likely , with the currently \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    site\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "having lain beyond this and providing agricultural facilities , most likely corrals and \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 77px;\">\n",
       "    pens\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "<span style=\"background: plum; top: 57px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "<span style=\"background: plum; top: 57px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "for livestock . \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    Animal\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "\n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    bone\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "</span>\n",
       "was absent , but the damp , low - lying ground would have been best suited to cattle . An \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 60px;\">\n",
       "    assemblage\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "of medieval \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 111px;\">\n",
       "    coins\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "<span style=\"background: plum; top: 57px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "<span style=\"background: plum; top: 74px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "<span style=\"background: plum; top: 91px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: plum; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "<span style=\"background: plum; top: 57px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "<span style=\"background: plum; top: 74px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "<span style=\"background: plum; top: 91px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       "recovered from the subsoil during a metal detector survey may represent a dispersed \n",
       "<span style=\"font-weight: bold; display: inline-block; position: relative; height: 94px;\">\n",
       "    hoard\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "<span style=\"background: lightblue; top: 57px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "<span style=\"background: plum; top: 74px; height: 4px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "</span>\n",
       "\n",
       "    \n",
       "<span style=\"background: lightblue; top: 40px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "<span style=\"background: lightblue; top: 57px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: lightblue; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_MONUMENT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "<span style=\"background: plum; top: 74px; height: 4px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; left: -1px; width: calc(100% + 2px); position: absolute;\">\n",
       "    <span style=\"background: plum; z-index: 10; color: #000; top: -0.5em; padding: 2px 3px; position: absolute; font-size: 0.6em; font-weight: bold; line-height: 1; border-radius: 3px\">\n",
       "        FISH_OBJECT\n",
       "    </span>\n",
       "</span>\n",
       "\n",
       "\n",
       "</span>\n",
       ". \n",
       " </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Using vocabulary pipeline components\n",
    "import spacy\n",
    "from rematch2.VocabularyRuler import *\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "\n",
    "txt = \"\"\"\n",
    "This collection comprises site data (images, a report, a project database and GIS data) from an archaeological excavation undertaken by Cotswold Archaeology between January and February 2020 at Lydney B Phase III, Archers Walk, Lydney, Gloucestershire. An area of 0.6ha was excavated within this phase (Phase III) of a wider development area.\n",
    "Aside from three residual flints, none closely datable, the earliest remains comprised a small assemblage of Roman pottery and ceramic building material, also residual and most likely derived from a Roman farmstead found immediately to the north within the Phase II excavation area. A single sherd of Anglo-Saxon grass-tempered pottery was also residual.\n",
    "The earliest features, which accounted for the majority of the remains on site, relate to medieval agricultural activity focused within a large enclosure. There was little to suggest domestic occupation within the site: the pottery assemblage was modest and well abraded, whilst charred plant remains were sparse, and, as with some metallurgical residues, point to waste disposal rather than the locations of processing or consumption. A focus of occupation within the Rodley Manor site, on higher ground 160m to the north-west, seems likely, with the currently site having lain beyond this and providing agricultural facilities, most likely corrals and pens for livestock. Animal bone was absent, but the damp, low-lying ground would have been best suited to cattle. An assemblage of medieval coins recovered from the subsoil during a metal detector survey may represent a dispersed hoard.\n",
    "\"\"\"\n",
    "\n",
    "# create pipeline and add one or more custom pipeline components\n",
    "nlp = spacy.load(\"en_core_web_sm\", disable=['ner'])\n",
    "# AAT vocabulary pipeline components\n",
    "#nlp.add_pipe(\"aat_activities_ruler\", last=True)\n",
    "# nlp.add_pipe(\"aat_agents_ruler\", last=True)\n",
    "# nlp.add_pipe(\"aat_associated_concepts_ruler\", last=True)\n",
    "# nlp.add_pipe(\"aat_materials_ruler\", last=True)\n",
    "# nlp.add_pipe(\"aat_objects_ruler\", last=True)\n",
    "# nlp.add_pipe(\"aat_physical_attributes_ruler\", last=True)\n",
    "# nlp.add_pipe(\"aat_styleperiods_ruler\", last=True)\n",
    "# FISH vocabulary pipeline components\n",
    "nlp.add_pipe(\"fish_archobjects_ruler\", last=True)\n",
    "# nlp.add_pipe(\"fish_archsciences_ruler\", last=True)\n",
    "# nlp.add_pipe(\"fish_building_materials_ruler\", last=True)\n",
    "# nlp.add_pipe(\"fish_components_ruler\", last=True)\n",
    "# nlp.add_pipe(\"fish_event_types_ruler\", last=True)\n",
    "# nlp.add_pipe(\"fish_evidence_ruler\", last=True)\n",
    "# nlp.add_pipe(\"fish_maritime_craft_ruler\", last=True)\n",
    "nlp.add_pipe(\"fish_monument_types_ruler\", last=True)\n",
    "# nlp.add_pipe(\"fish_periods_ruler\", last=True)\n",
    "nlp.add_pipe(\"child_span_remover\", last=True)\n",
    "# process example text using the modified pipeline\n",
    "doc = nlp(txt)\n",
    "\n",
    "# display highlighted entities in context\n",
    "html = DocSummary(doc).doctext(format=\"html\")\n",
    "display(HTML(html))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next example uses multiple components (yearspan_ruler and periodo_ruler) run against some pre-defined test text in different languages. A small user interface is included so you can change the selected language and re-run to observe the entities identified. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a144e653b1da4a1da3b3f8b703c9e1cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(Dropdown(description='Language:', index=1, options=(['German', 'de'], ['English', 'en'], ['Span…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b03cb61e5d74f6fbc25bc74b95a4322",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output(layout=Layout(border_bottom='1px solid black', border_left='1px solid black', border_right='1px solid b…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Testing temporal components using any of the supported languages\n",
    "import spacy\n",
    "import json\n",
    "from spacy import displacy\n",
    "from rematch2 import YearSpanRuler, PeriodoRuler\n",
    "\n",
    "#from test_examples_multilingual import test_examples_multilingual\n",
    "import ipywidgets as widgets\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "# load tests from JSON file\n",
    "test_file_path = \"./test_examples_multilingual.json\"\n",
    "with open(test_file_path, \"r\") as f:\n",
    "    test_records = json.load(f)  \n",
    "\n",
    "def run(btn):\n",
    "    language = dropdown_language.value\n",
    "    # get the (first) test present for the chosen language\n",
    "    test = next(filter(lambda test: test[\"language\"] == language, test_records), None)   \n",
    "    if(test):\n",
    "        # use predefined pipeline, disabling the default NER component\n",
    "        nlp = spacy.load(test[\"pipe\"], disable = ['ner'])\n",
    "\n",
    "        # add the required pipeline component(s) to the end of the pipeline\n",
    "        nlp.add_pipe(\"yearspan_ruler\", last=True)\n",
    "        nlp.add_pipe(\"periodo_ruler\", last=True, config={\"periodo_authority_id\": test[\"periodo_authority_id\"]})\n",
    "        #print(nlp.pipe_names)\n",
    "        \n",
    "        # process some example text using the modified pipeline \n",
    "        doc = nlp(test[\"text\"])\n",
    "        \n",
    "        # show highlighted entities in the example text\n",
    "        output.clear_output(wait=True)\n",
    "        with output:\n",
    "            options = {\n",
    "                \"spans_key\": \"custom\",\n",
    "                \"colors\": { \n",
    "                    \"PERIOD\": \"yellow\", \n",
    "                    \"YEARSPAN\": \"moccasin\", \n",
    "                    \"FISH_OBJECT\": \"plum\",\n",
    "                    \"FISH_MONUMENT\": \"plum\",\n",
    "                    \"FISH_ARCHSCIENCE\": \"lightpink\",\n",
    "                    \"FISH_EVIDENCE\": \"aliceblue\",\n",
    "                    \"FISH_MATERIAL\": \"antiquewhite\",\n",
    "                    \"FISH_EVENTTYPE\": \"coral\"\n",
    "                }\n",
    "            } \n",
    "            display(HTML(displacy.render(doc, style=\"span\", options=options)))        \n",
    "    else:\n",
    "        with output:\n",
    "            display(f\"No tests for language '{language_code}'\")\n",
    "\n",
    "# define UI components\n",
    "dropdown_language = widgets.Dropdown(\n",
    "    options = [\n",
    "        [\"German\", \"de\"], \n",
    "        [\"English\", \"en\"], \n",
    "        [\"Spanish\", \"es\"], \n",
    "        [\"French\", \"fr\"],\n",
    "        [\"Italian\", \"it\"], \n",
    "        [\"Dutch\", \"nl\"],\n",
    "        [\"Norwegian\", \"no\"],\n",
    "        [\"Swedish\", \"sv\"] \n",
    "    ],\n",
    "    value=\"en\", \n",
    "    description='Language:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "button_go = widgets.Button(description=\"Go\")\n",
    "input = widgets.HBox([dropdown_language, button_go])\n",
    "output = widgets.Output(layout=widgets.Layout(overflow='scroll', border='1px solid black', height='250px'))\n",
    "# what to do when the button is clicked..\n",
    "button_go.on_click(run)\n",
    "# display the UI components\n",
    "display(input, output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import json\n",
    "import ipywidgets as widgets\n",
    "\n",
    "# tell Python where the local modules are located\n",
    "import sys\n",
    "module_path = os.path.abspath(os.path.join('..', 'src'))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "\n",
    "# import local modules\n",
    "from TemporalRecognizer import TemporalRecognizer\n",
    "from PeriodoData import PeriodoData\n",
    "\n",
    "# modules for UI\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "testdata_file_name = \"../src/test-examples.json\"\n",
    "testData = None \n",
    "with open(testdata_file_name, \"r\") as f:  # what if file doesn't exist?            \n",
    "    testData = json.load(f)   \n",
    "\n",
    "# get list of all Perio.do authorities\n",
    "pd = PeriodoData(False) # create new instance, don't refresh cache\n",
    "authorities = pd.get_authority_list() # returns [{id: \"x\", label: \"label x\"}, {id: \"y\", label: \"label y\"}]\n",
    "authorities_for_dropdown = list(map(lambda x: [x[\"label\"], x[\"id\"]], authorities)) # dropdown requires array of tuples: [[label, id], [label, id]]\n",
    "\n",
    "# define dropdown control for list of Perio.do authorities\n",
    "dropdown_authorities = widgets.Dropdown(\n",
    "    options = authorities_for_dropdown, \n",
    "    value=None, \n",
    "    description='Authority:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "# define dropdown control for list of Perio.do periods\n",
    "dropdown_periods = widgets.Dropdown(\n",
    "    options = [], \n",
    "    value=None, \n",
    "    description='Periods:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "# what to do when selected perio.do authority is changed\n",
    "def on_dropdown_change(*args):\n",
    "    periods = pd.get_period_list(dropdown_authorities.value)\n",
    "    periods_for_dropdown = list(map(lambda item: [item[\"label\"], item[\"id\"]], periods))\n",
    "    dropdown_periods.options = periods_for_dropdown\n",
    "\n",
    "dropdown_authorities.observe(on_dropdown_change, 'value')\n",
    "\n",
    "# define input textarea\n",
    "#example_input_text_en = \"the artefact was medieval or circa 1250-1275 or maybe post medieval? Underlying the modern made ground on the site was a layer covering the entire shaft area. This has been dated to c.1480-1800/1900 and interpreted as a post-medieval cultivation soil. Historic mapping illustrates that the site remained undeveloped through the post medieval period until the mid 19th century, when urban development around the site accelerated and construction of railways in this part of London began. On Gascoigne's 1703 map the site was open ground, the later maps of Rocque in 1746 and Horwood in 1799 show the area was in use as fields and Stanford's map of 1862 depicts the area surrounding Eleanor Street comprising of market gardens. These are all consistent with the archaeological evidence. Underlying the layer were natural terrace gravels. The archaeological fieldwork has demonstrated that remains relating to the Prehistoric, Roman or medieval period have not survived to the modern era, if they were once present on site.\"\n",
    "\n",
    "textarea_input = widgets.Textarea(\n",
    "    value=\"\",\n",
    "    placeholder='Type something',\n",
    "    description='String:',\n",
    "    disabled=False,\n",
    "    layout=widgets.Layout(width='100%', height='250px')    \n",
    ")\n",
    "\n",
    "# define language dropdown\n",
    "dropdown_language = widgets.Dropdown(\n",
    "    options = [\n",
    "        [\"German\", \"de\"], \n",
    "        [\"English\", \"en\"], \n",
    "        [\"Spanish\", \"es\"], \n",
    "        [\"French\", \"fr\"],\n",
    "        [\"Italian\", \"it\"], \n",
    "        [\"Dutch\", \"nl\"],\n",
    "        [\"Norwegian\", \"no\"],\n",
    "        [\"Swedish\", \"sv\"] \n",
    "    ],\n",
    "    value=\"fr\", \n",
    "    description='Language:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "# define checkboxes for entity types\n",
    "data = [\"TEMPORAL\", \"MONUMENT\", \"MATERIAL\", \"EVENTTYPE\", \"ARCHSCIENCE\"]\n",
    "checkboxes = [widgets.Checkbox(value=False, description=label) for label in data]\n",
    "entity_type_choices = widgets.VBox(children=checkboxes)\n",
    "\n",
    "#input_text = \"\"\n",
    "for testItem in testData:\n",
    "        if testItem[\"language\"] == dropdown_language.value:\n",
    "            textarea_input.value = testItem[\"value\"]  \n",
    "            break       \n",
    "\n",
    "# define output format dropdown\n",
    "dropdown_output_format = widgets.Dropdown(\n",
    "    options = [\"html\", \"json\", \"csv\", \"tsv\"], \n",
    "    value=\"html\", \n",
    "    description='Format:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "output = widgets.Output(\n",
    "    layout=widgets.Layout(overflow='scroll', border='1px solid black', height='250px')\n",
    ")\n",
    "\n",
    "def on_button_go_clicked(b):    \n",
    "    input_text = textarea_input.value\n",
    "    language = dropdown_language.value\n",
    "    output_format = dropdown_output_format.value\n",
    "    authority_id = dropdown_authorities.value\n",
    "    tr = TemporalRecognizer(language, authority_id)\n",
    "    entities = tr.get_entities(input_text, output_format)\n",
    "    #formatted = TemporalRecognizer.format_entities(input_text, entities, output_format)\n",
    "    #print(formatted)\n",
    "    output.clear_output(wait=True)\n",
    "    with output:\n",
    "        #pprint(formatted)        \n",
    "        display(HTML(entities))\n",
    "        #display(HTML('<a href=\"http://example.com\">link</a>'))\n",
    "    #display(formatted)\n",
    "        \n",
    "# define 'Go' button and output area\n",
    "button_go = widgets.Button(description=\"Go\")\n",
    "button_go.on_click(on_button_go_clicked)\n",
    "    \n",
    "# display all controls on the screen\n",
    "\n",
    "display(entity_type_choices)\n",
    "display(dropdown_authorities)\n",
    "#display(dropdown_periods)\n",
    "display(dropdown_language)\n",
    "display(dropdown_output_format)\n",
    "display(textarea_input)\n",
    "display(button_go)\n",
    "display(output) \n",
    "#display(HTML('<a href=\"http://example.com\">link</a>'))\n",
    "\"\"\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 64-bit ('3.8.13')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "110cc1dee26208153f2972f08a2ad52b6a56238dc66d48e87fb757ef2996db56"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}