Pyserini+SciBERT_on_COVID_19_Demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/castorini/anserini-notebooks/blob/master/Pyserini%2BSciBERT_on_COVID_19_Demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vUpmSFflwzR1"
   },
   "source": [
    "# Pyserini Demo on COVID-19 Dataset (Title + Abstract Index) \n",
    "# with HuggingFace Transformers based visualization\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0RtnmBN6A-Oq"
   },
   "source": [
    "This notebook provides a demo on how to get started in searching the [COVID-19 Open Research Dataset](https://pages.semanticscholar.org/coronavirus-research) (release of 2020/03/20) from AI2.\n",
    "In this notebook, we'll be working with the title + abstract index. \n",
    "Specifically, we're not indexing the full text (that'll come later, soon!).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "q93fI77dY9HU"
   },
   "outputs": [],
   "source": [
    "from IPython.core.display import display, HTML"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "f2vNd7bpJlDZ"
   },
   "source": [
    "First, install Python dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "E_lt0-pXJia0"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install pyserini==0.8.1.0\n",
    "!pip install transformers\n",
    "\n",
    "import json\n",
    "import os\n",
    "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-11-openjdk-amd64\"\n",
    "# os.environ[\"JAVA_HOME\"] = \"/Library/Java/JavaVirtualMachines/jdk-13.0.2.jdk/Contents/Home\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xbXmstbGypef"
   },
   "source": [
    "Let's grab the pre-built index:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JT_OJKftdqGP"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!wget https://www.dropbox.com/s/j1epbu4ufunbbzv/lucene-index-covid-2020-03-27.tar.gz\n",
    "!tar xvfz lucene-index-covid-2020-03-27.tar.gz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sanity check of index size (should be 1.3G):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "KVXWA6WS0aqJ",
    "outputId": "3f2cf09f-2891-4433-ec59-21acfba6b345"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.4G\tlucene-index-covid-2020-03-27\n"
     ]
    }
   ],
   "source": [
    "!du -h lucene-index-covid-2020-03-27"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "e98LWXwXSxG1"
   },
   "source": [
    "Let's load BioBERT (https://arxiv.org/abs/1901.08746) from HuggingFace Transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UBlFYHejSr2H"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy\n",
    "from tqdm import tqdm\n",
    "from transformers import *\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UBlFYHejSr2H"
   },
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('monologg/biobert_v1.1_pubmed', do_lower_case=False)\n",
    "model = AutoModel.from_pretrained('monologg/biobert_v1.1_pubmed')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NkD0kKxW9mHP"
   },
   "source": [
    "You can use `pysearch` to search over an index. Here's the basic usage:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "COVID_INDEX = 'lucene-index-covid-2020-03-27/'\n",
    "\n",
    "def show_query(query):\n",
    "    \"\"\"HTML print format for the searched query\"\"\"\n",
    "    return HTML('<br/><div style=\"font-family: Times New Roman; font-size: 20px;'\n",
    "                'padding-bottom:12px\"><b>Query</b>: '+query+'</div>')\n",
    "\n",
    "def show_document(idx, doc):\n",
    "    \"\"\"HTML print format for document fields\"\"\"\n",
    "    have_body_text = 'body_text' in json.loads(doc.raw)\n",
    "    body_text = ' Full text available.' if have_body_text else ''\n",
    "    return HTML('<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\">' + \n",
    "               f'<b>Document {idx}:</b> {doc.docid} ({doc.score:1.2f}) -- ' +\n",
    "               f'{doc.lucene_document.get(\"authors\")} et al. ' +\n",
    "             # f'{doc.lucene_document.get(\"journal\")}. ' +\n",
    "             # f'{doc.lucene_document.get(\"publish_time\")}. ' +\n",
    "               f'{doc.lucene_document.get(\"title\")}. ' +\n",
    "               f'<a href=\"https://doi.org/{doc.lucene_document.get(\"doi\")}\">{doc.lucene_document.get(\"doi\")}</a>.'\n",
    "               + f'{body_text}</div>')\n",
    "\n",
    "def show_query_results(query, searcher, top_k=10):\n",
    "    \"\"\"HTML print format for the searched query\"\"\"\n",
    "    hits = searcher.search(query)\n",
    "    display(show_query(query))\n",
    "    for i, hit in enumerate(hits[:top_k]):\n",
    "        display(show_document(i+1, hit))\n",
    "    return hits[:top_k]   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 439
    },
    "colab_type": "code",
    "id": "yFZlcqEX0t1f",
    "outputId": "2dfd5af9-00fe-4a6c-95f2-d093c9dabea7"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<br/><div style=\"font-family: Times New Roman; font-size: 20px;padding-bottom:12px\"><b>Query</b>: these differences reside in the molecular structure of spike proteins and some other factors. Which receptor combination(s) will cause maximum harm</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 1:</b> 14203 (13.40) -- Daniel Wrapp et al. The 3.1-Angstrom Cryo-electron Microscopy Structure of the Porcine Epidemic Diarrhea Virus Spike Protein in the Prefusion Conformation. <a href=\"https://doi.org/10.1128/jvi.00923-19\">10.1128/jvi.00923-19</a>. Full text available.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 2:</b> 17064 (13.28) -- M Godet et al. Major receptor-binding and neutralization determinants are located within the same domain of the transmissible gastroenteritis virus (coronavirus) spike protein.. <a href=\"https://doi.org/\"></a>.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 3:</b> 23573 (13.17) -- Robert N. Kirchdoerfer et al. Prefusion structure of a human coronavirus spike protein. <a href=\"https://doi.org/10.1038/nature17200\">10.1038/nature17200</a>.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 4:</b> 16560 (13.08) -- Fandan Meng et al. Bacterial expression of antigenic sites A and D in the spike protein of transmissible gastroenteritis virus and evaluation of their inhibitory effects on viral infection. <a href=\"https://doi.org/10.1007/s11262-011-0637-1\">10.1007/s11262-011-0637-1</a>.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 5:</b> 22297 (12.46) -- Wuze Ren et al. Difference in Receptor Usage between Severe Acute Respiratory Syndrome (SARS) Coronavirus and SARS-Like Coronavirus of Bat Origin. <a href=\"https://doi.org/10.1128/jvi.01085-07\">10.1128/jvi.01085-07</a>.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 6:</b> 10655 (12.45) -- Ravindra Kumar et al. Prediction of endoplasmic reticulum resident proteins using fragmented amino acid composition and support vector machine. <a href=\"https://doi.org/10.7717/peerj.3561\">10.7717/peerj.3561</a>. Full text available.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 7:</b> 19793 (12.44) -- Jianhua Sui et al. Effects of Human Anti-Spike Protein Receptor Binding Domain Antibodies on Severe Acute Respiratory Syndrome Coronavirus Neutralization Escape and Fitness. <a href=\"https://doi.org/10.1128/jvi.02232-14\">10.1128/jvi.02232-14</a>.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 8:</b> 12085 (12.36) -- Wenfei Song et al. Cryo-EM structure of the SARS coronavirus spike glycoprotein in complex with its host cell receptor ACE2. <a href=\"https://doi.org/10.1371/journal.ppat.1007236\">10.1371/journal.ppat.1007236</a>. Full text available.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 9:</b> 22472 (12.34) -- Lubna Kazi et al. Expression of Hemagglutinin Esterase Protein from Recombinant Mouse Hepatitis Virus Enhances Neurovirulence. <a href=\"https://doi.org/10.1128/jvi.79.24.15064-15073.2005\">10.1128/jvi.79.24.15064-15073.2005</a>.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 10:</b> 1451 (12.32) -- Swatantra Kumar et al. Structural, glycosylation and antigenic variation between 2019 novel coronavirus (2019-nCoV) and SARS coronavirus (SARS-CoV). <a href=\"https://doi.org/10.1007/s13337-020-00571-5\">10.1007/s13337-020-00571-5</a>.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyserini.search import pysearch\n",
    "\n",
    "searcher = pysearch.SimpleSearcher(COVID_INDEX)\n",
    "query = ('these differences reside in the molecular structure of spike proteins and some other factors. Which receptor combination(s) will cause maximum harm')\n",
    "hits = show_query_results(query, searcher, top_k=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iiXaE-TDH7uc"
   },
   "source": [
    "From the hits array, use `.lucene_document` to access the underlying indexed Lucene `Document`, and from there, call `.get(field)` to fetch specific fields, like \"title\", \"doc\", etc.\n",
    "The complete list of available fields is [here](https://github.com/castorini/anserini/blob/master/src/main/java/io/anserini/index/generator/CovidGenerator.java#L46)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7Hbca-fTUDC-"
   },
   "source": [
    "Let's extract contextualized vectors of queries and abstracts from SciBERT for highlighting relevant paragraphs."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AWUg03EjXF-1"
   },
   "source": [
    "First, extract the contextualized vectors of the query above:\n",
    "\n",
    "$$q_1, \\ldots, q_T = \\text{SciBERT}(\\text{query})$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hWmb-SWKTcu1"
   },
   "outputs": [],
   "source": [
    "def extract_scibert(text, tokenizer, model):\n",
    "    text_ids = torch.tensor([tokenizer.encode(text, add_special_tokens=True)])\n",
    "    text_words = tokenizer.convert_ids_to_tokens(text_ids[0])[1:-1]\n",
    "\n",
    "    n_chunks = int(numpy.ceil(float(text_ids.size(1))/510))\n",
    "    states = []\n",
    "    \n",
    "    for ci in range(n_chunks):\n",
    "        text_ids_ = text_ids[0, 1+ci*510:1+(ci+1)*510]            \n",
    "        text_ids_ = torch.cat([text_ids[0, 0].unsqueeze(0), text_ids_])\n",
    "        if text_ids[0, -1] != text_ids[0, -1]:\n",
    "            text_ids_ = torch.cat([text_ids_, text_ids[0,-1].unsqueeze(0)])\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            state = model(text_ids_.unsqueeze(0))[0]\n",
    "            state = state[:, 1:-1, :]\n",
    "        states.append(state)\n",
    "\n",
    "    state = torch.cat(states, axis=1)\n",
    "    return text_ids, text_words, state[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "e9gFCKohUXNP"
   },
   "outputs": [],
   "source": [
    "query_ids, query_words, query_state = extract_scibert(query, tokenizer, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IcAVwQdCXE_P"
   },
   "source": [
    "Second, let's extract contextualized vectors of all the paragraphs from the hit #7:\n",
    "\n",
    "$$p_1^k, \\ldots, p_{T_k}^k = \\text{SciBERT}(\\text{paragraph}^k)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "d-NG7C1CUbDg",
    "outputId": "9d5db119-9a68-4de0-9ad0-8448ec93768f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 27/27 [00:10<00:00,  2.56it/s]\n"
     ]
    }
   ],
   "source": [
    "ii = 0\n",
    "doc_json = json.loads(hits[ii].raw)\n",
    "\n",
    "paragraph_states = []\n",
    "for par in tqdm(doc_json['body_text']):\n",
    "    state = extract_scibert(par['text'], tokenizer, model)\n",
    "    paragraph_states.append(state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p_bUm3TOXWdY"
   },
   "source": [
    "We then compute the cosine similarity matrix between the query and each paragraph:\n",
    "\n",
    "$$A^k = [a^k_{ij}] \\in \\mathbb{R}^{|\\text{query}| \\times |\\text{paragraph}^k|},$$\n",
    "\n",
    "where\n",
    "\n",
    "$$a^k_{ij} = \\frac{q_i^\\top p_j^k}{\\| q_i \\| \\| p_j^k \\|}$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "m6oZIW3OUbLv"
   },
   "outputs": [],
   "source": [
    "def cross_match(state1, state2):\n",
    "    state1 = state1 / torch.sqrt((state1 ** 2).sum(1, keepdims=True))\n",
    "    state2 = state2 / torch.sqrt((state2 ** 2).sum(1, keepdims=True))\n",
    "    sim = (state1.unsqueeze(1) * state2.unsqueeze(0)).sum(-1)\n",
    "    return sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "MK8Na3sGUbSG",
    "outputId": "b4cee510-a39d-4a44-f207-529a3f611673"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "27it [00:00, 92.87it/s]\n"
     ]
    }
   ],
   "source": [
    "sim_matrices = []\n",
    "for pid, par in tqdm(enumerate(doc_json['body_text'])):\n",
    "    sim_score = cross_match(query_state, paragraph_states[pid][-1])\n",
    "    sim_matrices.append(sim_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oRKhDGzdXYr4"
   },
   "source": [
    "Let's retrieve the most relevant paragraphs first, where define the top-$M$ most relevant paragraphs as \n",
    "\n",
    "$$\\arg\\text{top-$M$}_{k=1}^K \\max_{i=1,\\ldots,|\\text{query}|} \\max_{j=1,\\ldots, |\\text{paragraph}^k|} A_{ij}^k$$\n",
    "\n",
    "that is, a paragraph with the highly matched words to the query words is considered relevant."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-3hJ1dAAUSWQ"
   },
   "outputs": [],
   "source": [
    "paragraph_relevance = [torch.max(sim).item() for sim in sim_matrices]\n",
    "\n",
    "# Select the index of top 5 paragraphs with highest relevance\n",
    "rel_index = numpy.argsort(paragraph_relevance)[-5:][::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 537
    },
    "colab_type": "code",
    "id": "bQhkuCc1Xp8s",
    "outputId": "43f485ee-2e33-433a-ed71-e1c52fa715f6"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<br/><div style=\"font-family: Times New Roman; font-size: 20px;padding-bottom:12px\"><b>Query</b>: these differences reside in the molecular structure of spike proteins and some other factors. Which receptor combination(s) will cause maximum harm</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 0:</b> 14203 (13.40) -- Daniel Wrapp et al. The 3.1-Angstrom Cryo-electron Microscopy Structure of the Porcine Epidemic Diarrhea Virus Spike Protein in the Prefusion Conformation. <a href=\"https://doi.org/10.1128/jvi.00923-19\">10.1128/jvi.00923-19</a>. Full text available.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px; margin-left: 15px\"><b></b> -- Recent structural characterizations of S proteins from the betacoronaviruses severe acute respiratory syndrome coronavirus ( SARS - CoV ) and MERS - CoV have revealed that the S1 - CTDs from these spikes exist in a dynamic equilibrium between at least two distinct conformations . In one of these conformations , the S1 - CTDs pack tightly against the S2 subunit , making the receptor - binding motifs inaccessible to host cell receptors and neutralizing antibodies . In the alternative conformation , the S1 - CTD hinges away from the spike , such that it no longer associates with S2 and the receptor - binding motifs are no longer occluded ( 30 ) ( 31 ) ( 32 ) . It has been proposed that sequential receptor binding events trap this transient , receptor - accessible conformation and gradually destabilize S , leading to dissociation of S1 , refolding of S2 , and membrane fusion ( 32 , 33 ) . However , in the only alphacoronavirus S structure reported to date , no such S1 - CTD dynamics were reported and all three S1 - CTDs were in a compact , receptor - inaccessible conformation ( 34 ) . Although it is possible that the alphacoronaviruses make use of a triggering mechanism different from that used by the closely related betacoronaviruses , it seems more likely that this transient conformation simply has yet to be observed . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px; margin-left: 15px\"><b>RESULTS</b> -- In an attempt to determine whether the presence of sialic acid could alter the conformation of PEDV S DA , we incubated PEDV CV777 S with an [UNK] - fold molar excess of the sialic acid analog 3 = - sialyllactose . Screening this complex under cryogenic conditions yielded a 10 - Å reconstruction that was indistinguishable from the PEDV CV777 S apo structure ( Fig . 4 ) . These observations are consistent with the theory that host glycans such as sialic acid are likely attachment factors that do not have a dramatic influence on the conformation of the prefusion coronavirus spike ( 41 ) . Several previous studies have described the isolation of PEDV S DA - deletion variants that have arisen spontaneously in animals as well as in cell culture ( 23 , 24 ) . One such strain , PEDV PC177 , was recently used to inoculate piglets in a PEDV challenge model , and it was shown that not only was the primary infection with PEDV PC177 significantly attenuated compared to infection with PEDV CV777 , but also the piglets inoculated with the DA - deletion variant were significantly more susceptible to subsequent reinfection with PEDV CV777 than those that were originally inoculated with PEDV CV777 ( 25 ) . In an attempt to discover a structural explanation for these observations , we expressed and purified residues 1 to 1125 of PEDV PC177 S using the same C - terminal T4 fibritin motif and affinity tags that were used to express and purify PEDV CV777 S . Excluding the DA deletion at the N terminus of the spike , this variant shares 100 % sequence identity with PEDV CV777 S . Because of this deletion , we were able to detect a slight but measurable shift of the PEDV PC177 S SEC elution volume relative to the Residues that are predicted to form [UNK] - strands are colored in yellow , and residues that are predicted to form [UNK] - helices are colored in pink , based on PSIPRED secondary structure prediction . elution volume of PEDV CV777 S ( Fig . 5A ) . The purified PEDV PC177 S was then observed by negative - stain EM ( Fig . 5B ) . The final 3D reconstruction of this S variant has a resolution of [UNK] Å , which was sufficient to show that , other than the absence of DA , the structure of PC177 S does not dramatically differ from that of CV777 S ( Fig . 5C ) . Because the previously reported differences in susceptibility to reinfection cannot be ascribed to any large - scale conformational differences between the spikes , these data suggest that the enhanced protection against reinfection in piglets inoculated with PEDV CV777 is due to a porcine immune response against DA . S1 - CTD dynamics and S1 dissociation . The S1 - CTDs of PEDV S adopt the same arrangement that has been described in the alphacoronaviruses and deltacoronaviruses , wherein the putative receptor - binding loops are tucked against the NTD of the same protomer ( 34 , 38 , 42 ) . Although these loops are present at the apex of the trimer ( Fig . 6 , top panel ; see also Fig . S5 ) , it is unlikely that they would be able to engage a host cell protein receptor without the receptor sterically clashing with both the NTD of the same protomer and the S1 - CTD of the neighboring protomer ( Fig . 6 , bottom panel ) . For the betacoronaviruses SARS - CoV and MERS - CoV , this clash is avoided when the S1 - CTDs undergo a hinge - like movement that repositions the domains above the rest of S1 ( 31 , 32 ) . This conformational change is thought to allow the SARS - CoV and MERS - CoV S proteins to mask their determinants of receptor binding from antibodies in one conformation while making them readily accessible for host cell receptor engagement in another conformation . However , after extensive 3D classification of our PEDV CV777 data set , we were unable to observe any such conformational dynamism of the PEDV S1 - CTDs in the intact PEDV S trimer . Despite this , the inaccessibility of the putative receptor - binding loops in their observed conformation suggests that there is likely to be an equivalent phenomenon in the alphacoronaviruses . The S1 - CTD of the tan protomer is shown in ribbons , and the putative receptor - binding loops are colored red . A potential receptor ( porcine aminopeptidase N [ PDB ID : 4F5C ] ) is shown as a transparent molecular surface , colored blue . This model was generated by aligning the S1 - CTD of PEDV S with the PRCV RBD from the crystal structure of PRCV RBD bound to porcine aminopeptidase N ( PDB ID : 4F5C ) . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px; margin-left: 15px\"><b>RESULTS</b> -- During 2D classification of the PEDV CV777 S cryo - EM data set , roughly 10 % of particles were consistently assigned into classes that resembled a trimeric S1 ring , although these classes lacked the interior density that would correspond to the S2 fusion machinery ( Fig . 7A ) . The presence of these dissociated rings suggests that some small proportion of the purified spike was processed by endogenous trypsin - like proteases , even though SDS - PAGE analysis revealed only a single band corresponding to uncleaved S ( Fig . 2 ) . Similar dissociated S1 rings have been reported during cryo - EM analysis of MERS - CoV S ( 31 ) . Despite the relatively few dissociated PEDV S1 caps that were observed , a 3D reconstruction generated from these particles resulted in a 9 . 4 - Å map that showed evidence of the conformational rearrangement for two of the three S1 - CTDs ( Fig . 7B and C ) . These domains appear to have rotated [UNK] from the 3 - fold axis of symmetry , presumably undergoing a hinge - like motion similar to what has been observed in the betacoronaviruses ( Fig . 7D ) . The two S1 - CTDs that are observed in the \" out \" conformation are stacked directly on their respective SD - 1s such that their receptor - binding loops are now pointed away from the NTD and are presumably readily accessible to engage with a putative host cell receptor without forming any steric clashes . Although further experimentation will be required to obtain high - resolution structures of these transient conformations , these observations provide some of the first insights into the process of alphacoronavirus receptor engagement and triggering . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px; margin-left: 15px\"><b>DISCUSSION</b> -- In contrast to the structure of the NL63 spike ( 34 ) , our structure of PEDV S reveals an alternative DA conformation that would potentially make this domain more accessible to sialylated host cell surface proteins , as opposed to packing against the S2 fusion machinery . Although poor connectivity and a relatively low level of local resolution in this region of our map prevented us from reliably assigning atomic coordinates to the entirety of DA , the map clearly reveals that this domain is located at the apex of the trimer , [UNK] Å away from the DA conformation in NL63 S . One possible explanation for this novel conformation is that we are merely resolving one of several alternative conformations that DA can sample . However , because we observed only a single , discrete conformation of DA in our data set , this explanation seems unlikely . It is also possible that this conformational discrepancy can be explained by the differences in the expression platforms that were used to produce the S proteins . Coronavirus S proteins are densely glycosylated , and differences in N - linked glycan addition and maturation in mammalian cells ( used to produce PEDV S ) and in insect cells ( used to produce NL63 S ) could lead to electrostatic differences or steric hindrances that may result in the conformational change . A third possibility is that DA of PEDV S genuinely adopts a stable , alternative conformation from that of the NL63 S DA . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px; margin-left: 15px\"><b>DISCUSSION</b> -- The presence of these dissociated spike subunits suggests a degree of protein instability that was not readily apparent based on our biochemical purification . While these S1 rings did not hinder our ability to generate a high - resolution cryo - EM map of the intact PEDV S trimer , such heterogeneity would be detrimental to attempts to vaccinate animals with a single , conformationally homogenous immunogen . It is our hope that the structure of PEDV S presented here will make it possible to rationally engineer a vaccine antigen that exhibits enhanced stability in the prefusion conformation . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def show_sections(section, text):\n",
    "    \"\"\"HTML print format for document subsections\"\"\"\n",
    "    return HTML('<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px; margin-left: 15px\">' + \n",
    "        f'<b>{section}</b> -- {text.replace(\" ##\",\"\")} </div>')\n",
    "\n",
    "display(show_query(query))\n",
    "display(show_document(ii, hits[ii]))\n",
    "for ri in numpy.sort(rel_index):\n",
    "    display(show_sections(doc_json[\"body_text\"][ri]['section'], \" \".join(paragraph_states[ri][1])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "d3H0XQ8IZfAN"
   },
   "source": [
    "We want to look at more details by highlighting relevant phrases in each paragraph, where we define relevant phrases for each paragraph as\n",
    "\n",
    "$$\\arg\\text{top-$M$}_{j=1,\\ldots, |\\text{paragraph}^k|} \\max_{i=1,\\ldots,|\\text{query}|} A_{ij}^k$$\n",
    "\n",
    "that is, any word that had a high similarity to each of the query words is considered relevant. given these words, we highlight a window of 10 surrounding each of them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def highlight_paragraph(ptext, rel_words, max_win=10):\n",
    "    para = \"\"\n",
    "    prev_idx = 0\n",
    "    for jj in rel_words:\n",
    "        \n",
    "        if prev_idx > jj:\n",
    "            continue\n",
    "        \n",
    "        found_start = False\n",
    "        for kk in range(jj, prev_idx-1, -1):\n",
    "            if ptext[kk] == \".\" and (ptext[kk+1][0].isupper() or ptext[kk+1][0] == '['):\n",
    "                sent_start = kk\n",
    "                found_start = True\n",
    "                break\n",
    "        if not found_start:\n",
    "            sent_start = prev_idx-1\n",
    "            \n",
    "        found_end = False\n",
    "        for kk in range(jj, len(ptext)-1):\n",
    "            if ptext[kk] == \".\" and (ptext[kk+1][0].isupper() or ptext[kk+1][0] == '['):\n",
    "                sent_end = kk\n",
    "                found_end = True\n",
    "                break\n",
    "                \n",
    "        if not found_end:\n",
    "            if kk >= len(ptext) - 2:\n",
    "                sent_end = len(ptext)\n",
    "            else:\n",
    "                sent_end = jj\n",
    "        \n",
    "        para = para + \" \"\n",
    "        para = para + \" \".join(ptext[prev_idx:sent_start+1])\n",
    "        para = para + \" <font color='blue'>\"\n",
    "        para = para + \" \".join(ptext[sent_start+1:sent_end])\n",
    "        para = para + \"</font> \"\n",
    "        prev_idx = sent_end\n",
    "        \n",
    "    if prev_idx < len(ptext):\n",
    "        para = para + \" \".join(ptext[prev_idx:])\n",
    "\n",
    "    return para"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 647
    },
    "colab_type": "code",
    "id": "ZMxkQ0_PZiOQ",
    "outputId": "784d6f4d-6889-4b08-de67-fb089becd6f8"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<br/><div style=\"font-family: Times New Roman; font-size: 20px;padding-bottom:12px\"><b>Query</b>: these differences reside in the molecular structure of spike proteins and some other factors. Which receptor combination(s) will cause maximum harm</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px\"><b>Document 0:</b> 14203 (13.40) -- Daniel Wrapp et al. The 3.1-Angstrom Cryo-electron Microscopy Structure of the Porcine Epidemic Diarrhea Virus Spike Protein in the Prefusion Conformation. <a href=\"https://doi.org/10.1128/jvi.00923-19\">10.1128/jvi.00923-19</a>. Full text available.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px; margin-left: 15px\"><b></b> --   <font color='blue'>Recent structural characterizations of S proteins from the betacoronaviruses severe acute respiratory syndrome coronavirus ( SARS - CoV ) and MERS - CoV have revealed that the S1 - CTDs from these spikes exist in a dynamic equilibrium between at least two distinct conformations</font>  . In one of these conformations , the S1 - CTDs pack tightly against the S2 subunit , making the receptor - binding motifs inaccessible to host cell receptors and neutralizing antibodies . In the alternative conformation , the S1 - CTD hinges away from the spike , such that it no longer associates with S2 and the receptor - binding motifs are no longer occluded ( 30 ) ( 31 ) ( 32 ) . <font color='blue'>It has been proposed that sequential receptor binding events trap this transient , receptor - accessible conformation and gradually destabilize S , leading to dissociation of S1 , refolding of S2 , and membrane fusion ( 32 , 33 )</font> . However , in the only alphacoronavirus S structure reported to date , no such S1 - CTD dynamics were reported and all three S1 - CTDs were in a compact , receptor - inaccessible conformation ( 34 ) . Although it is possible that the alphacoronaviruses make use of a triggering mechanism different from that used by the closely related betacoronaviruses , it seems more likely that this transient conformation simply has yet to be observed . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px; margin-left: 15px\"><b>RESULTS</b> --  In an attempt to determine whether the presence of sialic acid could alter the conformation of PEDV S DA , we incubated PEDV CV777 S with an [UNK] - fold molar excess of the sialic acid analog 3 = - sialyllactose . Screening this complex under cryogenic conditions yielded a 10 - Å reconstruction that was indistinguishable from the PEDV CV777 S apo structure ( Fig . 4 ) . These observations are consistent with the theory that host glycans such as sialic acid are likely attachment factors that do not have a dramatic influence on the conformation of the prefusion coronavirus spike ( 41 ) . Several previous studies have described the isolation of PEDV S DA - deletion variants that have arisen spontaneously in animals as well as in cell culture ( 23 , 24 ) . One such strain , PEDV PC177 , was recently used to inoculate piglets in a PEDV challenge model , and it was shown that not only was the primary infection with PEDV PC177 significantly attenuated compared to infection with PEDV CV777 , but also the piglets inoculated with the DA - deletion variant were significantly more susceptible to subsequent reinfection with PEDV CV777 than those that were originally inoculated with PEDV CV777 ( 25 ) . In an attempt to discover a structural explanation for these observations , we expressed and purified residues 1 to 1125 of PEDV PC177 S using the same C - terminal T4 fibritin motif and affinity tags that were used to express and purify PEDV CV777 S . Excluding the DA deletion at the N terminus of the spike , this variant shares 100 % sequence identity with PEDV CV777 S . Because of this deletion , we were able to detect a slight but measurable shift of the PEDV PC177 S SEC elution volume relative to the Residues that are predicted to form [UNK] - strands are colored in yellow , and residues that are predicted to form [UNK] - helices are colored in pink , based on PSIPRED secondary structure prediction . elution volume of PEDV CV777 S ( Fig . 5A ) . The purified PEDV PC177 S was then observed by negative - stain EM ( Fig . 5B ) . The final 3D reconstruction of this S variant has a resolution of [UNK] Å , which was sufficient to show that , other than the absence of DA , the structure of PC177 S does not dramatically differ from that of CV777 S ( Fig . 5C ) . Because the previously reported differences in susceptibility to reinfection cannot be ascribed to any large - scale conformational differences between the spikes , these data suggest that the enhanced protection against reinfection in piglets inoculated with PEDV CV777 is due to a porcine immune response against DA . S1 - CTD dynamics and S1 dissociation . The S1 - CTDs of PEDV S adopt the same arrangement that has been described in the alphacoronaviruses and deltacoronaviruses , wherein the putative receptor - binding loops are tucked against the NTD of the same protomer ( 34 , 38 , 42 ) . Although these loops are present at the apex of the trimer ( Fig . 6 , top panel ; see also Fig . S5 ) , it is unlikely that they would be able to engage a host cell protein receptor without the receptor sterically clashing with both the NTD of the same protomer and the S1 - CTD of the neighboring protomer ( Fig . 6 , bottom panel ) . For the betacoronaviruses SARS - CoV and MERS - CoV , this clash is avoided when the S1 - CTDs undergo a hinge - like movement that repositions the domains above the rest of S1 ( 31 , 32 ) . This conformational change is thought to allow the SARS - CoV and MERS - CoV S proteins to mask their determinants of receptor binding from antibodies in one conformation while making them readily accessible for host cell receptor engagement in another conformation . However , after extensive 3D classification of our PEDV CV777 data set , we were unable to observe any such conformational dynamism of the PEDV S1 - CTDs in the intact PEDV S trimer . Despite this , the inaccessibility of the putative receptor - binding loops in their observed conformation suggests that there is likely to be an equivalent phenomenon in the alphacoronaviruses . <font color='blue'>The S1 - CTD of the tan protomer is shown in ribbons , and the putative receptor - binding loops are colored red</font>  . <font color='blue'>A potential receptor ( porcine aminopeptidase N [ PDB ID : 4F5C ] ) is shown as a transparent molecular surface , colored blue</font> . This model was generated by aligning the S1 - CTD of PEDV S with the PRCV RBD from the crystal structure of PRCV RBD bound to porcine aminopeptidase N ( PDB ID : 4F5C ) . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px; margin-left: 15px\"><b>RESULTS</b> --  During 2D classification of the PEDV CV777 S cryo - EM data set , roughly 10 % of particles were consistently assigned into classes that resembled a trimeric S1 ring , although these classes lacked the interior density that would correspond to the S2 fusion machinery ( Fig . 7A ) . The presence of these dissociated rings suggests that some small proportion of the purified spike was processed by endogenous trypsin - like proteases , even though SDS - PAGE analysis revealed only a single band corresponding to uncleaved S ( Fig . 2 ) . Similar dissociated S1 rings have been reported during cryo - EM analysis of MERS - CoV S ( 31 ) . Despite the relatively few dissociated PEDV S1 caps that were observed , a 3D reconstruction generated from these particles resulted in a 9 . 4 - Å map that showed evidence of the conformational rearrangement for two of the three S1 - CTDs ( Fig . 7B and C ) . These domains appear to have rotated [UNK] from the 3 - fold axis of symmetry , presumably undergoing a hinge - like motion similar to what has been observed in the betacoronaviruses ( Fig . 7D ) . <font color='blue'>The two S1 - CTDs that are observed in the \" out \" conformation are stacked directly on their respective SD - 1s such that their receptor - binding loops are now pointed away from the NTD and are presumably readily accessible to engage with a putative host cell receptor without forming any steric clashes</font>  . <font color='blue'>Although further experimentation will be required to obtain high - resolution structures of these transient conformations , these observations provide some of the first insights into the process of alphacoronavirus receptor engagement and triggering .</font>  </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px; margin-left: 15px\"><b>DISCUSSION</b> --  In contrast to the structure of the NL63 spike ( 34 ) , our structure of PEDV S reveals an alternative DA conformation that would potentially make this domain more accessible to sialylated host cell surface proteins , as opposed to packing against the S2 fusion machinery . Although poor connectivity and a relatively low level of local resolution in this region of our map prevented us from reliably assigning atomic coordinates to the entirety of DA , the map clearly reveals that this domain is located at the apex of the trimer , [UNK] Å away from the DA conformation in NL63 S . One possible explanation for this novel conformation is that we are merely resolving one of several alternative conformations that DA can sample . However , because we observed only a single , discrete conformation of DA in our data set , this explanation seems unlikely . <font color='blue'>It is also possible that this conformational discrepancy can be explained by the differences in the expression platforms that were used to produce the S proteins</font>  . <font color='blue'>Coronavirus S proteins are densely glycosylated , and differences in N - linked glycan addition and maturation in mammalian cells ( used to produce PEDV S ) and in insect cells ( used to produce NL63 S ) could lead to electrostatic differences or steric hindrances that may result in the conformational change</font> . A third possibility is that DA of PEDV S genuinely adopts a stable , alternative conformation from that of the NL63 S DA . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"font-family: Times New Roman; font-size: 18px; padding-bottom:10px; margin-left: 15px\"><b>DISCUSSION</b> --   <font color='blue'>The presence of these dissociated spike subunits suggests a degree of protein instability that was not readily apparent based on our biochemical purification</font>  . While these S1 rings did not hinder our ability to generate a high - resolution cryo - EM map of the intact PEDV S trimer , such heterogeneity would be detrimental to attempts to vaccinate animals with a single , conformationally homogenous immunogen . <font color='blue'>It is our hope that the structure of PEDV S presented here will make it possible to rationally engineer a vaccine antigen that exhibits enhanced stability in the prefusion conformation .</font>  </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(show_query(query))\n",
    "\n",
    "display(show_document(ii, hits[ii]))\n",
    "\n",
    "for ri in numpy.sort(rel_index):\n",
    "    sim = sim_matrices[ri].data.numpy()\n",
    "    \n",
    "    # Select the two highest scoring words in the paragraph\n",
    "    rel_words = numpy.sort(numpy.argsort(sim.max(0))[-2:][::-1])\n",
    "    p_tokens = paragraph_states[ri][1]\n",
    "    para = highlight_paragraph(p_tokens, rel_words)\n",
    "    display(show_sections(doc_json[\"body_text\"][ri]['section'], para))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "Pyserini+SciBERT_on_COVID_19_Demo.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "02c8c6601fd74532908897dd6ba00642": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1ef1ac1d6c1e456caa6992593d367352",
      "placeholder": "​",
      "style": "IPY_MODEL_d8178af03865496db91a56d5d3a0cc57",
      "value": " 313/313 [00:01&lt;00:00, 187B/s]"
     }
    },
    "0ef3894860ad41328dd9540076c684bf": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_da75ddbf1b5f40b29c5b3b09a9c3d843",
      "max": 222296,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_31117b7d0d624f50acc63317961c1633",
      "value": 222296
     }
    },
    "1ef1ac1d6c1e456caa6992593d367352": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "31117b7d0d624f50acc63317961c1633": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "3fc159bfdb1b46ea8b6d462f43c42980": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "4e91ba3985ab4306bf68027b01de8e49": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "55f0585b2b9c4872b76215540fe2d010": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "5afdbf46d5c246e98d0fa4fd7a0bd52b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5c40586819754261a141a6b88396e38b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "69b31d9a127846ce9a085c1718361919": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f672e067718a47e9bc1a741bbaa9dd36",
       "IPY_MODEL_02c8c6601fd74532908897dd6ba00642"
      ],
      "layout": "IPY_MODEL_ab609f410fa845ba86388ae0cdd8e308"
     }
    },
    "82b06cc5c5684ef5b6502eb99a6d32ec": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_890c6e99f177467481405dfe2a3fc339",
      "max": 442301670,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_55f0585b2b9c4872b76215540fe2d010",
      "value": 442301670
     }
    },
    "8655da061ea34a16800b111d6e20c69a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "890c6e99f177467481405dfe2a3fc339": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "93d84aebb3c547a9b1398b975e08c60e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4e91ba3985ab4306bf68027b01de8e49",
      "placeholder": "​",
      "style": "IPY_MODEL_94d6f73960f54bfba3897bdb3ff40b49",
      "value": " 442M/442M [00:12&lt;00:00, 35.1MB/s]"
     }
    },
    "94d6f73960f54bfba3897bdb3ff40b49": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9e04d92a35cd4230a1f68332eeb3989c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0ef3894860ad41328dd9540076c684bf",
       "IPY_MODEL_b3ea4f0305fe4066a2be697fdc3188d2"
      ],
      "layout": "IPY_MODEL_5c40586819754261a141a6b88396e38b"
     }
    },
    "a92e5c7910914adaa58f8f051f71f11b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ab609f410fa845ba86388ae0cdd8e308": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b3ea4f0305fe4066a2be697fdc3188d2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5afdbf46d5c246e98d0fa4fd7a0bd52b",
      "placeholder": "​",
      "style": "IPY_MODEL_e990c10c236f4b948ddf1ed248280bfe",
      "value": " 222k/222k [00:01&lt;00:00, 214kB/s]"
     }
    },
    "d8178af03865496db91a56d5d3a0cc57": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "da75ddbf1b5f40b29c5b3b09a9c3d843": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e990c10c236f4b948ddf1ed248280bfe": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f22cde11a4d14e02b43c24316881dcae": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_82b06cc5c5684ef5b6502eb99a6d32ec",
       "IPY_MODEL_93d84aebb3c547a9b1398b975e08c60e"
      ],
      "layout": "IPY_MODEL_8655da061ea34a16800b111d6e20c69a"
     }
    },
    "f672e067718a47e9bc1a741bbaa9dd36": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a92e5c7910914adaa58f8f051f71f11b",
      "max": 313,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3fc159bfdb1b46ea8b6d462f43c42980",
      "value": 313
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}