Untitled3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import itertools\n",
    "from itertools import product, chain\n",
    "\n",
    "from pytorch_pretrained_bert import tokenization, BertTokenizer, BertModel, BertForMaskedLM, BertForPreTraining, BertConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "01/24/2019 22:16:56 - INFO - pytorch_pretrained_bert.tokenization -   loading vocabulary file /nas/pretrain-bert/pretrain-tensorflow/uncased_L-12_H-768_A-12/vocab.txt\n"
     ]
    }
   ],
   "source": [
    "CONFIG_NAME = 'bert_config.json'\n",
    "BERT_DIR = '/nas/pretrain-bert/pretrain-tensorflow/uncased_L-12_H-768_A-12/'\n",
    "tokenizer = BertTokenizer.from_pretrained(os.path.join(BERT_DIR, 'vocab.txt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reverse(l):\n",
    "    return list(reversed(l))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mask(ent_str):\n",
    "    tokens = ent_str.strip().split()\n",
    "    if len(tokens) == 1:\n",
    "        return '[%s]' % tokens[0]\n",
    "    elif len(tokens) == 2:\n",
    "        assert tokens[0] == 'the', ent_str\n",
    "        return '%s [%s]' % (tokens[0], tokens[1])\n",
    "    else:\n",
    "        assert False, ent_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "A_template = \"{dt} {ent0} {rel} {dt} {ent1} {rel_suffix}\"\n",
    "B_template = \"{dt} {ent} {pred}\"\n",
    "\n",
    "causal_templates = [[\"{A} because {B}.\"],# \"{B} so {A}.\"], \n",
    "                    [\"{A} so {B}.\"],# \"{B} because {A}.\"]\n",
    "                   ]\n",
    "turning_templates = [[\"{A} although {B}.\"],# \"{B} but {A}.\"], \n",
    "                     [\"{A} but {B}.\"],# \"{B} although {A}.\"]\n",
    "                    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_sentences(A_template, B_template, causal_templates, turning_templates,\n",
    "                   index=-1, orig_sentence='', entities=[\"John\", \"Mary\"], entity_substitutes=None, determiner=\"\", \n",
    "                   packed_relations=[\"rel/~rel\", \"rev_rel/~rev_rel\"], packed_relation_substitutes=None, relation_suffix=\"\",\n",
    "                   packed_predicates=[\"pred0/~pred0\", \"pred1/~pred1\"], predicate_substitutes=None,\n",
    "                   predicate_dichotomy=True, reverse_causal=False):\n",
    "    assert entities[0].lower() in tokenizer.vocab , entities[0]\n",
    "    assert entities[1].lower() in tokenizer.vocab , entities[1]\n",
    "    \n",
    "    relations, neg_relations = zip(*[rel.split(\"/\") for rel in packed_relations])\n",
    "    relations, neg_relations = list(relations), list(neg_relations)\n",
    "    predicates, neg_predicates = zip(*[pred.split(\"/\") for pred in packed_predicates])\n",
    "    predicates, neg_predicates = list(predicates), list(neg_predicates)\n",
    "        \n",
    "    As = [A_template.format(dt=determiner, ent0=ent0, ent1=ent1, rel=rel, rel_suffix=relation_suffix) \n",
    "          for ent0, ent1, rel in [entities + relations[:1], reverse(entities) + reverse(relations)[:1]]]\n",
    "    negAs = [A_template.format(dt=determiner, ent0=ent0, ent1=ent1, rel=rel, rel_suffix=relation_suffix) \n",
    "             for ent0, ent1, rel in [entities + neg_relations[:1], reverse(entities) + reverse(neg_relations)[:1]]]\n",
    "\n",
    "    Bs = [B_template.format(dt=determiner, ent=mask(ent), pred=pred) for ent, pred in zip(entities, predicates)]\n",
    "    negBs = [B_template.format(dt=determiner, ent=mask(ent), pred=pred) for ent, pred in zip(entities, neg_predicates)]\n",
    "    if predicate_dichotomy:\n",
    "        Bs += [B_template.format(dt=determiner, ent=mask(ent), pred=pred) for ent, pred in zip(entities, reversed(neg_predicates))]\n",
    "        negBs += [B_template.format(dt=determiner, ent=mask(ent), pred=pred) for ent, pred in zip(entities, reversed(predicates))]\n",
    "\n",
    "    def form_sentences(sentence_template, As, Bs):\n",
    "        return [\" \".join(sentence_template.format(A=A, B=B).split()) for A, B in product(As, Bs)]\n",
    "\n",
    "    causal_sentences = []\n",
    "    for causal_template in causal_templates[int(reverse_causal)]:\n",
    "        for A, B in [(As, Bs), (negAs, negBs)]:\n",
    "            causal_sentences.extend(form_sentences(causal_template, A, B))\n",
    "\n",
    "    turning_sentences = []\n",
    "    for turning_template in turning_templates[int(reverse_causal)]:\n",
    "        for A, B in [(As, negBs), (negAs, Bs)]:\n",
    "            turning_sentences.extend(form_sentences(turning_template, A, B))\n",
    "            \n",
    "    sentences = causal_sentences + turning_sentences\n",
    "    substituted_sentences = sentences\n",
    "    \n",
    "    if packed_relation_substitutes is not None:\n",
    "        packed_relation_substitutes = list(itertools.product(packed_relations[:1] + packed_relation_substitutes[0], \n",
    "                                                             packed_relations[1:] + packed_relation_substitutes[1]))\n",
    "        substituted_sentences = []\n",
    "        for packed_sub_relations in packed_relation_substitutes:\n",
    "            sub_relations, sub_neg_relations = zip(*[rel.split(\"/\") for rel in packed_sub_relations])\n",
    "            substituted_sentences += [sent.replace(relations[0], sub_relations[0]).replace(relations[1], sub_relations[1])\n",
    "                                      .replace(neg_relations[0], sub_neg_relations[0]).replace(neg_relations[1], sub_neg_relations[1]) \n",
    "                                      for sent in sentences]\n",
    "        substituted_sentences = list(set(substituted_sentences))\n",
    "        \n",
    "    if entity_substitutes is not None:\n",
    "        for sub in entity_substitutes:\n",
    "            for ent in sub:\n",
    "                assert ent.lower() in tokenizer.vocab , ent + \" not in BERT vocab\"\n",
    "        assert len(set(chain.from_iterable(entity_substitutes))) == 4, entity_substitutes\n",
    "        assert len(set(chain.from_iterable(entity_substitutes)).union(set(entities))) == 6 \n",
    "        \n",
    "        entity_substitutes = list(itertools.product(entities[:1] + entity_substitutes[0], entities[1:] + entity_substitutes[1]))\n",
    "        substituted_sentences = [sent.replace(entities[0], sub[0]).replace(entities[1], sub[1]) \n",
    "                                 for sent in substituted_sentences for sub in entity_substitutes]\n",
    "    return causal_sentences, turning_sentences, substituted_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = \\\n",
    "[\n",
    "    {\n",
    "        \"index\": 2,\n",
    "        \"orig_sentence\": \"The trophy doesn't fit into the brown suitcase because [it] is too large/small.\",\n",
    "        \"entities\": [\"trophy\", \"suitcase\"],\n",
    "        \"entitity_substitutes\": [[\"ball\", \"toy\"], [\"bag\", \"box\"]],\n",
    "        \"determiner\": \"the\",\n",
    "        \"packed_relations\": [\"doesn't fit into/can fit into\", \"doesn't hold/can hold\"],\n",
    "        \"packed_relation_substitutes\": [[\"can't be put into/can be put into\"], [\"doesn't have enough room for/has enough room for\"]],\n",
    "        \"relation_suffix\": \"\",\n",
    "        \"packed_predicates\": [\"is large/isn't large\", \"is small/isn't small\"],\n",
    "        \"predicate_dichotomy\": True,\n",
    "        \"reverse_causal\": False\n",
    "    },\n",
    "    {\n",
    "        \"index\": 4,\n",
    "        \"orig_sentence\": \"Joan made sure to thank Susan for all the help [she] had recieved/given.\",\n",
    "        \"entities\": [\"John\", \"Susan\"],\n",
    "        \"entity_substitutes\": [[\"David\", \"Michael\"], [\"Mary\", \"Linda\"]],\n",
    "        \"determiner\": \"\",\n",
    "        \"packed_relations\": [\"thanked/didn't thank\", \"took good care of/didn't good care of\"],\n",
    "        \"packed_relation_substitutes\": [[\"felt grateful to/didn't feel grateful to\"], [\"was appreciated by/wasn't appreciated by\"]],\n",
    "        \"relation_suffix\": \"\",\n",
    "        \"packed_predicates\": [\"had received a lot of help/hadn't received a lot of help\", \"had given a lot of help/hadn't given a lot of help\"],\n",
    "        \"predicate_dichotomy\": False,\n",
    "        \"reverse_causal\": False\n",
    "    },\n",
    "    {\n",
    "        \"index\": 4000,\n",
    "        \"orig_sentence\": \"John gave a lot of money to Susan because [he] was very rich/poor.\",\n",
    "        \"entities\": [\"John\", \"Susan\"],\n",
    "        \"entity_substitutes\": [[\"David\", \"Michael\"], [\"Mary\", \"Linda\"]],\n",
    "        \"determiner\": \"\",\n",
    "        \"packed_relations\": [\"gave a lot of money to/didn't give a lot of money to\", \"received a lot of money from/didn't receive a lot of money from\"],\n",
    "        \"packed_relation_substitutes\": [[\"subsidized/didn't subsidize\"], [\"borrowed a lot of money from/didn't borrow any money from\"]],\n",
    "        \"relation_suffix\": \"\",\n",
    "        \"packed_predicates\": [\"was rich/wasn't rich\", \"was poor/wasn't poor\"],\n",
    "        \"predicate_dichotomy\": True,\n",
    "        \"reverse_causal\": False\n",
    "    },\n",
    "    {\n",
    "        \"index\": 10,\n",
    "        \"orig_sentence\": \"The delivery truck zoomed by the school bus because [it] was going so fast/slow.\",\n",
    "        \"entities\": [\"truck\", \"bus\"],\n",
    "        \"entity_substitutes\": [[\"car\", \"ambulance\"], [\"bicycle\", \"tram\"]],\n",
    "        \"determiner\": \"the\",\n",
    "        \"packed_relations\": [\"overtook/couldn't overtake\", \"fell far behind/didn't fall far behind\"],\n",
    "        \"packed_relation_substitutes\": [[\"zoomed by/didn't pass\"], [\"was left behind/wasn't left far behind\"]],\n",
    "        \"relation_suffix\": \"\",\n",
    "        \"packed_predicates\": [\"was going fast/wasn't going fast\", \"was going slow/wasn't going slow\"],\n",
    "        \"predicate_dichotomy\": True,\n",
    "        \"reverse_causal\": False\n",
    "    },\n",
    "    {\n",
    "        \"index\": 12,\n",
    "        \"orig_sentence\": \"Frank felt vindicated/crushed when his longtime rival Bill revealed that [he] was the winner of the competition.\",\n",
    "        \"entities\": [\"John\", \"Susan\"],\n",
    "        \"entity_substitutes\": [[\"David\", \"Michael\"], [\"Mary\", \"Linda\"]],\n",
    "        \"determiner\": \"\",\n",
    "        \"packed_relations\": [\"beat/didn't beat\", \"lost to/didn't lose to\"],\n",
    "        \"relation_suffix\": \"in the game\",\n",
    "        \"packed_predicates\": [\"was happy/wasn't happy\", \"was sad/wasn't sad\"],\n",
    "        \"packed_relation_substitutes\": None,\n",
    "        \"predicate_dichotomy\": True,\n",
    "        \"reverse_causal\": True\n",
    "    },\n",
    "    {\n",
    "        \"index\": 16,\n",
    "        \"orig_sentence\": \"The large ball crashed right through the table because [it] was made of steel/styrofoam.\",\n",
    "        \"entities\": [\"ball\", \"board\"],\n",
    "        \"substitutes\": [[\"bullet\", \"arrow\"], [\"shield\", \"disk\"]],\n",
    "        \"determiner\": \"the\",\n",
    "        \"relations\": [\"crashed right through\", \"failed to block\"],\n",
    "        \"neg_relations\": [\"didn't crash through\", \"blocked\"],\n",
    "        \"relation_suffix\": \"\",\n",
    "        \"predicates\": [\"was hard\", \"was soft\"],\n",
    "        \"neg_predicates\": [\"wasn't hard\", \"wasn't soft\"],\n",
    "        \"predicate_dichotomy\": True,\n",
    "        \"reverse_causal\": False\n",
    "    },\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "causal_sentences, turning_sentences, substituted_sentences = \\\n",
    "    make_sentences(A_template, B_template, causal_templates, turning_templates, **frames[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['John beat Susan in the game so [John] was happy.',\n",
       " 'John beat Susan in the game so [Susan] was sad.',\n",
       " \"John beat Susan in the game so [John] wasn't sad.\",\n",
       " \"John beat Susan in the game so [Susan] wasn't happy.\",\n",
       " 'Susan lost to John in the game so [John] was happy.',\n",
       " 'Susan lost to John in the game so [Susan] was sad.',\n",
       " \"Susan lost to John in the game so [John] wasn't sad.\",\n",
       " \"Susan lost to John in the game so [Susan] wasn't happy.\",\n",
       " \"John didn't beat Susan in the game so [John] wasn't happy.\",\n",
       " \"John didn't beat Susan in the game so [Susan] wasn't sad.\",\n",
       " \"John didn't beat Susan in the game so [John] was sad.\",\n",
       " \"John didn't beat Susan in the game so [Susan] was happy.\",\n",
       " \"Susan didn't lose to John in the game so [John] wasn't happy.\",\n",
       " \"Susan didn't lose to John in the game so [Susan] wasn't sad.\",\n",
       " \"Susan didn't lose to John in the game so [John] was sad.\",\n",
       " \"Susan didn't lose to John in the game so [Susan] was happy.\",\n",
       " \"John beat Susan in the game but [John] wasn't happy.\",\n",
       " \"John beat Susan in the game but [Susan] wasn't sad.\",\n",
       " 'John beat Susan in the game but [John] was sad.',\n",
       " 'John beat Susan in the game but [Susan] was happy.',\n",
       " \"Susan lost to John in the game but [John] wasn't happy.\",\n",
       " \"Susan lost to John in the game but [Susan] wasn't sad.\",\n",
       " 'Susan lost to John in the game but [John] was sad.',\n",
       " 'Susan lost to John in the game but [Susan] was happy.',\n",
       " \"John didn't beat Susan in the game but [John] was happy.\",\n",
       " \"John didn't beat Susan in the game but [Susan] was sad.\",\n",
       " \"John didn't beat Susan in the game but [John] wasn't sad.\",\n",
       " \"John didn't beat Susan in the game but [Susan] wasn't happy.\",\n",
       " \"Susan didn't lose to John in the game but [John] was happy.\",\n",
       " \"Susan didn't lose to John in the game but [Susan] was sad.\",\n",
       " \"Susan didn't lose to John in the game but [John] wasn't sad.\",\n",
       " \"Susan didn't lose to John in the game but [Susan] wasn't happy.\"]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "['John beat Susan in the game so [John] was happy.',\n",
       " 'John beat Mary in the game so [John] was happy.',\n",
       " 'John beat Linda in the game so [John] was happy.',\n",
       " 'David beat Susan in the game so [David] was happy.',\n",
       " 'David beat Mary in the game so [David] was happy.',\n",
       " 'David beat Linda in the game so [David] was happy.',\n",
       " 'Michael beat Susan in the game so [Michael] was happy.',\n",
       " 'Michael beat Mary in the game so [Michael] was happy.',\n",
       " 'Michael beat Linda in the game so [Michael] was happy.',\n",
       " 'John beat Susan in the game so [Susan] was sad.',\n",
       " 'John beat Mary in the game so [Mary] was sad.',\n",
       " 'John beat Linda in the game so [Linda] was sad.',\n",
       " 'David beat Susan in the game so [Susan] was sad.',\n",
       " 'David beat Mary in the game so [Mary] was sad.',\n",
       " 'David beat Linda in the game so [Linda] was sad.',\n",
       " 'Michael beat Susan in the game so [Susan] was sad.',\n",
       " 'Michael beat Mary in the game so [Mary] was sad.',\n",
       " 'Michael beat Linda in the game so [Linda] was sad.',\n",
       " \"John beat Susan in the game so [John] wasn't sad.\",\n",
       " \"John beat Mary in the game so [John] wasn't sad.\",\n",
       " \"John beat Linda in the game so [John] wasn't sad.\",\n",
       " \"David beat Susan in the game so [David] wasn't sad.\",\n",
       " \"David beat Mary in the game so [David] wasn't sad.\",\n",
       " \"David beat Linda in the game so [David] wasn't sad.\",\n",
       " \"Michael beat Susan in the game so [Michael] wasn't sad.\",\n",
       " \"Michael beat Mary in the game so [Michael] wasn't sad.\",\n",
       " \"Michael beat Linda in the game so [Michael] wasn't sad.\",\n",
       " \"John beat Susan in the game so [Susan] wasn't happy.\",\n",
       " \"John beat Mary in the game so [Mary] wasn't happy.\",\n",
       " \"John beat Linda in the game so [Linda] wasn't happy.\",\n",
       " \"David beat Susan in the game so [Susan] wasn't happy.\",\n",
       " \"David beat Mary in the game so [Mary] wasn't happy.\",\n",
       " \"David beat Linda in the game so [Linda] wasn't happy.\",\n",
       " \"Michael beat Susan in the game so [Susan] wasn't happy.\",\n",
       " \"Michael beat Mary in the game so [Mary] wasn't happy.\",\n",
       " \"Michael beat Linda in the game so [Linda] wasn't happy.\",\n",
       " 'Susan lost to John in the game so [John] was happy.',\n",
       " 'Mary lost to John in the game so [John] was happy.',\n",
       " 'Linda lost to John in the game so [John] was happy.',\n",
       " 'Susan lost to David in the game so [David] was happy.',\n",
       " 'Mary lost to David in the game so [David] was happy.',\n",
       " 'Linda lost to David in the game so [David] was happy.',\n",
       " 'Susan lost to Michael in the game so [Michael] was happy.',\n",
       " 'Mary lost to Michael in the game so [Michael] was happy.',\n",
       " 'Linda lost to Michael in the game so [Michael] was happy.',\n",
       " 'Susan lost to John in the game so [Susan] was sad.',\n",
       " 'Mary lost to John in the game so [Mary] was sad.',\n",
       " 'Linda lost to John in the game so [Linda] was sad.',\n",
       " 'Susan lost to David in the game so [Susan] was sad.',\n",
       " 'Mary lost to David in the game so [Mary] was sad.',\n",
       " 'Linda lost to David in the game so [Linda] was sad.',\n",
       " 'Susan lost to Michael in the game so [Susan] was sad.',\n",
       " 'Mary lost to Michael in the game so [Mary] was sad.',\n",
       " 'Linda lost to Michael in the game so [Linda] was sad.',\n",
       " \"Susan lost to John in the game so [John] wasn't sad.\",\n",
       " \"Mary lost to John in the game so [John] wasn't sad.\",\n",
       " \"Linda lost to John in the game so [John] wasn't sad.\",\n",
       " \"Susan lost to David in the game so [David] wasn't sad.\",\n",
       " \"Mary lost to David in the game so [David] wasn't sad.\",\n",
       " \"Linda lost to David in the game so [David] wasn't sad.\",\n",
       " \"Susan lost to Michael in the game so [Michael] wasn't sad.\",\n",
       " \"Mary lost to Michael in the game so [Michael] wasn't sad.\",\n",
       " \"Linda lost to Michael in the game so [Michael] wasn't sad.\",\n",
       " \"Susan lost to John in the game so [Susan] wasn't happy.\",\n",
       " \"Mary lost to John in the game so [Mary] wasn't happy.\",\n",
       " \"Linda lost to John in the game so [Linda] wasn't happy.\",\n",
       " \"Susan lost to David in the game so [Susan] wasn't happy.\",\n",
       " \"Mary lost to David in the game so [Mary] wasn't happy.\",\n",
       " \"Linda lost to David in the game so [Linda] wasn't happy.\",\n",
       " \"Susan lost to Michael in the game so [Susan] wasn't happy.\",\n",
       " \"Mary lost to Michael in the game so [Mary] wasn't happy.\",\n",
       " \"Linda lost to Michael in the game so [Linda] wasn't happy.\",\n",
       " \"John didn't beat Susan in the game so [John] wasn't happy.\",\n",
       " \"John didn't beat Mary in the game so [John] wasn't happy.\",\n",
       " \"John didn't beat Linda in the game so [John] wasn't happy.\",\n",
       " \"David didn't beat Susan in the game so [David] wasn't happy.\",\n",
       " \"David didn't beat Mary in the game so [David] wasn't happy.\",\n",
       " \"David didn't beat Linda in the game so [David] wasn't happy.\",\n",
       " \"Michael didn't beat Susan in the game so [Michael] wasn't happy.\",\n",
       " \"Michael didn't beat Mary in the game so [Michael] wasn't happy.\",\n",
       " \"Michael didn't beat Linda in the game so [Michael] wasn't happy.\",\n",
       " \"John didn't beat Susan in the game so [Susan] wasn't sad.\",\n",
       " \"John didn't beat Mary in the game so [Mary] wasn't sad.\",\n",
       " \"John didn't beat Linda in the game so [Linda] wasn't sad.\",\n",
       " \"David didn't beat Susan in the game so [Susan] wasn't sad.\",\n",
       " \"David didn't beat Mary in the game so [Mary] wasn't sad.\",\n",
       " \"David didn't beat Linda in the game so [Linda] wasn't sad.\",\n",
       " \"Michael didn't beat Susan in the game so [Susan] wasn't sad.\",\n",
       " \"Michael didn't beat Mary in the game so [Mary] wasn't sad.\",\n",
       " \"Michael didn't beat Linda in the game so [Linda] wasn't sad.\",\n",
       " \"John didn't beat Susan in the game so [John] was sad.\",\n",
       " \"John didn't beat Mary in the game so [John] was sad.\",\n",
       " \"John didn't beat Linda in the game so [John] was sad.\",\n",
       " \"David didn't beat Susan in the game so [David] was sad.\",\n",
       " \"David didn't beat Mary in the game so [David] was sad.\",\n",
       " \"David didn't beat Linda in the game so [David] was sad.\",\n",
       " \"Michael didn't beat Susan in the game so [Michael] was sad.\",\n",
       " \"Michael didn't beat Mary in the game so [Michael] was sad.\",\n",
       " \"Michael didn't beat Linda in the game so [Michael] was sad.\",\n",
       " \"John didn't beat Susan in the game so [Susan] was happy.\",\n",
       " \"John didn't beat Mary in the game so [Mary] was happy.\",\n",
       " \"John didn't beat Linda in the game so [Linda] was happy.\",\n",
       " \"David didn't beat Susan in the game so [Susan] was happy.\",\n",
       " \"David didn't beat Mary in the game so [Mary] was happy.\",\n",
       " \"David didn't beat Linda in the game so [Linda] was happy.\",\n",
       " \"Michael didn't beat Susan in the game so [Susan] was happy.\",\n",
       " \"Michael didn't beat Mary in the game so [Mary] was happy.\",\n",
       " \"Michael didn't beat Linda in the game so [Linda] was happy.\",\n",
       " \"Susan didn't lose to John in the game so [John] wasn't happy.\",\n",
       " \"Mary didn't lose to John in the game so [John] wasn't happy.\",\n",
       " \"Linda didn't lose to John in the game so [John] wasn't happy.\",\n",
       " \"Susan didn't lose to David in the game so [David] wasn't happy.\",\n",
       " \"Mary didn't lose to David in the game so [David] wasn't happy.\",\n",
       " \"Linda didn't lose to David in the game so [David] wasn't happy.\",\n",
       " \"Susan didn't lose to Michael in the game so [Michael] wasn't happy.\",\n",
       " \"Mary didn't lose to Michael in the game so [Michael] wasn't happy.\",\n",
       " \"Linda didn't lose to Michael in the game so [Michael] wasn't happy.\",\n",
       " \"Susan didn't lose to John in the game so [Susan] wasn't sad.\",\n",
       " \"Mary didn't lose to John in the game so [Mary] wasn't sad.\",\n",
       " \"Linda didn't lose to John in the game so [Linda] wasn't sad.\",\n",
       " \"Susan didn't lose to David in the game so [Susan] wasn't sad.\",\n",
       " \"Mary didn't lose to David in the game so [Mary] wasn't sad.\",\n",
       " \"Linda didn't lose to David in the game so [Linda] wasn't sad.\",\n",
       " \"Susan didn't lose to Michael in the game so [Susan] wasn't sad.\",\n",
       " \"Mary didn't lose to Michael in the game so [Mary] wasn't sad.\",\n",
       " \"Linda didn't lose to Michael in the game so [Linda] wasn't sad.\",\n",
       " \"Susan didn't lose to John in the game so [John] was sad.\",\n",
       " \"Mary didn't lose to John in the game so [John] was sad.\",\n",
       " \"Linda didn't lose to John in the game so [John] was sad.\",\n",
       " \"Susan didn't lose to David in the game so [David] was sad.\",\n",
       " \"Mary didn't lose to David in the game so [David] was sad.\",\n",
       " \"Linda didn't lose to David in the game so [David] was sad.\",\n",
       " \"Susan didn't lose to Michael in the game so [Michael] was sad.\",\n",
       " \"Mary didn't lose to Michael in the game so [Michael] was sad.\",\n",
       " \"Linda didn't lose to Michael in the game so [Michael] was sad.\",\n",
       " \"Susan didn't lose to John in the game so [Susan] was happy.\",\n",
       " \"Mary didn't lose to John in the game so [Mary] was happy.\",\n",
       " \"Linda didn't lose to John in the game so [Linda] was happy.\",\n",
       " \"Susan didn't lose to David in the game so [Susan] was happy.\",\n",
       " \"Mary didn't lose to David in the game so [Mary] was happy.\",\n",
       " \"Linda didn't lose to David in the game so [Linda] was happy.\",\n",
       " \"Susan didn't lose to Michael in the game so [Susan] was happy.\",\n",
       " \"Mary didn't lose to Michael in the game so [Mary] was happy.\",\n",
       " \"Linda didn't lose to Michael in the game so [Linda] was happy.\",\n",
       " \"John beat Susan in the game but [John] wasn't happy.\",\n",
       " \"John beat Mary in the game but [John] wasn't happy.\",\n",
       " \"John beat Linda in the game but [John] wasn't happy.\",\n",
       " \"David beat Susan in the game but [David] wasn't happy.\",\n",
       " \"David beat Mary in the game but [David] wasn't happy.\",\n",
       " \"David beat Linda in the game but [David] wasn't happy.\",\n",
       " \"Michael beat Susan in the game but [Michael] wasn't happy.\",\n",
       " \"Michael beat Mary in the game but [Michael] wasn't happy.\",\n",
       " \"Michael beat Linda in the game but [Michael] wasn't happy.\",\n",
       " \"John beat Susan in the game but [Susan] wasn't sad.\",\n",
       " \"John beat Mary in the game but [Mary] wasn't sad.\",\n",
       " \"John beat Linda in the game but [Linda] wasn't sad.\",\n",
       " \"David beat Susan in the game but [Susan] wasn't sad.\",\n",
       " \"David beat Mary in the game but [Mary] wasn't sad.\",\n",
       " \"David beat Linda in the game but [Linda] wasn't sad.\",\n",
       " \"Michael beat Susan in the game but [Susan] wasn't sad.\",\n",
       " \"Michael beat Mary in the game but [Mary] wasn't sad.\",\n",
       " \"Michael beat Linda in the game but [Linda] wasn't sad.\",\n",
       " 'John beat Susan in the game but [John] was sad.',\n",
       " 'John beat Mary in the game but [John] was sad.',\n",
       " 'John beat Linda in the game but [John] was sad.',\n",
       " 'David beat Susan in the game but [David] was sad.',\n",
       " 'David beat Mary in the game but [David] was sad.',\n",
       " 'David beat Linda in the game but [David] was sad.',\n",
       " 'Michael beat Susan in the game but [Michael] was sad.',\n",
       " 'Michael beat Mary in the game but [Michael] was sad.',\n",
       " 'Michael beat Linda in the game but [Michael] was sad.',\n",
       " 'John beat Susan in the game but [Susan] was happy.',\n",
       " 'John beat Mary in the game but [Mary] was happy.',\n",
       " 'John beat Linda in the game but [Linda] was happy.',\n",
       " 'David beat Susan in the game but [Susan] was happy.',\n",
       " 'David beat Mary in the game but [Mary] was happy.',\n",
       " 'David beat Linda in the game but [Linda] was happy.',\n",
       " 'Michael beat Susan in the game but [Susan] was happy.',\n",
       " 'Michael beat Mary in the game but [Mary] was happy.',\n",
       " 'Michael beat Linda in the game but [Linda] was happy.',\n",
       " \"Susan lost to John in the game but [John] wasn't happy.\",\n",
       " \"Mary lost to John in the game but [John] wasn't happy.\",\n",
       " \"Linda lost to John in the game but [John] wasn't happy.\",\n",
       " \"Susan lost to David in the game but [David] wasn't happy.\",\n",
       " \"Mary lost to David in the game but [David] wasn't happy.\",\n",
       " \"Linda lost to David in the game but [David] wasn't happy.\",\n",
       " \"Susan lost to Michael in the game but [Michael] wasn't happy.\",\n",
       " \"Mary lost to Michael in the game but [Michael] wasn't happy.\",\n",
       " \"Linda lost to Michael in the game but [Michael] wasn't happy.\",\n",
       " \"Susan lost to John in the game but [Susan] wasn't sad.\",\n",
       " \"Mary lost to John in the game but [Mary] wasn't sad.\",\n",
       " \"Linda lost to John in the game but [Linda] wasn't sad.\",\n",
       " \"Susan lost to David in the game but [Susan] wasn't sad.\",\n",
       " \"Mary lost to David in the game but [Mary] wasn't sad.\",\n",
       " \"Linda lost to David in the game but [Linda] wasn't sad.\",\n",
       " \"Susan lost to Michael in the game but [Susan] wasn't sad.\",\n",
       " \"Mary lost to Michael in the game but [Mary] wasn't sad.\",\n",
       " \"Linda lost to Michael in the game but [Linda] wasn't sad.\",\n",
       " 'Susan lost to John in the game but [John] was sad.',\n",
       " 'Mary lost to John in the game but [John] was sad.',\n",
       " 'Linda lost to John in the game but [John] was sad.',\n",
       " 'Susan lost to David in the game but [David] was sad.',\n",
       " 'Mary lost to David in the game but [David] was sad.',\n",
       " 'Linda lost to David in the game but [David] was sad.',\n",
       " 'Susan lost to Michael in the game but [Michael] was sad.',\n",
       " 'Mary lost to Michael in the game but [Michael] was sad.',\n",
       " 'Linda lost to Michael in the game but [Michael] was sad.',\n",
       " 'Susan lost to John in the game but [Susan] was happy.',\n",
       " 'Mary lost to John in the game but [Mary] was happy.',\n",
       " 'Linda lost to John in the game but [Linda] was happy.',\n",
       " 'Susan lost to David in the game but [Susan] was happy.',\n",
       " 'Mary lost to David in the game but [Mary] was happy.',\n",
       " 'Linda lost to David in the game but [Linda] was happy.',\n",
       " 'Susan lost to Michael in the game but [Susan] was happy.',\n",
       " 'Mary lost to Michael in the game but [Mary] was happy.',\n",
       " 'Linda lost to Michael in the game but [Linda] was happy.',\n",
       " \"John didn't beat Susan in the game but [John] was happy.\",\n",
       " \"John didn't beat Mary in the game but [John] was happy.\",\n",
       " \"John didn't beat Linda in the game but [John] was happy.\",\n",
       " \"David didn't beat Susan in the game but [David] was happy.\",\n",
       " \"David didn't beat Mary in the game but [David] was happy.\",\n",
       " \"David didn't beat Linda in the game but [David] was happy.\",\n",
       " \"Michael didn't beat Susan in the game but [Michael] was happy.\",\n",
       " \"Michael didn't beat Mary in the game but [Michael] was happy.\",\n",
       " \"Michael didn't beat Linda in the game but [Michael] was happy.\",\n",
       " \"John didn't beat Susan in the game but [Susan] was sad.\",\n",
       " \"John didn't beat Mary in the game but [Mary] was sad.\",\n",
       " \"John didn't beat Linda in the game but [Linda] was sad.\",\n",
       " \"David didn't beat Susan in the game but [Susan] was sad.\",\n",
       " \"David didn't beat Mary in the game but [Mary] was sad.\",\n",
       " \"David didn't beat Linda in the game but [Linda] was sad.\",\n",
       " \"Michael didn't beat Susan in the game but [Susan] was sad.\",\n",
       " \"Michael didn't beat Mary in the game but [Mary] was sad.\",\n",
       " \"Michael didn't beat Linda in the game but [Linda] was sad.\",\n",
       " \"John didn't beat Susan in the game but [John] wasn't sad.\",\n",
       " \"John didn't beat Mary in the game but [John] wasn't sad.\",\n",
       " \"John didn't beat Linda in the game but [John] wasn't sad.\",\n",
       " \"David didn't beat Susan in the game but [David] wasn't sad.\",\n",
       " \"David didn't beat Mary in the game but [David] wasn't sad.\",\n",
       " \"David didn't beat Linda in the game but [David] wasn't sad.\",\n",
       " \"Michael didn't beat Susan in the game but [Michael] wasn't sad.\",\n",
       " \"Michael didn't beat Mary in the game but [Michael] wasn't sad.\",\n",
       " \"Michael didn't beat Linda in the game but [Michael] wasn't sad.\",\n",
       " \"John didn't beat Susan in the game but [Susan] wasn't happy.\",\n",
       " \"John didn't beat Mary in the game but [Mary] wasn't happy.\",\n",
       " \"John didn't beat Linda in the game but [Linda] wasn't happy.\",\n",
       " \"David didn't beat Susan in the game but [Susan] wasn't happy.\",\n",
       " \"David didn't beat Mary in the game but [Mary] wasn't happy.\",\n",
       " \"David didn't beat Linda in the game but [Linda] wasn't happy.\",\n",
       " \"Michael didn't beat Susan in the game but [Susan] wasn't happy.\",\n",
       " \"Michael didn't beat Mary in the game but [Mary] wasn't happy.\",\n",
       " \"Michael didn't beat Linda in the game but [Linda] wasn't happy.\",\n",
       " \"Susan didn't lose to John in the game but [John] was happy.\",\n",
       " \"Mary didn't lose to John in the game but [John] was happy.\",\n",
       " \"Linda didn't lose to John in the game but [John] was happy.\",\n",
       " \"Susan didn't lose to David in the game but [David] was happy.\",\n",
       " \"Mary didn't lose to David in the game but [David] was happy.\",\n",
       " \"Linda didn't lose to David in the game but [David] was happy.\",\n",
       " \"Susan didn't lose to Michael in the game but [Michael] was happy.\",\n",
       " \"Mary didn't lose to Michael in the game but [Michael] was happy.\",\n",
       " \"Linda didn't lose to Michael in the game but [Michael] was happy.\",\n",
       " \"Susan didn't lose to John in the game but [Susan] was sad.\",\n",
       " \"Mary didn't lose to John in the game but [Mary] was sad.\",\n",
       " \"Linda didn't lose to John in the game but [Linda] was sad.\",\n",
       " \"Susan didn't lose to David in the game but [Susan] was sad.\",\n",
       " \"Mary didn't lose to David in the game but [Mary] was sad.\",\n",
       " \"Linda didn't lose to David in the game but [Linda] was sad.\",\n",
       " \"Susan didn't lose to Michael in the game but [Susan] was sad.\",\n",
       " \"Mary didn't lose to Michael in the game but [Mary] was sad.\",\n",
       " \"Linda didn't lose to Michael in the game but [Linda] was sad.\",\n",
       " \"Susan didn't lose to John in the game but [John] wasn't sad.\",\n",
       " \"Mary didn't lose to John in the game but [John] wasn't sad.\",\n",
       " \"Linda didn't lose to John in the game but [John] wasn't sad.\",\n",
       " \"Susan didn't lose to David in the game but [David] wasn't sad.\",\n",
       " \"Mary didn't lose to David in the game but [David] wasn't sad.\",\n",
       " \"Linda didn't lose to David in the game but [David] wasn't sad.\",\n",
       " \"Susan didn't lose to Michael in the game but [Michael] wasn't sad.\",\n",
       " \"Mary didn't lose to Michael in the game but [Michael] wasn't sad.\",\n",
       " \"Linda didn't lose to Michael in the game but [Michael] wasn't sad.\",\n",
       " \"Susan didn't lose to John in the game but [Susan] wasn't happy.\",\n",
       " \"Mary didn't lose to John in the game but [Mary] wasn't happy.\",\n",
       " \"Linda didn't lose to John in the game but [Linda] wasn't happy.\",\n",
       " \"Susan didn't lose to David in the game but [Susan] wasn't happy.\",\n",
       " \"Mary didn't lose to David in the game but [Mary] wasn't happy.\",\n",
       " \"Linda didn't lose to David in the game but [Linda] wasn't happy.\",\n",
       " \"Susan didn't lose to Michael in the game but [Susan] wasn't happy.\",\n",
       " \"Mary didn't lose to Michael in the game but [Mary] wasn't happy.\",\n",
       " \"Linda didn't lose to Michael in the game but [Linda] wasn't happy.\"]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "causal_sentences\n",
    "turning_sentences\n",
    "# substituted_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "examples = [(2,\n",
    "  \"The trophy doesn't fit into the brown suitcase because [it] is too large.\",\n",
    "  'fit into:large/small'),\n",
    " (4,\n",
    "  'Joan made sure to thank Susan for all the help [she] had recieved.',\n",
    "  'thank:receive/give'),\n",
    " (10,\n",
    "  'The delivery truck zoomed by the school bus because [it] was going so fast.',\n",
    "  'zoom by:fast/slow'),\n",
    " (12,\n",
    "  'Frank felt vindicated when his longtime rival Bill revealed that [he] was the winner of the competition.',\n",
    "  'vindicated/crushed:be the winner'),\n",
    " (16,\n",
    "  'The large ball crashed right through the table because [it] was made of steel.',\n",
    "  'crash through:[hard]/[soft]'),\n",
    " (18,\n",
    "  \"John couldn't see the stage with Billy in front of him because [he] is so short.\",\n",
    "  '[block]:short/tall'),\n",
    " (20,\n",
    "  'Tom threw his schoolbag down to Ray after [he] reached the top of the stairs.',\n",
    "  'down to:top/bottom'),\n",
    " (22,\n",
    "  'Although they ran at about the same speed, Sue beat Sally because [she] had such a good start.',\n",
    "  'beat:good/bad'),\n",
    " (26,\n",
    "  \"Sam's drawing was hung just above Tina's and [it] did look much better with another one below it.\",\n",
    "  'above/below'),\n",
    " (28,\n",
    "  'Anna did a lot better than her good friend Lucy on the test because [she] had studied so hard.',\n",
    "  'better/worse:study hard'),\n",
    " (30,\n",
    "  'The firemen arrived after the police because [they] were coming from so far away.',\n",
    "  'after/before:far away'),\n",
    " (32,\n",
    "  \"Frank was upset with Tom because the toaster [he] had bought from him didn't work.\",\n",
    "  'be upset with:buy from not work/sell not work'),\n",
    " (36,\n",
    "  'The sack of potatoes had been placed above the bag of flour, so [it] had to be moved first.',\n",
    "  'above/below:moved first'),\n",
    " (38,\n",
    "  'Pete envies Martin although [he] is very successful.',\n",
    "  'although/because'),\n",
    " (42,\n",
    "  'I poured water from the bottle into the cup until [it] was empty.',\n",
    "  'pour:empty/full'),\n",
    " (46,\n",
    "  \"Sid explained his theory to Mark but [he] couldn't convince him.\",\n",
    "  'explain:convince/understand'),\n",
    " (48,\n",
    "  \"Susan knew that Ann's son had been in a car accident, so [she] told her about it.\",\n",
    "  '?know tell:so/because'),\n",
    " (50,\n",
    "  \"Joe's uncle can still beat him at tennis, even though [he] is 30 years younger.\",\n",
    "  'beat:younger/older'),\n",
    " (64,\n",
    "  'In the middle of the outdoor concert, the rain started falling, but [it] continued until 10.',\n",
    "  'but/and'),\n",
    " (68,\n",
    "  'Ann asked Mary what time the library closes, because [she] had forgotten.',\n",
    "  'because/but'),\n",
    " (84,\n",
    "  'If the con artist has succeeded in fooling Sam, [he] would have gotten a lot of money.',\n",
    "  'fool:get/lose'),\n",
    " (92,\n",
    "  'Alice tried frantically to stop her daughter from chatting at the party, leaving us to wonder why [she] was behaving so strangely.',\n",
    "  '?stop normal/stop abnormal:strange'),\n",
    " (98,\n",
    "  \"I was trying to open the lock with the key, but someone had filled the  keyhole with chewing gum, and I couldn't get [it] in.\",\n",
    "  'put ... into filled with ... :get in/get out'),\n",
    " (100,\n",
    "  'The dog chased the cat, which ran up a tree. [It] waited at the bottom.',\n",
    "  'up:at the bottom/at the top'),\n",
    " (106,\n",
    "  'John was doing research in the library when he heard a man humming and  whistling. [He] was very annoyed.',\n",
    "  'hear ... humming and whistling:annoyed/annoying'),\n",
    " (108,\n",
    "  'John was jogging through the park when he saw a man juggling watermelons. [He] was very impressed.',\n",
    "  'see ... juggling watermelons:impressed/impressive'),\n",
    " (132,\n",
    "  'Jane knocked on the door, and Susan answered it. [She] invited her to come out.',\n",
    "  'visit:invite come out/invite come in'),\n",
    " (150,\n",
    "  'Jackson was greatly influenced by Arnold, though [he] lived two centuries later.',\n",
    "  'influence:later/earlier'),\n",
    " (160,\n",
    "  'The actress used to be named Terpsichore, but she changed it to Tina a  few years ago, because she figured [it] was too hard to pronounce.',\n",
    "  'change:hard/easy'),\n",
    " (166,\n",
    "  'Fred is the only man still alive who remembers my great-grandfather. [He] is a remarkable man.',\n",
    "  'alive:is/was'),\n",
    " (170,\n",
    "  \"In July, Kamtchatka declared war on Yakutsk. Since Yakutsk's army was much  better equipped and ten times larger, [they] were defeated within weeks.\",\n",
    "  'better equipped and large:defeated/victorious'),\n",
    " (186,\n",
    "  'When the sponsors of the bill got to the town hall, they were surprised to find that the room was full of opponents. [They] were very much in the minority.',\n",
    "  'be full of:minority/majority'),\n",
    " (188,\n",
    "  'Everyone really loved the oatmeal cookies; only a few people liked the  chocolate chip cookies. Next time, we should make more of [them] .',\n",
    "  'like over:more/fewer'),\n",
    " (190,\n",
    "  'We had hoped to place copies of our newsletter on all the chairs in the auditorium, but there were simply not enough of [them] .',\n",
    "  'place on all:not enough/too many'),\n",
    " (196,\n",
    "  \"Steve follows Fred's example in everything. [He] admires him hugely.\",\n",
    "  'follow:admire/influence'),\n",
    " (198,\n",
    "  \"The table won't fit through the doorway because [it] is too wide.\",\n",
    "  'fit through:wide/narrow'),\n",
    " (200,\n",
    "  'Grace was happy to trade me her sweater for my jacket. She thinks [it] looks dowdy on her.',\n",
    "  'trade:dowdy/great'),\n",
    " (202,\n",
    "  'John hired Bill to take care of [him] .',\n",
    "  'hire/hire oneself to:take care of'),\n",
    " (204,\n",
    "  'John promised Bill to leave, so an hour later [he] left.',\n",
    "  'promise/order'),\n",
    " (210,\n",
    "  \"Jane knocked on Susan's door but [she] did not get an answer.\",\n",
    "  'knock:get an answer/answer'),\n",
    " (212,\n",
    "  'Joe paid the detective after [he] received the final report on the case.',\n",
    "  'pay:receive/deliver'),\n",
    " (226,\n",
    "  'Bill passed the half-empty plate to John because [he] was full.',\n",
    "  'pass the plate:full/hungry'),\n",
    " (252,\n",
    "  'George got free tickets to the play, but he gave them to Eric, even though [he] was particularly eager to see it.',\n",
    "  'even though/because/not'),\n",
    " (255,\n",
    "  \"Jane gave Joan candy because [she] wasn't hungry.\",\n",
    "  'give:not hungry/hungry'),\n",
    " (259,\n",
    "  'James asked Robert for a favor but [he] was refused.',\n",
    "  'ask for a favor:refuse/be refused`'),\n",
    " (261,\n",
    "  'Kirilov ceded the presidency to Shatov because [he] was less popular.',\n",
    "  'cede:less popular/more popular'),\n",
    " (263,\n",
    "  'Emma did not pass the ball to Janie although [she] saw that she was open.',\n",
    "  'not pass although:see open/open')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(examples)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}