lm_truthfulness_gpt-j_sparse.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL = \"gpt-j\"\n",
    "N_DEV = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import jsonlines\n",
    "import json\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "from transformers import AutoTokenizer, GPT2LMHeadModel, GPT2Tokenizer, GPTJForCausalLM\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib.patches import Rectangle\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import scipy.stats\n",
    "\n",
    "torch.manual_seed(0)\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set()\n",
    "sns.set_style(\"white\")\n",
    "sns.set_style({'font.family':'serif', 'font.serif':'Times New Roman'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_boolq(split, max_examples=None):\n",
    "    data = jsonlines.open(f\"boolean-questions/{split}.jsonl\")\n",
    "    questions = []\n",
    "    for line in data:\n",
    "        q = line[\"question\"]\n",
    "        q = f\"{q}?\"\n",
    "        \n",
    "        a_good = str(line[\"answer\"]).lower()\n",
    "        a_bad = \"false\" if a_good == \"true\" else \"true\"\n",
    "        \n",
    "        questions.append((q, a_good, a_bad))\n",
    "        if max_examples is not None and len(questions) == max_examples:\n",
    "            break\n",
    "    np.random.shuffle(questions)\n",
    "    return questions\n",
    "\n",
    "def load_sciq(split, max_examples=None):\n",
    "    questions = []\n",
    "    with open(f\"sciq/{split}.json\") as reader:\n",
    "        data = json.load(reader)\n",
    "        \n",
    "    for line in data:\n",
    "        q = line[\"question\"]\n",
    "        \n",
    "        a_good = line[\"correct_answer\"]\n",
    "        a_bad = line[\"distractor1\"]\n",
    "        \n",
    "        questions.append((q, a_good, a_bad))\n",
    "        if max_examples is not None and len(questions) == max_examples:\n",
    "            break\n",
    "    np.random.shuffle(questions)\n",
    "    return questions\n",
    "\n",
    "def load_creak(split, max_examples=None):\n",
    "    questions = []\n",
    "    reader = jsonlines.open(f\"creak/data/creak/{split}.json\")\n",
    "    for line in reader:\n",
    "        q = line[\"sentence\"]\n",
    "        a_good = line[\"label\"]\n",
    "        a_bad = \"true\" if line[\"label\"] == \"false\" else \"false\"\n",
    "        questions.append((q, a_good, a_bad))\n",
    "    np.random.shuffle(questions)\n",
    "    return questions\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if MODEL.startswith(\"gpt2\"):\n",
    "    #lm = GPT2LMHeadModel.from_pretrained(MODEL).eval().cuda()\n",
    "    tokenizer = GPT2Tokenizer.from_pretrained(MODEL)\n",
    "elif MODEL == \"gpt-j\":\n",
    "    #lm = GPTJForCausalLM.from_pretrained(\n",
    "    #    \"EleutherAI/gpt-j-6B\",\n",
    "    #    revision=\"float16\",\n",
    "    #    torch_dtype=torch.float16\n",
    "    #).eval().cuda()\n",
    "    tokenizer = AutoTokenizer.from_pretrained(\"EleutherAI/gpt-j-6B\")\n",
    "\n",
    "probe_layer = -1\n",
    "probe_reduction = lambda tensor: tensor[:, -1, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lm_pred(toks):\n",
    "    results = lm(toks, labels=toks, output_hidden_states=True)\n",
    "    assert toks.shape[0] == 1\n",
    "    results.logprob = -results[\"loss\"] * toks.shape[1]\n",
    "    return results\n",
    "\n",
    "@torch.no_grad()\n",
    "def get_preds_and_reprs(data):\n",
    "    ratios = []\n",
    "    corrects = []\n",
    "    qa_good_reprs = []\n",
    "    qa_bad_reprs = []\n",
    "    for q, a_good, a_bad in tqdm(data):\n",
    "        qa_good_tok = tokenizer(f\"{q} {a_good}\", return_tensors=\"pt\").input_ids.cuda()\n",
    "        qa_good_pred = lm_pred(qa_good_tok)\n",
    "        qa_good_repr = probe_reduction(qa_good_pred.hidden_states[probe_layer])\n",
    "\n",
    "        qa_bad_tok = tokenizer(f\"{q} {a_bad}\", return_tensors=\"pt\").input_ids.cuda()\n",
    "        qa_bad_pred = lm_pred(qa_bad_tok)\n",
    "        qa_bad_repr = probe_reduction(qa_bad_pred.hidden_states[probe_layer])\n",
    "        \n",
    "        ratios.append(qa_good_pred.logprob - qa_bad_pred.logprob)\n",
    "        corrects.append(qa_good_pred.logprob > qa_bad_pred.logprob)\n",
    "        qa_good_reprs.append(qa_good_repr)\n",
    "        qa_bad_reprs.append(qa_bad_repr)\n",
    "    return {\n",
    "        \"ratio\": torch.tensor(ratios),\n",
    "        \"correct\": torch.tensor(corrects),\n",
    "        \"good_repr\": torch.cat(qa_good_reprs, dim=0),\n",
    "        \"bad_repr\": torch.cat(qa_bad_reprs, dim=0)\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    dataset_name: {\n",
    "        split_name: loader(split_name) for split_name in [\"train\", \"valid\"]\n",
    "    }\n",
    "    for dataset_name, loader in [(\"boolq\", load_boolq), (\"sciq\", load_sciq), (\"creak\", load_creak)]\n",
    "}\n",
    "\n",
    "def process_data(data, suffix):\n",
    "    data_path = f\"cache/data_processed_{MODEL}{suffix}.pt\"\n",
    "    if os.path.exists(data_path):\n",
    "        data_processed = torch.load(data_path)\n",
    "    else:\n",
    "        data_processed = {}\n",
    "\n",
    "    changed = False\n",
    "    for dataset_name, dataset in data.items():\n",
    "        if dataset_name not in data_processed:\n",
    "            data_processed[dataset_name] = {}\n",
    "        for split_name, split_data in dataset.items():\n",
    "            if split_name not in data_processed[dataset_name]:\n",
    "                changed = True\n",
    "                data_processed[dataset_name][split_name] = get_preds_and_reprs(split_data)\n",
    "\n",
    "    if changed:\n",
    "        torch.save(data_processed, data_path)\n",
    "        \n",
    "    return data_processed\n",
    "\n",
    "data_processed = process_data(data, \"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Probes are more accurate than models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "boolq\n",
      "probe acc 0.6238532066345215\n",
      "model acc 0.6149846911430359\n",
      "\n",
      "sciq\n",
      "probe acc 0.8860000371932983\n",
      "model acc 0.8379999995231628\n",
      "\n",
      "creak\n",
      "probe acc 0.7089715600013733\n",
      "model acc 0.50255286693573\n",
      "\n"
     ]
    }
   ],
   "source": [
    "class Probe(nn.Module):\n",
    "    def __init__(self, model_dim):\n",
    "        super().__init__()\n",
    "        self.layer = nn.Linear(model_dim, 2)\n",
    "    \n",
    "    def forward(self, reprs):\n",
    "        return self.layer(reprs)\n",
    "    \n",
    "    def predict(self, good_reprs, bad_reprs):\n",
    "        return (self(good_reprs)[:, 1] > self(bad_reprs)[:, 1]).float()\n",
    "    \n",
    "    def l1(self):\n",
    "        return torch.norm(self.layer.weight, 1)\n",
    "    \n",
    "@torch.no_grad()\n",
    "def eval_probe(probe, data_processed):\n",
    "    return probe.predict(\n",
    "        data_processed[\"good_repr\"],\n",
    "        data_processed[\"bad_repr\"]\n",
    "    ).mean().item()\n",
    "\n",
    "def train_probe(train_data_processed, val_data_processed, l1=0):\n",
    "    n_train, model_dim = train_data_processed[\"good_repr\"].shape\n",
    "    probe = Probe(model_dim).float().cuda()\n",
    "    objective = nn.CrossEntropyLoss()\n",
    "    opt = optim.Adam(probe.parameters(), lr=0.003)\n",
    "    for i in range(200):\n",
    "        loss_good = objective(probe(train_data_processed[\"good_repr\"][:-N_DEV].float()), torch.ones(n_train-N_DEV).long().cuda())\n",
    "        loss_bad = objective(probe(train_data_processed[\"bad_repr\"][:-N_DEV].float()), torch.zeros(n_train-N_DEV).long().cuda())\n",
    "        loss = loss_good + loss_bad + l1 * probe.l1()\n",
    "        opt.zero_grad()\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        if False:\n",
    "            if (i+1) % 10  == 0:\n",
    "                print(loss.item())\n",
    "    return probe.half()\n",
    "\n",
    "#def plot_accuracy(model_preds, probe_preds):\n",
    "#    df = pd.DataFrame({\"model\": model_preds > 0.5, \"probe\": probe_preds > 0.5})\n",
    "#    sns.barplot(data=df)\n",
    "#    plt.ylim(0.5, 1)\n",
    "#    plt.show()\n",
    "    \n",
    "\n",
    "probes = {}\n",
    "probe_preds = {}\n",
    "probe_dev_preds = {}\n",
    "for dataset in data:\n",
    "    print(dataset)\n",
    "    probe = train_probe(data_processed[dataset][\"train\"], data_processed[dataset][\"valid\"])\n",
    "    probes[dataset] = probe\n",
    "    probe_preds[dataset] = torch.sigmoid(\n",
    "        probe(data_processed[dataset][\"valid\"][\"good_repr\"])[:, 1]\n",
    "        - probe(data_processed[dataset][\"valid\"][\"bad_repr\"])[:, 1]\n",
    "    ).detach().cpu().numpy()\n",
    "    probe_dev_preds[dataset] = torch.sigmoid(\n",
    "        probe(data_processed[dataset][\"train\"][\"good_repr\"][-N_DEV:])[:, 1]\n",
    "        - probe(data_processed[dataset][\"train\"][\"bad_repr\"][-N_DEV:])[:, 1]\n",
    "    ).detach().cpu().numpy()\n",
    "\n",
    "    print(\"probe acc\", eval_probe(probe, data_processed[dataset][\"valid\"]))\n",
    "    print(\"model acc\", data_processed[dataset][\"valid\"][\"correct\"].float().mean().item())\n",
    "    #plot_accuracy(\n",
    "    #    torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()),\n",
    "    #    probe_preds[dataset]\n",
    "    #)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Probes and predictions are sensitive to formatting!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qa\n",
      "boolq\n",
      "probe acc 0.6360856294631958\n",
      "model acc 0.6140672564506531\n",
      "\n",
      "sciq\n",
      "probe acc 0.9040000438690186\n",
      "model acc 0.8740000128746033\n",
      "\n",
      "creak\n",
      "probe acc 0.7381473183631897\n",
      "model acc 0.5069292783737183\n",
      "\n",
      "\n",
      "yn\n",
      "boolq\n",
      "probe acc 0.6174311637878418\n",
      "model acc 0.43180426955223083\n",
      "\n",
      "sciq\n",
      "probe acc 0.0\n",
      "model acc 0.0\n",
      "\n",
      "creak\n",
      "probe acc 0.7359591722488403\n",
      "model acc 0.470459520816803\n",
      "\n",
      "\n",
      "cap\n",
      "boolq\n",
      "probe acc 0.635168194770813\n",
      "model acc 0.621100902557373\n",
      "\n",
      "sciq\n",
      "probe acc 0.8980000615119934\n",
      "model acc 0.8370000123977661\n",
      "\n",
      "creak\n",
      "probe acc 0.6965718269348145\n",
      "model acc 0.5076586604118347\n",
      "\n",
      "\n",
      "dot\n",
      "boolq\n",
      "probe acc 0.6348623633384705\n",
      "model acc 0.48990824818611145\n",
      "\n",
      "sciq\n",
      "probe acc 0.9000000357627869\n",
      "model acc 0.8479999899864197\n",
      "\n",
      "creak\n",
      "probe acc 0.7293946146965027\n",
      "model acc 0.50255286693573\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf_to_yn = lambda a: \"yes\" if a == \"true\" else \"no\"\n",
    "\n",
    "formatters = {\n",
    "    \"qa\": lambda q, ag, ab: (f\"Q: {q} A:\", ag, ab),\n",
    "    \"yn\": lambda q, ag, ab: (q,  tf_to_yn(ag), tf_to_yn(ab)),\n",
    "    \"cap\": lambda q, ag, ab: (q, ag.capitalize(), ab.capitalize()),\n",
    "    \"dot\": lambda q, ag, ab: (q, ag + \".\", ab + \".\"),\n",
    "}\n",
    "batch_format = lambda formatter, dataset: [formatter(*datum) for datum in dataset]\n",
    "\n",
    "for format_name, formatter in formatters.items():\n",
    "    data_fmt = {\n",
    "        dataset_name: {\n",
    "            split_name: batch_format(formatter, loader(split_name)) \n",
    "            for split_name in [\"train\", \"valid\"]\n",
    "        }\n",
    "        for dataset_name, loader in [(\"boolq\", load_boolq), (\"sciq\", load_sciq), (\"creak\", load_creak)]\n",
    "    }\n",
    "    data_processed_fmt = process_data(data_fmt, f\"_{format_name}\")\n",
    "\n",
    "    print(format_name)\n",
    "    for dataset in data_fmt:\n",
    "        print(dataset)\n",
    "        n_train, model_dim = data_processed_fmt[dataset][\"train\"][\"good_repr\"].shape\n",
    "        probe = train_probe(data_processed_fmt[dataset][\"train\"], data_processed_fmt[dataset][\"valid\"])\n",
    "        print(\"probe acc\", eval_probe(probe, data_processed_fmt[dataset][\"valid\"]))\n",
    "        print(\"model acc\", data_processed_fmt[dataset][\"valid\"][\"correct\"].float().mean().item())\n",
    "        print()\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "boolq\n",
      "probe acc 0.6266055107116699\n",
      "probe sparse 3841 8192\n",
      "model acc 0.6149846911430359\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO9klEQVR4nO3da2wUhb/G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIi6IFShGqtGznvODQ/2labtOn3U77/bxiZ6Yzv437dWe63d2UIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqIrbhfJkM6fP69EIpHsMdAKXLhwQdfyxz+RDKl79+6SpG+++SbJk6Cly8zMvKbtuEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAgxYTUnV1das8NpqHtGQP4BKLxbR69WqVlpY26XG7deump556qkmPieanxYQkSaWlpfr999+TPQZaoRZzagckEyEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2kM2fO6Pjx4+7dAs1a6JBmzpxZ7/L9+/dr/PjxysnJCT0UEDWhQ/r111/rXT5q1Cht27ZN69evDz0UEDWhQ0pJSbnsuoqKCp0/fz7sroHIua53yC5dulTLli2ruT1w4MDLbjt27NjwUwERc10h3XvvvZKkIAi0atUqTZkype4O09LUs2dPPfroo54JgQi47pAuxXT48GHNmDGjUYYCoib0NdJHH310xfU7duwIu2sgchrtBdns7OzG2jXQ7IT+OK6TJ09qwYIF2rFjh06cOMGHJKJVCx3SW2+9pZKSEo0ZM0adO3dWLPafJ7dLv4wAWovQIf3www/6+uuvddNNN9W7Ph6Phx4KiJrQ10g9e/a8bESS9MEHH4TdNRA5oUMaP368Nm3adNn1zz33XNhdA5ET+tSuuLhYubm5+vjjj9WnTx+1b9++1voff/yxwcMBURE6pA0bNqh79+46e/as9u/fX2d9RUVFgwYDoiR0SP369dPatWsvuz4rKyvsroHICX2N9Pbbb19xPS/IojUJHdKIESOuuH7nzp1hdw1ETuhTu6t9odfKlSs1derUsLsHIiV0SGPGjLnim/uA1iR0SD169NBrr71Wa9m5c+dUUlKinTt3atq0aQ0eDoiK0CFNmDBBEydOrHddSUmJli9fHnooIGpC/7Jh9uzZl13Xt29f7d27N+yugchplPcjHTp0SKdOnWqMXQPNUuhTu2effbbOsiAIVF5ersOHD+vpp59u0GBAlIQO6cCBAxo8eHCtZampqerfv7+mTZt22esnoCUKHVKvXr2Ul5fnnAWIrNDXSF988YVxDCDaQj8jderUSZJUVFSkbdu2qaysTJ07d9YDDzxw1T8fAlqa0CElEgnNnj1bGzduVBAENctzcnL02GOPacGCBbU+xwFoyUKH9Mknn2jXrl2aM2eOMjIy1KlTJ50+fVp79+7V559/rpycHL3yyivOWYFmK3RI69evV15ennr16lVreUZGhkaPHq3p06cTElqN0OdesVisTkSX9OrVS6mpqaGHAqImdEiVlZUqKyurd92pU6f4Whe0KqFDyszM1AsvvKAtW7bo5MmTunDhgk6ePKnNmzfrpZde0iOPPOKcE2jWQl8jzZw5Uy+++KJeffXVOu9LGjZsWJ23WAAtWeiQOnTooPz8fG3YsEHbt2+veR1p1KhReuKJJ5SWFnrXQOSEfrSXlZVp9+7d6tKlixYuXFizfOPGjTpz5oy6dOliGRCIgtDXSPn5+Zo5c6bWrVtXa/n333+vrKws/fLLLw0eDoiK0M9IW7Zs0WeffaaRI0fWWj5v3jyNHj1a2dnZysnJafCAQBSEfkYKgqBORJc8/PDDKi0tDT0UEDWhQyovL2/QeqAlCR1S3759tXjxYiUSiVrLE4mEFi1apD59+jR4OCAqGvQ60tSpU7V69WoNHDhQ6enpKi8vV3Fxsc6ePcunCKFVCR3S4MGDlZeXp+zsbH377beqrq5WLBbT8OHDNXv2bA0aNMg5J9CsNehV0yFDhigvL0///vuvysvLlZ6ernbt2rlmAyLD8ucH7dq1IyC0aryFFTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkNAsBUEQqWPz1eNollJSUvTXX3+pqqqqSY/bpk0bde/e/bp/jpDQbFVVVamysjLZY1wTTu0AA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAipkQXV1a3y2K1NWrIHaOlSYjEdWZ+tf04cbdLjtu96p3o/ObtJj9maEVIT+OfEUf3zZ0myx0Aj4tSuFUsk8dQvmcduDDwjtWKpsZjm/89/62h5WZMe9870zvqvhx5p0mM2NkJq5Y6Wl+mXkyeSPUbkcWoHGKQEQRAke4jrNWTIECUSCfXo0aPW8nPnzimRSDTpLKmpqbrxxhuvuM2FinIFiQtNNNFFKalpSuuQftXtTv/7jy408fVKWiymTu3aX3W7pv5veUlqamrNv//44w+lpqbqwIEDV/yZSJ7a3XDDDaqsrKyz/GoP6GS5lgd0slzLAzpZ/v8DOlnS0tLUtm3bq24XyWckoLnhGgkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQIiIej2vJkiWKx+PJHgX1IKSIOHbsmJYuXapjx44lexTUg5AAAz5FKAJyc3O1atUqHT16VHfeeac6dOigs2fP6tSpU6qoqFCfPn00depUPfPMMyoqKtK8efNUUlKijIwMjRkzRgUFBSopKdGbb76p4uJiHTp0SPF4XMOHD9fcuXN122231RwrCAJ9+eWXWrlypSSpqqpKY8eO1euvv6727ZvvR3clXYBIKCwsDAYMGBAUFhbWLNu3b18wYMCAYPXq1bW23bp1azB9+vSa27/99lswYMCAYOTIkcGePXuCIAiC06dPB1lZWcG4ceOCqqqqmm3fe++9YNiwYcG+ffuCIAiC0tLSYMKECcHLL7/ciPcu+ji1i7ChQ4eqf//+WrNmTa3lBQUFmjx5cp3tMzMzlZGRIUlKT0/XjBkzdOTIEa1du1aSdPToUeXl5WnixIkaOnSoJKlr166aPn26tm7dqqKioka9P1FGSBE3adIk7dmzR4cPH5YklZeXa/fu3Ro9enSdbe+6665aty/FsmfPHknSd999p+rqag0fPrzenyssLLTP31JE8iOL8R9PPvmkFi5cqIKCAr3xxhv66quvNG7cOLVp06bOth07dqx1Oz394kcp//nnn5KksrKLX++yaNEiffrppzXbVVdXq2vXrqqoqGisuxF5hBRxXbt21YMPPqi1a9dq1qxZWrNmjebPn1/vtn///Xet26dPn5Yk3XrrrZKkzp07S5LmzJmjsWPHNt7QLRCndhGRlnbx/3nB//2S9eDBgzpy5IgkafLkySotLVVubq5isVidU7hLfv7551q39+/fL0m6++67JUn333+/YrGYiouL6/zsu+++q127dnnuTAtESBHRs2dPpaSk6Pjx45IuPrD37dsnSXrooYd0yy23aNGiRZo0adJl97Fjxw7t3btX0sVrqWXLlql3797KysqSJN1xxx16/vnnlZ+fr4MHD0q6GO6KFSu0efNmDRo0qPHuYMTxOlKELFmyRAUFBerYsaN69+6tDz/8sOYrR95//30tX75c27dv180331zr5+LxuDIzMzV37lz99NNPOnjwoOLxuO655556X0fKz8/XihUrlEgk1L59e/Xr10+zZs3S7bff3qT3N0oIqYVYt26dtm7dqoULF9ZZdymk+fPnX/EZC+FxatdCbNq0qd7XjtA0CCnCpkyZooqKCpWUlCgej+u+++5L9kitFr/+jrA2bdro8ccfV3p6ut555x2lpKTU2SY3N1cFBQWSpMWLF6uwsFDZ2dlNPWqLxzUSYMCpHWBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYPC/WvFQp8SvArkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "boolq\n",
      "probe acc 0.6214067339897156\n",
      "probe sparse 362 8192\n",
      "model acc 0.6149846911430359\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO7UlEQVR4nO3da2wUhb/G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIi6IFShGqtGznvODQ/2labtOn3U77/bxyZ2Znfhv3y850u9uUIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqIrbhfJkM6fP69EIpHsMdAKXLhwQdfyyz+RDKl79+6SpG+++SbJk6Cly8zMvKbtuEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwaDEhVVdXt8pjo3lIS/YALrFYTKtXr1ZpaWmTHrdbt2566qmnmvSYaH5aTEiSVFpaqt9//z3ZY6AVajGndkAyERJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2kM2fO6Pjx4+7dAs1a6JBmzpxZ7/L9+/dr/PjxysnJCT0UEDWhQ/r111/rXT5q1Cht27ZN69evDz0UEDWhQ0pJSbnsuoqKCp0/fz7sroHIua7vbFi6dKmWLVtWc3vgwIGX3Xbs2LHhpwIi5rpCuvfeeyVJQRBo1apVmjJlSt0dpqWpZ8+eevTRRz0TAhFw3SFdiunw4cOaMWNGowwFRE3oa6SPPvroiut37NgRdtdA5DTaG7LZ2dmNtWug2Qn9BZEnT57UggULtGPHDp04cYKv7UWrFjqkt956SyUlJRozZow6d+6sWOw/L26XfhgBtBahQ/rhhx/09ddf66abbqp3fTweDz0UEDWhr5F69ux52Ygk6YMPPgi7ayByQoc0fvx4bdq06bLrn3vuubC7BiIn9KldcXGxcnNz9fHHH6tPnz5q3759rfU//vhjg4cDoiJ0SBs2bFD37t119uxZ7d+/v876ioqKBg0GREnokPr166e1a9dedn1WVlbYXQORE/oa6e23377iet6QRWsSOqQRI0Zccf3OnTvD7hqInNCndlf7E5MrV67U1KlTw+4eiJTQIY0ZM+aKH+4DWpPQIfXo0UOvvfZarWXnzp1TSUmJdu7cqWnTpjV4OCAqQoc0YcIETZw4sd51JSUlWr58eeihgKgJ/cOG2bNnX3Zd3759tXfv3rC7BiKnUT6PdOjQIZ06daoxdg00S6FP7Z599tk6y4IgUHl5uQ4fPqynn366QYMBURI6pAMHDmjw4MG1lqWmpqp///6aNm3aZa+fgJYodEi9evVSXl6ecxYgskJfI33xxRfGMYBoC/2K1KlTJ0lSUVGRtm3bprKyMnXu3FkPPPDAVX99CGhpQoeUSCQ0e/Zsbdy4UUEQ1CzPycnRY489pgULFtT6HgegJQsd0ieffKJdu3Zpzpw5ysjIUKdOnXT69Gnt3btXn3/+uXJycvTKK684ZwWardAhrV+/Xnl5eerVq1et5RkZGRo9erSmT59OSGg1Qp97xWKxOhFd0qtXL6WmpoYeCoia0CFVVlaqrKys3nWnTp3iz7qgVQkdUmZmpl544QVt2bJFJ0+e1IULF3Ty5Elt3rxZL730kh555BHnnECzFvoaaebMmXrxxRf16quv1vlc0rBhw+p8xAJoyUKH1KFDB+Xn52vDhg3avn17zftIo0aN0hNPPKG0tNC7BiIn9LO9rKxMu3fvVpcuXbRw4cKa5Rs3btSZM2fUpUsXy4BAFIS+RsrPz9fMmTO1bt26Wsu///57ZWVl6ZdffmnwcEBUhH5F2rJliz777DONHDmy1vJ58+Zp9OjRys7OVk5OToMHBKIg9CtSEAR1Irrk4YcfVmlpaeihgKgJHVJ5eXmD1gMtSeiQ+vbtq8WLFyuRSNRankgktGjRIvXp06fBwwFR0aD3kaZOnarVq1dr4MCBSk9PV3l5uYqLi3X27Fm+RQitSuiQBg8erLy8PGVnZ+vbb79VdXW1YrGYhg8frtmzZ2vQoEHOOYFmrUHvmg4ZMkR5eXn6999/VV5ervT0dLVr1841GxAZll8/aNeuHQGhVeMjrIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEpqlIAgidWz+PiWapZSUFP3111+qqqpq0uO2adNG3bt3v+77ERKaraqqKlVWViZ7jGvCqR1gQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASI0sqK5ulcdubdKSPUBLlxKL6cj6bP1z4miTHrd91zvV+8nZV9wmUV2t1Fhy/i1N5rEbAyE1gX9OHNU/f5Yke4w6UmMxzf+f/9bR8rImPe6d6Z31Xw890qTHbGwpQRAEyR7ieg0ZMkSJREI9evSotfzcuXNKJBJNOktqaqpuvPHGK25zoaJcQeJCE010UUpqmtI6pF91u9P//qMLTXwKmBaLqVO79lfdrqn/X16Smppa899//PGHUlNTdeDAgSveJ5KvSDfccIMqKyvrLL/aEzpZruUJnSzX8oROlv//hE6WtLQ0tW3b9qrbRfIVCWhuWs7VHpBEhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBBSRMTjcS1ZskTxeDzZo6AehBQRx44d09KlS3Xs2LFkj4J6EBJgwLcIRUBubq5WrVqlo0eP6s4771SHDh109uxZnTp1ShUVFerTp4+mTp2qZ555RkVFRZo3b55KSkqUkZGhMWPGqKCgQCUlJXrzzTdVXFysQ4cOKR6Pa/jw4Zo7d65uu+22mmMFQaAvv/xSK1eulCRVVVVp7Nixev3119W+ffP96q6kCxAJhYWFwYABA4LCwsKaZfv27QsGDBgQrF69uta2W7duDaZPn15z+7fffgsGDBgQjBw5MtizZ08QBEFw+vTpICsrKxg3blxQVVVVs+17770XDBs2LNi3b18QBEFQWloaTJgwIXj55Zcb8dFFH6d2ETZ06FD1799fa9asqbW8oKBAkydPrrN9ZmamMjIyJEnp6emaMWOGjhw5orVr10qSjh49qry8PE2cOFFDhw6VJHXt2lXTp0/X1q1bVVRU1KiPJ8oIKeImTZqkPXv26PDhw5Kk8vJy7d69W6NHj66z7V133VXr9qVY9uzZI0n67rvvVF1dreHDh9d7v8LCQvv8LUUkv7IY//Hkk09q4cKFKigo0BtvvKGvvvpK48aNU5s2beps27Fjx1q309MvfpXyn3/+KUkqK7v4ZfqLFi3Sp59+WrNddXW1unbtqoqKisZ6GJFHSBHXtWtXPfjgg1q7dq1mzZqlNWvWaP78+fVu+/fff9e6ffr0aUnSrbfeKknq3LmzJGnOnDkaO3Zs4w3dAnFqFxFpaRf/zQv+74esBw8e1JEjRyRJkydPVmlpqXJzcxWLxeqcwl3y888/17q9f/9+SdLdd98tSbr//vsVi8VUXFxc577vvvuudu3a5XkwLRAhRUTPnj2VkpKi48ePS7r4xN63b58k6aGHHtItt9yiRYsWadKkSZfdx44dO7R3715JF6+lli1bpt69eysrK0uSdMcdd+j5559Xfn6+Dh48KOliuCtWrNDmzZs1aNCgxnuAEcf7SBGyZMkSFRQUqGPHjurdu7c+/PDDmj858v7772v58uXavn27br755lr3i8fjyszM1Ny5c/XTTz/p4MGDisfjuueee+p9Hyk/P18rVqxQIpFQ+/bt1a9fP82aNUu33357kz7eKCGkFmLdunXaunWrFi5cWGfdpZDmz59/xVcshMepXQuxadOmet87QtMgpAibMmWKKioqVFJSong8rvvuuy/ZI7Va/Pg7wtq0aaPHH39c6enpeuedd5SSklJnm9zcXBUUFEiSFi9erMLCQmVnZzf1qC0e10iAAad2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAwf8CtAhLOm3vUEMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "boolq\n",
      "probe acc 0.6217125058174133\n",
      "probe sparse 43 8192\n",
      "model acc 0.6149846911430359\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO7UlEQVR4nO3da2wUhb/G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIi6IFShGqtGznvODQ/2labtOn3U77/bxyZ2Znfhv3y850u9uUIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqIrbhfJkM6fP69EIpHsMdAKXLhwQdfyyz+RDKl79+6SpG+++SbJk6Cly8zMvKbtuEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwaDEhVVdXt8pjo3lIS/YALrFYTKtXr1ZpaWmTHrdbt2566qmnmvSYaH5aTEiSVFpaqt9//z3ZY6AVajGndkAyERJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2kM2fO6Pjx4+7dAs1a6JBmzpxZ7/L9+/dr/PjxysnJCT0UEDWhQ/r111/rXT5q1Cht27ZN69evDz0UEDWhQ0pJSbnsuoqKCp0/fz7sroHIua7vbFi6dKmWLVtWc3vgwIGX3Xbs2LHhpwIi5rpCuvfeeyVJQRBo1apVmjJlSt0dpqWpZ8+eevTRRz0TAhFw3SFdiunw4cOaMWNGowwFRE3oa6SPPvroiut37NgRdtdA5DTaG7LZ2dmNtWug2Qn9BZEnT57UggULtGPHDp04cYKv7UWrFjqkt956SyUlJRozZow6d+6sWOw/L26XfhgBtBahQ/rhhx/09ddf66abbqp3fTweDz0UEDWhr5F69ux52Ygk6YMPPgi7ayByQoc0fvx4bdq06bLrn3vuubC7BiIn9KldcXGxcnNz9fHHH6tPnz5q3759rfU//vhjg4cDoiJ0SBs2bFD37t119uxZ7d+/v876ioqKBg0GREnokPr166e1a9dedn1WVlbYXQORE/oa6e23377iet6QRWsSOqQRI0Zccf3OnTvD7hqInNCndlf7E5MrV67U1KlTw+4eiJTQIY0ZM+aKH+4DWpPQIfXo0UOvvfZarWXnzp1TSUmJdu7cqWnTpjV4OCAqQoc0YcIETZw4sd51JSUlWr58eeihgKgJ/cOG2bNnX3Zd3759tXfv3rC7BiKnUT6PdOjQIZ06daoxdg00S6FP7Z599tk6y4IgUHl5uQ4fPqynn366QYMBURI6pAMHDmjw4MG1lqWmpqp///6aNm3aZa+fgJYodEi9evVSXl6ecxYgskJfI33xxRfGMYBoC/2K1KlTJ0lSUVGRtm3bprKyMnXu3FkPPPDAVX99CGhpQoeUSCQ0e/Zsbdy4UUEQ1CzPycnRY489pgULFtT6HgegJQsd0ieffKJdu3Zpzpw5ysjIUKdOnXT69Gnt3btXn3/+uXJycvTKK684ZwWardAhrV+/Xnl5eerVq1et5RkZGRo9erSmT59OSGg1Qp97xWKxOhFd0qtXL6WmpoYeCoia0CFVVlaqrKys3nWnTp3iz7qgVQkdUmZmpl544QVt2bJFJ0+e1IULF3Ty5Elt3rxZL730kh555BHnnECzFvoaaebMmXrxxRf16quv1vlc0rBhw+p8xAJoyUKH1KFDB+Xn52vDhg3avn17zftIo0aN0hNPPKG0tNC7BiIn9LO9rKxMu3fvVpcuXbRw4cKa5Rs3btSZM2fUpUsXy4BAFIS+RsrPz9fMmTO1bt26Wsu///57ZWVl6ZdffmnwcEBUhH5F2rJliz777DONHDmy1vJ58+Zp9OjRys7OVk5OToMHBKIg9CtSEAR1Irrk4YcfVmlpaeihgKgJHVJ5eXmD1gMtSeiQ+vbtq8WLFyuRSNRankgktGjRIvXp06fBwwFR0aD3kaZOnarVq1dr4MCBSk9PV3l5uYqLi3X27Fm+RQitSuiQBg8erLy8PGVnZ+vbb79VdXW1YrGYhg8frtmzZ2vQoEHOOYFmrUHvmg4ZMkR5eXn6999/VV5ervT0dLVr1841GxAZll8/aNeuHQGhVeMjrIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEhGYpCIJIHZs/q4dmKSUlRX/99Zeqqqqa9Lht2rRR9+7dr/t+hIRmq6qqSpWVlcke45pwagcYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASI0sqK5ulcdubdKSPUBLlxKL6cj6bP1z4miTHrd91zvV+8nZV9wmUV2t1Fhy/i1N5rEbAyE1gX9OHNU/f5Yke4w6UmMxzf+f/9bR8rImPe6d6Z31Xw890qTHbGwpQRAEyR7ieg0ZMkSJREI9evSotfzcuXNKJBJNOktqaqpuvPHGK25zoaJcQeJCE010UUpqmtI6pF91u9P//qMLTXwKmBaLqVO79lfdrqn/X16Smppa899//PGHUlNTdeDAgSveJ5KvSDfccIMqKyvrLL/aEzpZruUJnSzX8oROlv//hE6WtLQ0tW3b9qrbRfIVCWhuWs7VHpBEhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBBSRMTjcS1ZskTxeDzZo6AehBQRx44d09KlS3Xs2LFkj4J6EBJgwLcIRUBubq5WrVqlo0eP6s4771SHDh109uxZnTp1ShUVFerTp4+mTp2qZ555RkVFRZo3b55KSkqUkZGhMWPGqKCgQCUlJXrzzTdVXFysQ4cOKR6Pa/jw4Zo7d65uu+22mmMFQaAvv/xSK1eulCRVVVVp7Nixev3119W+ffP96q6kCxAJhYWFwYABA4LCwsKaZfv27QsGDBgQrF69uta2W7duDaZPn15z+7fffgsGDBgQjBw5MtizZ08QBEFw+vTpICsrKxg3blxQVVVVs+17770XDBs2LNi3b18QBEFQWloaTJgwIXj55Zcb8dFFH6d2ETZ06FD1799fa9asqbW8oKBAkydPrrN9ZmamMjIyJEnp6emaMWOGjhw5orVr10qSjh49qry8PE2cOFFDhw6VJHXt2lXTp0/X1q1bVVRU1KiPJ8oIKeImTZqkPXv26PDhw5Kk8vJy7d69W6NHj66z7V133VXr9qVY9uzZI0n67rvvVF1dreHDh9d7v8LCQvv8LUUkv7IY//Hkk09q4cKFKigo0BtvvKGvvvpK48aNU5s2beps27Fjx1q309MvfpXyn3/+KUkqK7v4ZfqLFi3Sp59+WrNddXW1unbtqoqKisZ6GJFHSBHXtWtXPfjgg1q7dq1mzZqlNWvWaP78+fVu+/fff9e6ffr0aUnSrbfeKknq3LmzJGnOnDkaO3Zs4w3dAnFqFxFpaRf/zQv+74esBw8e1JEjRyRJkydPVmlpqXJzcxWLxeqcwl3y888/17q9f/9+SdLdd98tSbr//vsVi8VUXFxc577vvvuudu3a5XkwLRAhRUTPnj2VkpKi48ePS7r4xN63b58k6aGHHtItt9yiRYsWadKkSZfdx44dO7R3715JF6+lli1bpt69eysrK0uSdMcdd+j5559Xfn6+Dh48KOliuCtWrNDmzZs1aNCgxnuAEcf7SBGyZMkSFRQUqGPHjurdu7c+/PDDmj858v7772v58uXavn27br755lr3i8fjyszM1Ny5c/XTTz/p4MGDisfjuueee+p9Hyk/P18rVqxQIpFQ+/bt1a9fP82aNUu33357kz7eKCGkFmLdunXaunWrFi5cWGfdpZDmz59/xVcshMepXQuxadOmet87QtMgpAibMmWKKioqVFJSong8rvvuuy/ZI7Va/Pg7wtq0aaPHH39c6enpeuedd5SSklJnm9zcXBUUFEiSFi9erMLCQmVnZzf1qC0e10iAAad2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAwf8CcF1LOlx57hUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "boolq\n",
      "probe acc 0.6217125058174133\n",
      "probe sparse 43 8192\n",
      "model acc 0.6149846911430359\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO7UlEQVR4nO3da2wUhb/G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIi6IFShGqtGznvODQ/2labtOn3U77/bxyZ2Znfhv3y850u9uUIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqIrbhfJkM6fP69EIpHsMdAKXLhwQdfyyz+RDKl79+6SpG+++SbJk6Cly8zMvKbtuEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwaDEhVVdXt8pjo3lIS/YALrFYTKtXr1ZpaWmTHrdbt2566qmnmvSYaH5aTEiSVFpaqt9//z3ZY6AVajGndkAyERJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2kM2fO6Pjx4+7dAs1a6JBmzpxZ7/L9+/dr/PjxysnJCT0UEDWhQ/r111/rXT5q1Cht27ZN69evDz0UEDWhQ0pJSbnsuoqKCp0/fz7sroHIua7vbFi6dKmWLVtWc3vgwIGX3Xbs2LHhpwIi5rpCuvfeeyVJQRBo1apVmjJlSt0dpqWpZ8+eevTRRz0TAhFw3SFdiunw4cOaMWNGowwFRE3oa6SPPvroiut37NgRdtdA5DTaG7LZ2dmNtWug2Qn9BZEnT57UggULtGPHDp04cYKv7UWrFjqkt956SyUlJRozZow6d+6sWOw/L26XfhgBtBahQ/rhhx/09ddf66abbqp3fTweDz0UEDWhr5F69ux52Ygk6YMPPgi7ayByQoc0fvx4bdq06bLrn3vuubC7BiIn9KldcXGxcnNz9fHHH6tPnz5q3759rfU//vhjg4cDoiJ0SBs2bFD37t119uxZ7d+/v876ioqKBg0GREnokPr166e1a9dedn1WVlbYXQORE/oa6e23377iet6QRWsSOqQRI0Zccf3OnTvD7hqInNCndlf7E5MrV67U1KlTw+4eiJTQIY0ZM+aKH+4DWpPQIfXo0UOvvfZarWXnzp1TSUmJdu7cqWnTpjV4OCAqQoc0YcIETZw4sd51JSUlWr58eeihgKgJ/cOG2bNnX3Zd3759tXfv3rC7BiKnUT6PdOjQIZ06daoxdg00S6FP7Z599tk6y4IgUHl5uQ4fPqynn366QYMBURI6pAMHDmjw4MG1lqWmpqp///6aNm3aZa+fgJYodEi9evVSXl6ecxYgskJfI33xxRfGMYBoC/2K1KlTJ0lSUVGRtm3bprKyMnXu3FkPPPDAVX99CGhpQoeUSCQ0e/Zsbdy4UUEQ1CzPycnRY489pgULFtT6HgegJQsd0ieffKJdu3Zpzpw5ysjIUKdOnXT69Gnt3btXn3/+uXJycvTKK684ZwWardAhrV+/Xnl5eerVq1et5RkZGRo9erSmT59OSGg1Qp97xWKxOhFd0qtXL6WmpoYeCoia0CFVVlaqrKys3nWnTp3iz7qgVQkdUmZmpl544QVt2bJFJ0+e1IULF3Ty5Elt3rxZL730kh555BHnnECzFvoaaebMmXrxxRf16quv1vlc0rBhw+p8xAJoyUKH1KFDB+Xn52vDhg3avn17zftIo0aN0hNPPKG0tNC7BiIn9LO9rKxMu3fvVpcuXbRw4cKa5Rs3btSZM2fUpUsXy4BAFIS+RsrPz9fMmTO1bt26Wsu///57ZWVl6ZdffmnwcEBUhH5F2rJliz777DONHDmy1vJ58+Zp9OjRys7OVk5OToMHBKIg9CtSEAR1Irrk4YcfVmlpaeihgKgJHVJ5eXmD1gMtSeiQ+vbtq8WLFyuRSNRankgktGjRIvXp06fBwwFR0aD3kaZOnarVq1dr4MCBSk9PV3l5uYqLi3X27Fm+RQitSuiQBg8erLy8PGVnZ+vbb79VdXW1YrGYhg8frtmzZ2vQoEHOOYFmrUHvmg4ZMkR5eXn6999/VV5ervT0dLVr1841GxAZll8/aNeuHQGhVeMjrIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEhGYpCIJIHZs/q4dmKSUlRX/99Zeqqqqa9Lht2rRR9+7dr/t+hIRmq6qqSpWVlcke45pwagcYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASI0sqK5ulcdubdKSPUBLlxKL6cj6bP1z4miTHrd91zvV+8nZV9wmUV2t1Fhy/i1N5rEbAyE1gX9OHNU/f5Yke4w6UmMxzf+f/9bR8rImPe6d6Z31Xw890qTHbGwpQRAEyR7ieg0ZMkSJREI9evSotfzcuXNKJBJNOktqaqpuvPHGK25zoaJcQeJCE010UUpqmtI6pF91u9P//qMLTXwKmBaLqVO79lfdrqn/X16Smppa899//PGHUlNTdeDAgSveJ5KvSDfccIMqKyvrLL/aEzpZruUJnSzX8oROlv//hE6WtLQ0tW3b9qrbRfIVCWhuWs7VHpBEhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBBSRMTjcS1ZskTxeDzZo6AehBQRx44d09KlS3Xs2LFkj4J6EBJgwLcIRUBubq5WrVqlo0eP6s4771SHDh109uxZnTp1ShUVFerTp4+mTp2qZ555RkVFRZo3b55KSkqUkZGhMWPGqKCgQCUlJXrzzTdVXFysQ4cOKR6Pa/jw4Zo7d65uu+22mmMFQaAvv/xSK1eulCRVVVVp7Nixev3119W+ffP96q6kCxAJhYWFwYABA4LCwsKaZfv27QsGDBgQrF69uta2W7duDaZPn15z+7fffgsGDBgQjBw5MtizZ08QBEFw+vTpICsrKxg3blxQVVVVs+17770XDBs2LNi3b18QBEFQWloaTJgwIXj55Zcb8dFFH6d2ETZ06FD1799fa9asqbW8oKBAkydPrrN9ZmamMjIyJEnp6emaMWOGjhw5orVr10qSjh49qry8PE2cOFFDhw6VJHXt2lXTp0/X1q1bVVRU1KiPJ8oIKeImTZqkPXv26PDhw5Kk8vJy7d69W6NHj66z7V133VXr9qVY9uzZI0n67rvvVF1dreHDh9d7v8LCQvv8LUUkv7IY//Hkk09q4cKFKigo0BtvvKGvvvpK48aNU5s2beps27Fjx1q309MvfpXyn3/+KUkqK7v4ZfqLFi3Sp59+WrNddXW1unbtqoqKisZ6GJFHSBHXtWtXPfjgg1q7dq1mzZqlNWvWaP78+fVu+/fff9e6ffr0aUnSrbfeKknq3LmzJGnOnDkaO3Zs4w3dAnFqFxFpaRf/zQv+74esBw8e1JEjRyRJkydPVmlpqXJzcxWLxeqcwl3y888/17q9f/9+SdLdd98tSbr//vsVi8VUXFxc577vvvuudu3a5XkwLRAhRUTPnj2VkpKi48ePS7r4xN63b58k6aGHHtItt9yiRYsWadKkSZfdx44dO7R3715JF6+lli1bpt69eysrK0uSdMcdd+j5559Xfn6+Dh48KOliuCtWrNDmzZs1aNCgxnuAEcf7SBGyZMkSFRQUqGPHjurdu7c+/PDDmj858v7772v58uXavn27br755lr3i8fjyszM1Ny5c/XTTz/p4MGDisfjuueee+p9Hyk/P18rVqxQIpFQ+/bt1a9fP82aNUu33357kz7eKCGkFmLdunXaunWrFi5cWGfdpZDmz59/xVcshMepXQuxadOmet87QtMgpAibMmWKKioqVFJSong8rvvuuy/ZI7Va/Pg7wtq0aaPHH39c6enpeuedd5SSklJnm9zcXBUUFEiSFi9erMLCQmVnZzf1qC0e10iAAad2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAwf8CcF1LOlx57hUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "boolq\n",
      "probe acc 0.6217125058174133\n",
      "probe sparse 154 8192\n",
      "model acc 0.6149846911430359\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO7UlEQVR4nO3da2wUhb/G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIi6IFShGqtGznvODQ/2labtOn3U77/bxyZ2Znfhv3y850u9uUIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqIrbhfJkM6fP69EIpHsMdAKXLhwQdfyyz+RDKl79+6SpG+++SbJk6Cly8zMvKbtuEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwaDEhVVdXt8pjo3lIS/YALrFYTKtXr1ZpaWmTHrdbt2566qmnmvSYaH5aTEiSVFpaqt9//z3ZY6AVajGndkAyERJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2kM2fO6Pjx4+7dAs1a6JBmzpxZ7/L9+/dr/PjxysnJCT0UEDWhQ/r111/rXT5q1Cht27ZN69evDz0UEDWhQ0pJSbnsuoqKCp0/fz7sroHIua7vbFi6dKmWLVtWc3vgwIGX3Xbs2LHhpwIi5rpCuvfeeyVJQRBo1apVmjJlSt0dpqWpZ8+eevTRRz0TAhFw3SFdiunw4cOaMWNGowwFRE3oa6SPPvroiut37NgRdtdA5DTaG7LZ2dmNtWug2Qn9BZEnT57UggULtGPHDp04cYKv7UWrFjqkt956SyUlJRozZow6d+6sWOw/L26XfhgBtBahQ/rhhx/09ddf66abbqp3fTweDz0UEDWhr5F69ux52Ygk6YMPPgi7ayByQoc0fvx4bdq06bLrn3vuubC7BiIn9KldcXGxcnNz9fHHH6tPnz5q3759rfU//vhjg4cDoiJ0SBs2bFD37t119uxZ7d+/v876ioqKBg0GREnokPr166e1a9dedn1WVlbYXQORE/oa6e23377iet6QRWsSOqQRI0Zccf3OnTvD7hqInNCndlf7E5MrV67U1KlTw+4eiJTQIY0ZM+aKH+4DWpPQIfXo0UOvvfZarWXnzp1TSUmJdu7cqWnTpjV4OCAqQoc0YcIETZw4sd51JSUlWr58eeihgKgJ/cOG2bNnX3Zd3759tXfv3rC7BiKnUT6PdOjQIZ06daoxdg00S6FP7Z599tk6y4IgUHl5uQ4fPqynn366QYMBURI6pAMHDmjw4MG1lqWmpqp///6aNm3aZa+fgJYodEi9evVSXl6ecxYgskJfI33xxRfGMYBoC/2K1KlTJ0lSUVGRtm3bprKyMnXu3FkPPPDAVX99CGhpQoeUSCQ0e/Zsbdy4UUEQ1CzPycnRY489pgULFtT6HgegJQsd0ieffKJdu3Zpzpw5ysjIUKdOnXT69Gnt3btXn3/+uXJycvTKK684ZwWardAhrV+/Xnl5eerVq1et5RkZGRo9erSmT59OSGg1Qp97xWKxOhFd0qtXL6WmpoYeCoia0CFVVlaqrKys3nWnTp3iz7qgVQkdUmZmpl544QVt2bJFJ0+e1IULF3Ty5Elt3rxZL730kh555BHnnECzFvoaaebMmXrxxRf16quv1vlc0rBhw+p8xAJoyUKH1KFDB+Xn52vDhg3avn17zftIo0aN0hNPPKG0tNC7BiIn9LO9rKxMu3fvVpcuXbRw4cKa5Rs3btSZM2fUpUsXy4BAFIS+RsrPz9fMmTO1bt26Wsu///57ZWVl6ZdffmnwcEBUhH5F2rJliz777DONHDmy1vJ58+Zp9OjRys7OVk5OToMHBKIg9CtSEAR1Irrk4YcfVmlpaeihgKgJHVJ5eXmD1gMtSeiQ+vbtq8WLFyuRSNRankgktGjRIvXp06fBwwFR0aD3kaZOnarVq1dr4MCBSk9PV3l5uYqLi3X27Fm+RQitSuiQBg8erLy8PGVnZ+vbb79VdXW1YrGYhg8frtmzZ2vQoEHOOYFmrUHvmg4ZMkR5eXn6999/VV5ervT0dLVr1841GxAZll8/aNeuHQGhVeMjrIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEhGYpCIJIHZs/q4dmKSUlRX/99Zeqqqqa9Lht2rRR9+7dr/t+hIRmq6qqSpWVlcke45pwagcYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASI0sqK5ulcdubdKSPUBLlxKL6cj6bP1z4miTHrd91zvV+8nZV9wmUV2t1Fhy/i1N5rEbAyE1gX9OHNU/f5Yke4w6UmMxzf+f/9bR8rImPe6d6Z31Xw890qTHbGwpQRAEyR7ieg0ZMkSJREI9evSotfzcuXNKJBJNOktqaqpuvPHGK25zoaJcQeJCE010UUpqmtI6pF91u9P//qMLTXwKmBaLqVO79lfdrqn/X16Smppa899//PGHUlNTdeDAgSveJ5KvSDfccIMqKyvrLL/aEzpZruUJnSzX8oROlv//hE6WtLQ0tW3b9qrbRfIVCWhuWs7VHpBEhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBBSRMTjcS1ZskTxeDzZo6AehBQRx44d09KlS3Xs2LFkj4J6EBJgwLcIRUBubq5WrVqlo0eP6s4771SHDh109uxZnTp1ShUVFerTp4+mTp2qZ555RkVFRZo3b55KSkqUkZGhMWPGqKCgQCUlJXrzzTdVXFysQ4cOKR6Pa/jw4Zo7d65uu+22mmMFQaAvv/xSK1eulCRVVVVp7Nixev3119W+ffP96q6kCxAJhYWFwYABA4LCwsKaZfv27QsGDBgQrF69uta2W7duDaZPn15z+7fffgsGDBgQjBw5MtizZ08QBEFw+vTpICsrKxg3blxQVVVVs+17770XDBs2LNi3b18QBEFQWloaTJgwIXj55Zcb8dFFH6d2ETZ06FD1799fa9asqbW8oKBAkydPrrN9ZmamMjIyJEnp6emaMWOGjhw5orVr10qSjh49qry8PE2cOFFDhw6VJHXt2lXTp0/X1q1bVVRU1KiPJ8oIKeImTZqkPXv26PDhw5Kk8vJy7d69W6NHj66z7V133VXr9qVY9uzZI0n67rvvVF1dreHDh9d7v8LCQvv8LUUkv7IY//Hkk09q4cKFKigo0BtvvKGvvvpK48aNU5s2beps27Fjx1q309MvfpXyn3/+KUkqK7v4ZfqLFi3Sp59+WrNddXW1unbtqoqKisZ6GJFHSBHXtWtXPfjgg1q7dq1mzZqlNWvWaP78+fVu+/fff9e6ffr0aUnSrbfeKknq3LmzJGnOnDkaO3Zs4w3dAnFqFxFpaRf/zQv+74esBw8e1JEjRyRJkydPVmlpqXJzcxWLxeqcwl3y888/17q9f/9+SdLdd98tSbr//vsVi8VUXFxc577vvvuudu3a5XkwLRAhRUTPnj2VkpKi48ePS7r4xN63b58k6aGHHtItt9yiRYsWadKkSZfdx44dO7R3715JF6+lli1bpt69eysrK0uSdMcdd+j5559Xfn6+Dh48KOliuCtWrNDmzZs1aNCgxnuAEcf7SBGyZMkSFRQUqGPHjurdu7c+/PDDmj858v7772v58uXavn27br755lr3i8fjyszM1Ny5c/XTTz/p4MGDisfjuueee+p9Hyk/P18rVqxQIpFQ+/bt1a9fP82aNUu33357kz7eKCGkFmLdunXaunWrFi5cWGfdpZDmz59/xVcshMepXQuxadOmet87QtMgpAibMmWKKioqVFJSong8rvvuuy/ZI7Va/Pg7wtq0aaPHH39c6enpeuedd5SSklJnm9zcXBUUFEiSFi9erMLCQmVnZzf1qC0e10iAAad2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAwf8CcF1LOlx57hUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "sciq\n",
      "probe acc 0.8820000290870667\n",
      "probe sparse 3873 8192\n",
      "model acc 0.8379999995231628\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO+klEQVR4nO3deWwUBf/H8U+3BQHRUgQMihJOA+GoQiRGPKAooNEUSAxG8A7RhIgkBnyiIcFo0CIxXDHWGjVtuBIaDoU/ngh5AKVCwx2rYiHigmiBUoQqx3Z+f/Cjz9O0XNNPu532/fqLnZmd+W7ctzvT7W5TgiAIBKBeYskeAGgOCAkwICTAgJAAA0ICDAgJMCAkwICQAIO0ZA8QxtChQ3Xu3Dl17tw52aOgmSsrK1Pr1q1VXFx8xe0iGdLZs2eVSCSSPQZagAsXLuhafvknkiF16dJFkvTNN98keRI0d1lZWde0HddIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGDSbkKqqqlrksdE0pCV7AJdYLKYVK1aorKysUY/buXNnPfXUU416TDQ9zSYkSSorK9ORI0eSPQZaoGZzagckEyEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBvaQTp06paNHj7p3CzRpoUOaNm1ancv37NmjMWPGKDc3N/RQQNSEDunXX3+tc/nw4cO1efNmrVmzJvRQQNSEDiklJeWy6yorK3X27NmwuwYiJ+16Nl60aJEWL15cfbtfv36X3XbUqFHhpwIi5rpCuvfeeyVJQRBo+fLlmjhxYu0dpqWpW7duevTRRz0TAhFw3SFdiunAgQOaOnVqgwwFRE3oa6SPPvroiuu3bt0adtdA5DTYG7I5OTkNtWugybmuU7v/dfz4cc2dO1dbt27VsWPHVFVV5ZwLiJTQIb311lsqLS3VyJEjlZGRoVjsvy9ul34YAbQUoUP64Ycf9PXXX+umm26qc308Hg89FBA1oa+RunXrdtmIJOmDDz4Iu2sgckKHNGbMGK1fv/6y65977rmwuwYiJ/SpXUlJifLy8vTxxx+rZ8+eatu2bY31P/74Y72HA6IidEhr165Vly5ddPr0ae3Zs6fW+srKynoNBkRJ6JB69+6tVatWXXZ9dnZ22F0DkRP6Guntt9++4nrekEVLEjqkoUOHXnH9tm3bwu4aiJzQp3ZHjhy54vply5Zp0qRJYXcPRErokEaOHHnFD/cBLUnokLp27arXXnutxrIzZ86otLRU27Zt0+TJk+s9HBAVoUMaO3asxo0bV+e60tJSLVmyJPRQQNSE/mHDjBkzLruuV69e2rVrV9hdA5HTIJ9H2r9/v06cONEQuwaapNCnds8++2ytZUEQqKKiQgcOHNDTTz9dr8GAKAkd0t69ezVgwIAay1JTU9WnTx9Nnjz5stdPQHMUOqTu3bsrPz/fOQsQWaGvkb744gvjGEC0hX5F6tChgySpuLhYmzdvVnl5uTIyMvTAAw9c9deHgOYmdEiJREIzZszQunXrFARB9fLc3Fw99thjmjt3bo3vcQCas9AhffLJJ9q+fbtmzpypzMxMdejQQSdPntSuXbv0+eefKzc3V6+88opzVqDJCh3SmjVrlJ+fr+7du9dYnpmZqREjRmjKlCmEhBYj9LlXLBarFdEl3bt3V2pqauihgKgJHdK5c+dUXl5e57oTJ07wZ13QooQOKSsrSy+88II2btyo48eP68KFCzp+/Lg2bNigl156SY888ohzTqBJC32NNG3aNL344ot69dVXa30uafDgwbU+YgE0Z6FDateunQoKCrR27Vpt2bKl+n2k4cOH64knnlBaWuhdA5ET+tleXl6uHTt2qGPHjpo3b1718nXr1unUqVPq2LGjZUAgCkJfIxUUFGjatGlavXp1jeXff/+9srOz9csvv9R7OCAqQr8ibdy4UZ999pmGDRtWY/ns2bM1YsQI5eTkKDc3t94DAlEQ+hUpCIJaEV3y8MMPq6ysLPRQQNSEDqmioqJe64HmJHRIvXr10oIFC5RIJGosTyQSmj9/vnr27Fnv4YCoqNf7SJMmTdKKFSvUr18/paenq6KiQiUlJTp9+jTfIoQWJXRIAwYMUH5+vnJycvTtt9+qqqpKsVhMQ4YM0YwZM9S/f3/nnECTVq93TQcOHKj8/Hz9888/qqioUHp6utq0aeOaDYgMy68ftGnThoDQovERVsCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQGlhQVdUij93SpCV7gOYuJRbTwTU5+vvYoUY9bttOd6rHkzMa9ZgtGSE1gr+PHdLff5Qmeww0IE7tAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCasESSfy8UjKP3RD4GEULlhqLac5//q1DFeWNetw70zP0r4ceadRjNjRCauEOVZTrl+PHkj1G5HFqBxgQEmBASIABIQEGhAQYEBJgQEhokoIgiNSxeR8JTVJKSor+/PNPnT9/vlGP26pVK3Xp0uW670dIaLLOnz+vc+fOJXuMa8KpHWBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmCQEiTzgx8hDRw4UIlEQl27dq2x/MyZM0okEo06S2pqqm688cYrbnOhskJB4kIjTXRRSmqa0tqlX3W7k//8rQuN/GWNabGYOrRpe9XtGvu/5SWpqanV//7999+VmpqqvXv3XvE+kfwYxQ033FDnr9df7QmdLNfyhE6Wa3lCJ8v/PqGTJS0tTa1bt77qdpF8RQKaGq6RAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCCki4vG4Fi5cqHg8nuxRUAdCiojDhw9r0aJFOnz4cLJHQR0ICTDgW4QiIC8vT8uXL9ehQ4d05513ql27djp9+rROnDihyspK9ezZU5MmTdIzzzyj4uJizZ49W6WlpcrMzNTIkSNVWFio0tJSvfnmmyopKdH+/fsVj8c1ZMgQzZo1S7fddlv1sYIg0Jdffqlly5ZJuviXxUeNGqXXX39dbds23a/uSroAkVBUVBT07ds3KCoqql62e/fuoG/fvsGKFStqbLtp06ZgypQp1bd/++23oG/fvsGwYcOCnTt3BkEQBCdPngyys7OD0aNHB+fPn6/e9r333gsGDx4c7N69OwiCICgrKwvGjh0bvPzyyw346KKPU7sIGzRokPr06aOVK1fWWF5YWKgJEybU2j4rK0uZmZmSpPT0dE2dOlUHDx7UqlWrJEmHDh1Sfn6+xo0bp0GDBkmSOnXqpClTpmjTpk0qLi5u0McTZYQUcePHj9fOnTt14MABSVJFRYV27NihESNG1Nr2rrvuqnH7Uiw7d+6UJH333XeqqqrSkCFD6rxfUVGRff7mIpJfWYz/evLJJzVv3jwVFhbqjTfe0FdffaXRo0erVatWtbZt3759jdvp6Re/SvmPP/6QJJWXl0uS5s+fr08//bR6u6qqKnXq1EmVlZUN9TAij5AirlOnTnrwwQe1atUqTZ8+XStXrtScOXPq3Pavv/6qcfvkyZOSpFtvvVWSlJGRIUmaOXOmRo0a1XBDN0Oc2kVEWtrF/+cF//9D1n379ungwYOSpAkTJqisrEx5eXmKxWK1TuEu+fnnn2vc3rNnjyTp7rvvliTdf//9isViKikpqXXfd999V9u3b/c8mGaIkCKiW7duSklJ0dGjRyVdfGLv3r1bkvTQQw/plltu0fz58zV+/PjL7mPr1q3atWuXpIvXUosXL1aPHj2UnZ0tSbrjjjv0/PPPq6CgQPv27ZN0MdylS5dqw4YN6t+/f8M9wIjjfaQIWbhwoQoLC9W+fXv16NFDH374YfWfHHn//fe1ZMkSbdmyRTfffHON+8XjcWVlZWnWrFn66aeftG/fPsXjcd1zzz11vo9UUFCgpUuXKpFIqG3bturdu7emT5+u22+/vVEfb5QQUjOxevVqbdq0SfPmzau17lJIc+bMueIrFsLj1K6ZWL9+fZ3vHaFxEFKETZw4UZWVlSotLVU8Htd9992X7JFaLH78HWGtWrXS448/rvT0dL3zzjtKSUmptU1eXp4KCwslSQsWLFBRUZFycnIae9Rmj2skwIBTO8CAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwOD/AOAaUKe1jmeTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "sciq\n",
      "probe acc 0.8840000629425049\n",
      "probe sparse 837 8192\n",
      "model acc 0.8379999995231628\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO+UlEQVR4nO3da2wUhf7G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIC6IFShGqULbzf8GfntO03KZPu532+3nFzszO/Dbsl53pdpeUIAgCAaiXWLIHAJoDQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWzYMJ0/f15dunRJ9iho5kpLS9W6dWsVFRVdcbtIhnTu3DklEolkj4EW4MKFC7qWX/6JZEhdu3aVJH3zzTdJngTNXWZm5jVtxzUSYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGzSakqqqqFnlsNA1pyR7AJRaLaeXKlSotLW3U43bp0kVPPfVUox4TTU+zCUmSSktLdfTo0WSPgRao2ZzaAclESIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAgT2k06dP69ixY+7dAk1a6JCmT59e5/K9e/dq7NixysnJCT0UEDWhQ/r111/rXD5ixAht2bJFa9euDT0UEDWhQ0pJSbnsuoqKCp07dy7sroHISbuejRcvXqwlS5ZU3+7fv/9ltx09enT4qYCIua6Q7r33XklSEARasWKFJk2aVHuHaWnq3r27Hn30Uc+EQARcd0iXYjp48KCmTZvWIEMBURP6Gumjjz664vpt27aF3TUQOQ32hmx2dnZD7Rpocq7r1O5/nThxQvPmzdO2bdt0/PhxVVVVOecCIiV0SG+99ZZKSko0atQodezYUbHYf1/cLv0wAmgpQof0ww8/6Ouvv9ZNN91U5/p4PB56KCBqQl8jde/e/bIRSdIHH3wQdtdA5IQOaezYsdqwYcNl1z/33HNhdw1ETuhTu+LiYuXm5urjjz9Wr1691LZt2xrrf/zxx3oPB0RF6JDWrVunrl276syZM9q7d2+t9RUVFfUaDIiS0CH16dNHq1evvuz6rKyssLsGIif0NdLbb799xfW8IYuWJHRIw4YNu+L67du3h901EDmhT+2OHj16xfXLly/X5MmTw+4eiJTQIY0aNeqKH+4DWpLQIXXr1k2vvfZajWVnz55VSUmJtm/frilTptR7OCAqQoc0btw4jR8/vs51JSUlWrp0aeihgKgJ/cOGmTNnXnZd7969tXv37rC7BiKnQT6PdODAAZ08ebIhdg00SaFP7Z599tlay4IgUHl5uQ4ePKinn366XoMBURI6pH379mngwIE1lqWmpqpv376aMmXKZa+fgOYodEg9evRQXl6ecxYgskJfI33xxRfGMYBoC/2K1KFDB0lSUVGRtmzZorKyMnXs2FEPPPDAVX99CGhuQoeUSCQ0c+ZMrV+/XkEQVC/PycnRY489pnnz5tX4HgegOQsd0ieffKIdO3Zo1qxZysjIUIcOHXTq1Cnt3r1bn3/+uXJycvTKK684ZwWarNAhrV27Vnl5eerRo0eN5RkZGRo5cqSmTp1KSGgxQp97xWKxWhFd0qNHD6WmpoYeCoia0CGdP39eZWVlda47efIk/60LWpTQIWVmZuqFF17Qpk2bdOLECV24cEEnTpzQxo0b9dJLL+mRRx5xzgk0aaGvkaZPn64XX3xRr776aq3PJQ0ZMqTWRyyA5ix0SO3atVN+fr7WrVunrVu3Vr+PNGLECD3xxBNKSwu9ayByQj/by8rKtHPnTnXq1Enz58+vXr5+/XqdPn1anTp1sgwIREHoa6T8/HxNnz5da9asqbH8+++/V1ZWln755Zd6DwdERehXpE2bNumzzz7T8OHDayyfM2eORo4cqezsbOXk5NR7QCAKQr8iBUFQK6JLHn74YZWWloYeCoia0CGVl5fXaz3QnIQOqXfv3lq4cKESiUSN5YlEQgsWLFCvXr3qPRwQFfV6H2ny5MlauXKl+vfvr/T0dJWXl6u4uFhnzpzhW4TQooQOaeDAgcrLy1N2dra+/fZbVVVVKRaLaejQoZo5c6YGDBjgnBNo0ur1rumgQYOUl5enf/75R+Xl5UpPT1ebNm1cswGRYfn1gzZt2hAQWjQ+wgoYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhNTAgqqqFnnsliYt2QM0dymxmA6tzdbfxw836nHbdr5TPZ+c2ajHbMkIqRH8ffyw/v6jJNljoAFxagcYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEgtWCKJH/xL5rEbAp9HasFSYzHN/c+/dbi8rFGPe2d6R/3roUca9ZgNjZBauMPlZfrlxPFkjxF5nNqhSQqCIFLH5hUJTVJKSor+/PNPVVZWNupxW7Vqpa5du173/QgJTVZlZaXOnz+f7DGuCad2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBikBMn8KGJIgwYNUiKRULdu3WosP3v2rBKJRKPOkpqaqhtvvPGK21yoKFeQuNBIE12UkpqmtHbpV93u1D9/60IjfxFJWiymDm3aXnW7xv67vCQ1NbX6z7///rtSU1O1b9++K94nkh/su+GGG+r8wNfVntDJci1P6GS5lid0svzvEzpZ0tLS1Lp166tuF8lXJKCp4RoJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkCIiHo9r0aJFisfjyR4FdSCkiDhy5IgWL16sI0eOJHsU1IGQAAO+RSgCcnNztWLFCh0+fFh33nmn2rVrpzNnzujkyZOqqKhQr169NHnyZD3zzDMqKirSnDlzVFJSooyMDI0aNUoFBQUqKSnRm2++qeLiYh04cEDxeFxDhw7V7Nmzddttt1UfKwgCffnll1q+fLkkqbKyUqNHj9brr7+utm2b7ld3JV2ASCgsLAz69esXFBYWVi/bs2dP0K9fv2DlypU1tt28eXMwderU6tu//fZb0K9fv2D48OHBrl27giAIglOnTgVZWVnBmDFjgsrKyupt33vvvWDIkCHBnj17giAIgtLS0mDcuHHByy+/3ICPLvo4tYuwwYMHq2/fvlq1alWN5QUFBZo4cWKt7TMzM5WRkSFJSk9P17Rp03To0CGtXr1aknT48GHl5eVp/PjxGjx4sCSpc+fOmjp1qjZv3qyioqIGfTxRRkgRN2HCBO3atUsHDx6UJJWXl2vnzp0aOXJkrW3vuuuuGrcvxbJr1y5J0nfffaeqqioNHTq0zvsVFhba528uIvmVxfivJ598UvPnz1dBQYHeeOMNffXVVxozZoxatWpVa9v27dvXuJ2efvGrlP/44w9JUllZmSRpwYIF+vTTT6u3q6qqUufOnVVRUdFQDyPyCCniOnfurAcffFCrV6/WjBkztGrVKs2dO7fObf/6668at0+dOiVJuvXWWyVJHTt2lCTNmjVLo0ePbrihmyFO7SIiLe3iv3nB//+Qdf/+/Tp06JAkaeLEiSotLVVubq5isVitU7hLfv755xq39+7dK0m6++67JUn333+/YrGYiouLa9333Xff1Y4dOzwPphkipIjo3r27UlJSdOzYMUkXn9h79uyRJD300EO65ZZbtGDBAk2YMOGy+9i2bZt2794t6eK11JIlS9SzZ09lZWVJku644w49//zzys/P1/79+yVdDHfZsmXauHGjBgwY0HAPMOJ4HylCFi1apIKCArVv3149e/bUhx9+WP1fjrz//vtaunSptm7dqptvvrnG/eLxuDIzMzV79mz99NNP2r9/v+LxuO65554630fKz8/XsmXLlEgk1LZtW/Xp00czZszQ7bff3qiPN0oIqZlYs2aNNm/erPnz59dadymkuXPnXvEVC+FxatdMbNiwoc73jtA4CCnCJk2apIqKCpWUlCgej+u+++5L9kgtFj/+jrBWrVrp8ccfV3p6ut555x2lpKTU2iY3N1cFBQWSpIULF6qwsFDZ2dmNPWqzxzUSYMCpHWBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYPB/pNpQp0PpQe0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "sciq\n",
      "probe acc 0.8700000643730164\n",
      "probe sparse 371 8192\n",
      "model acc 0.8379999995231628\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO+UlEQVR4nO3daWwVhf7G8aenBQHRAgIGRQmrgbBUIRIjLlAU0GgKJAYjuIdoQkQSA95oSDAatEgMW4y1Rk0btoSGReHFjZALKBUa9lgVCxEPiBYoRahCOZ3/C/703qZlmz7t6bTfzyvOzHTmd8L5cmZ6FlKCIAgEoF5iyR4AaA4ICTAgJMCAkAADQgIMCAkwICTAgJAAg7RkDxDGsGHDdP78eXXp0iXZo6CZKy0tVevWrVVUVHTF7SIZ0rlz55RIJJI9BlqACxcu6Fre/BPJkLp27SpJ+uabb5I8CZq7zMzMa9qOayTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAoNmEVFVV1SKPjaYhLdkDuMRiMa1cuVKlpaWNetwuXbroqaeeatRjoulpNiFJUmlpqY4ePZrsMdACNZtTOyCZCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADe0inT5/WsWPH3LsFmrTQIU2fPr3O5Xv37tXYsWOVk5MTeiggakKH9Ouvv9a5fMSIEdqyZYvWrl0beiggakKHlJKSctl1FRUVOnfuXNhdA5GTdj0bL168WEuWLKm+3b9//8tuO3r06PBTARFzXSHde++9kqQgCLRixQpNmjSp9g7T0tS9e3c9+uijngmBCLjukC7FdPDgQU2bNq1BhgKiJvQ10kcffXTF9du2bQu7ayByGuwF2ezs7IbaNdDkXNep3f86ceKE5s2bp23btun48eOqqqpyzgVESuiQ3nrrLZWUlGjUqFHq2LGjYrH/Prld+mUE0FKEDumHH37Q119/rZtuuqnO9fF4PPRQQNSEvkbq3r37ZSOSpA8++CDsroHICR3S2LFjtWHDhsuuf+6558LuGoic0Kd2xcXFys3N1ccff6xevXqpbdu2Ndb/+OOP9R4OiIrQIa1bt05du3bVmTNntHfv3lrrKyoq6jUYECWhQ+rTp49Wr1592fVZWVlhdw1ETuhrpLfffvuK63lBFi1J6JCGDRt2xfXbt28Pu2sgckKf2h09evSK65cvX67JkyeH3T0QKaFDGjVq1BU/3Ae0JKFD6tatm1577bUay86ePauSkhJt375dU6ZMqfdwQFSEDmncuHEaP358netKSkq0dOnS0EMBURP6lw0zZ8687LrevXtr9+7dYXcNRE6DfB7pwIEDOnnyZEPsGmiSQp/aPfvss7WWBUGg8vJyHTx4UE8//XS9BgOiJHRI+/bt08CBA2ssS01NVd++fTVlypTLXj8BzVHokHr06KG8vDznLEBkhb5G+uKLL4xjANEW+hmpQ4cOkqSioiJt2bJFZWVl6tixox544IGrvn0IaG5Ch5RIJDRz5kytX79eQRBUL8/JydFjjz2mefPm1fgeB6A5Cx3SJ598oh07dmjWrFnKyMhQhw4ddOrUKe3evVuff/65cnJy9MorrzhnBZqs0CGtXbtWeXl56tGjR43lGRkZGjlypKZOnUpIaDFCn3vFYrFaEV3So0cPpaamhh4KiJrQIZ0/f15lZWV1rjt58iT/rQtalNAhZWZm6oUXXtCmTZt04sQJXbhwQSdOnNDGjRv10ksv6ZFHHnHOCTRpoa+Rpk+frhdffFGvvvpqrc8lDRkypNZHLIDmLHRI7dq1U35+vtatW6etW7dWv440YsQIPfHEE0pLC71rIHJCP9rLysq0c+dOderUSfPnz69evn79ep0+fVqdOnWyDAhEQehrpPz8fE2fPl1r1qypsfz7779XVlaWfvnll3oPB0RF6GekTZs26bPPPtPw4cNrLJ8zZ45Gjhyp7Oxs5eTk1HtAIApCPyMFQVAroksefvhhlZaWhh4KiJrQIZWXl9drPdCchA6pd+/eWrhwoRKJRI3liURCCxYsUK9eveo9HBAV9XodafLkyVq5cqX69++v9PR0lZeXq7i4WGfOnOFbhNCihA5p4MCBysvLU3Z2tr799ltVVVUpFotp6NChmjlzpgYMGOCcE2jS6vWq6aBBg5SXl6d//vlH5eXlSk9PV5s2bVyzAZFheftBmzZtCAgtGh9hBQwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQGlhQVdUij93SpCV7gOYuJRbTobXZ+vv44UY9btvOd6rnkzMb9ZgtGSE1gr+PH9bff5Qkeww0IE7tAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0JqwRJJfC9eMo/dEHiLUAuWGotp7n/+rcPlZY163DvTO+pfDz3SqMdsaITUwh0uL9MvJ44ne4zI49QOMCAkwICQAANCAgwICTAgJMCAkNAkBUEQqWPzOhKapJSUFP3555+qrKxs1OO2atVKXbt2ve6fIyQ0WZWVlTp//nyyx7gmnNoBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYJASJPOjiCENGjRIiURC3bp1q7H87NmzSiQSjTpLamqqbrzxxituc6GiXEHiQiNNdFFKaprS2qVfdbtT//ytC4389cFpsZg6tGl71e0a++/yktTU1Oo///7770pNTdW+ffuu+DOR/GDfDTfcUOcHvq72gE6Wa3lAJ8u1PKCT5X8f0MmSlpam1q1bX3W7SD4jAU0N10iAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEFBHxeFyLFi1SPB5P9iioAyFFxJEjR7R48WIdOXIk2aOgDoQEGPAtQhGQm5urFStW6PDhw7rzzjvVrl07nTlzRidPnlRFRYV69eqlyZMn65lnnlFRUZHmzJmjkpISZWRkaNSoUSooKFBJSYnefPNNFRcX68CBA4rH4xo6dKhmz56t2267rfpYQRDoyy+/1PLlyyVJlZWVGj16tF5//XW1bdt0v7or6QJEQmFhYdCvX7+gsLCwetmePXuCfv36BStXrqyx7ebNm4OpU6dW3/7tt9+Cfv36BcOHDw927doVBEEQnDp1KsjKygrGjBkTVFZWVm/73nvvBUOGDAn27NkTBEEQlJaWBuPGjQtefvnlBrx30cepXYQNHjxYffv21apVq2osLygo0MSJE2ttn5mZqYyMDElSenq6pk2bpkOHDmn16tWSpMOHDysvL0/jx4/X4MGDJUmdO3fW1KlTtXnzZhUVFTXo/YkyQoq4CRMmaNeuXTp48KAkqby8XDt37tTIkSNrbXvXXXfVuH0pll27dkmSvvvuO1VVVWno0KF1/lxhYaF9/uYikl9ZjP968sknNX/+fBUUFOiNN97QV199pTFjxqhVq1a1tm3fvn2N2+npF79K+Y8//pAklZWVSZIWLFigTz/9tHq7qqoqde7cWRUVFQ11NyKPkCKuc+fOevDBB7V69WrNmDFDq1at0ty5c+vc9q+//qpx+9SpU5KkW2+9VZLUsWNHSdKsWbM0evTohhu6GeLULiLS0i7+mxf8/y9Z9+/fr0OHDkmSJk6cqNLSUuXm5ioWi9U6hbvk559/rnF77969kqS7775bknT//fcrFoupuLi41s++++672rFjh+fONEOEFBHdu3dXSkqKjh07JuniA3vPnj2SpIceeki33HKLFixYoAkTJlx2H9u2bdPu3bslXbyWWrJkiXr27KmsrCxJ0h133KHnn39e+fn52r9/v6SL4S5btkwbN27UgAEDGu4ORhyvI0XIokWLVFBQoPbt26tnz5768MMPq//Lkffff19Lly7V1q1bdfPNN9f4uXg8rszMTM2ePVs//fST9u/fr3g8rnvuuafO15Hy8/O1bNkyJRIJtW3bVn369NGMGTN0++23N+r9jRJCaibWrFmjzZs3a/78+bXWXQpp7ty5V3zGQnic2jUTGzZsqPO1IzQOQoqwSZMmqaKiQiUlJYrH47rvvvuSPVKLxa+/I6xVq1Z6/PHHlZ6ernfeeUcpKSm1tsnNzVVBQYEkaeHChSosLFR2dnZjj9rscY0EGHBqBxgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGPwfZCtQpxFXcngAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "sciq\n",
      "probe acc 0.7780000567436218\n",
      "probe sparse 187 8192\n",
      "model acc 0.8379999995231628\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO+0lEQVR4nO3deWwUhd/H8U+3BQHRUgQMihJOA+GoQiRGPKAooNEUSAxG8A7RhIgkBnyiIcFo0CIxXDHWGjVtuBIaDoU/fhHyAEqFhjtWxULEBdECpQhVju08f/BQf03LNf2022nfr7/YmdmZ78Z9uzPd7jYlCIJAAOolluwBgOaAkAADQgIMCAkwICTAgJAAA0ICDAgJMEhL9gBhDB06VOfOnVPnzp2TPQqaubKyMrVu3VrFxcVX3C6SIZ09e1aJRCLZY6AFuHDhgq7ll38iGVKXLl0kSd98802SJ0Fzl5WVdU3bcY0EGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBs0mpKqqqhZ5bDQNackewCUWi2nFihUqKytr1ON27txZTz31VKMeE01PswlJksrKynTkyJFkj4EWqNmc2gHJREiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgYA/p1KlTOnr0qHu3QJMWOqRp06bVuXzPnj0aM2aMcnNzQw8FRE3okH799dc6lw8fPlybN2/WmjVrQg8FRE3okFJSUi67rrKyUmfPng27ayBy0q5n40WLFmnx4sXVt/v163fZbUeNGhV+KiBiriuke++9V5IUBIGWL1+uiRMn1t5hWpq6deumRx991DMhEAHXHdKlmA4cOKCpU6c2yFBA1IS+Rvroo4+uuH7r1q1hdw1EToO9IZuTk9NQuwaanOs6tftvx48f19y5c7V161YdO3ZMVVVVzrmASAkd0ltvvaXS0lKNHDlSGRkZisX+fXG79MMIoKUIHdIPP/ygr7/+WjfddFOd6+PxeOihgKgJfY3UrVu3y0YkSR988EHYXQOREzqkMWPGaP369Zdd/9xzz4XdNRA5oU/tSkpKlJeXp48//lg9e/ZU27Zta6z/8ccf6z0cEBWhQ1q7dq26dOmi06dPa8+ePbXWV1ZW1mswIEpCh9S7d2+tWrXqsuuzs7PD7hqInNDXSG+//fYV1/OGLFqS0CENHTr0iuu3bdsWdtdA5IQ+tTty5MgV1y9btkyTJk0Ku3sgUkKHNHLkyCt+uA9oSUKH1LVrV7322ms1lp05c0alpaXatm2bJk+eXO/hgKgIHdLYsWM1bty4OteVlpZqyZIloYcCoib0DxtmzJhx2XW9evXSrl27wu4aiJwG+TzS/v37deLEiYbYNdAkhT61e/bZZ2stC4JAFRUVOnDggJ5++ul6DQZESeiQ9u7dqwEDBtRYlpqaqj59+mjy5MmXvX4CmqPQIXXv3l35+fnOWYDICn2N9MUXXxjHAKIt9CtShw4dJEnFxcXavHmzysvLlZGRoQceeOCqvz4ENDehQ0okEpoxY4bWrVunIAiql+fm5uqxxx7T3Llza3yPA9CchQ7pk08+0fbt2zVz5kxlZmaqQ4cOOnnypHbt2qXPP/9cubm5euWVV5yzAk1W6JDWrFmj/Px8de/evcbyzMxMjRgxQlOmTCEktBihz71isVitiC7p3r27UlNTQw8FRE3okM6dO6fy8vI61504cYI/64IWJXRIWVlZeuGFF7Rx40YdP35cFy5c0PHjx7Vhwwa99NJLeuSRR5xzAk1a6GukadOm6cUXX9Srr75a63NJgwcPrvURC6A5Cx1Su3btVFBQoLVr12rLli3V7yMNHz5cTzzxhNLSQu8aiJzQz/by8nLt2LFDHTt21Lx586qXr1u3TqdOnVLHjh0tAwJREPoaqaCgQNOmTdPq1atrLP/++++VnZ2tX375pd7DAVER+hVp48aN+uyzzzRs2LAay2fPnq0RI0YoJydHubm59R4QiILQr0hBENSK6JKHH35YZWVloYcCoiZ0SBUVFfVaDzQnoUPq1auXFixYoEQiUWN5IpHQ/Pnz1bNnz3oPB0RFvd5HmjRpklasWKF+/fopPT1dFRUVKikp0enTp/kWIbQooUMaMGCA8vPzlZOTo2+//VZVVVWKxWIaMmSIZsyYof79+zvnBJq0er1rOnDgQOXn5+uff/5RRUWF0tPT1aZNG9dsQGRYfv2gTZs2BIQWjY+wAgaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhtWCJqqoWeeyGkJbsAZA8qbGY5vzvf3SoorxRj3tneob+56FHGvWYDY2QWrhDFeX65fixZI8ReZzaAQaEBBgQEmBASIABIQEGhAQYEBJgQEgNLEjiO/jJPHZLwxuyDSwlFtPBNTn6+9ihRj1u2053qseTMxr1mC0ZITWCv48d0t9/lCZ7DDQgTu0AA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQkKTFARBpI7N13GhSUpJSdGff/6p8+fPN+pxW7VqpS5dulz3/QgJTdb58+d17ty5ZI9xTTi1AwwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgxSgmR+8COkgQMHKpFIqGvXrjWWnzlzRolEolFnSU1N1Y033njFbS5UVihIXGikiS5KSU1TWrv0q2538p+/daGqqhEm+ldaLKYObdpedbvG/m95SWpqavW/f//9d6Wmpmrv3r1XvE8kP0Zxww031Pnr9Vd7QifLtTyhk+VantDJ8t9P6GRJS0tT69atr7pdJF+RgKaGayTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCioh4PK6FCxcqHo8nexTUgZAi4vDhw1q0aJEOHz6c7FFQB0ICDPgWoQjIy8vT8uXLdejQId15551q166dTp8+rRMnTqiyslI9e/bUpEmT9Mwzz6i4uFizZ89WaWmpMjMzNXLkSBUWFqq0tFRvvvmmSkpKtH//fsXjcQ0ZMkSzZs3SbbfdVn2sIAj05ZdfatmyZZIu/mXxUaNG6fXXX1fbtk33q7uSLkAkFBUVBX379g2Kioqql+3evTvo27dvsGLFihrbbtq0KZgyZUr17d9++y3o27dvMGzYsGDnzp1BEATByZMng+zs7GD06NHB+fPnq7d97733gsGDBwe7d+8OgiAIysrKgrFjxwYvv/xyAz666OPULsIGDRqkPn36aOXKlTWWFxYWasKECbW2z8rKUmZmpiQpPT1dU6dO1cGDB7Vq1SpJ0qFDh5Sfn69x48Zp0KBBkqROnTppypQp2rRpk4qLixv08UQZIUXc+PHjtXPnTh04cECSVFFRoR07dmjEiBG1tr3rrrtq3L4Uy86dOyVJ3333naqqqjRkyJA671dUVGSfv7mI5FcW419PPvmk5s2bp8LCQr3xxhv66quvNHr0aLVq1arWtu3bt69xOz394lcp//HHH5Kk8vJySdL8+fP16aefVm9XVVWlTp06qbKysqEeRuQRUsR16tRJDz74oFatWqXp06dr5cqVmjNnTp3b/vXXXzVunzx5UpJ06623SpIyMjIkSTNnztSoUaMabuhmiFO7iEhLu/j/vOD/f8i6b98+HTx4UJI0YcIElZWVKS8vT7FYrNYp3CU///xzjdt79uyRJN19992SpPvvv1+xWEwlJSW17vvuu+9q+/btngfTDBFSRHTr1k0pKSk6evSopItP7N27d0uSHnroId1yyy2aP3++xo8ff9l9bN26Vbt27ZJ08Vpq8eLF6tGjh7KzsyVJd9xxh55//nkVFBRo3759ki6Gu3TpUm3YsEH9+/dvuAcYcbyPFCELFy5UYWGh2rdvrx49eujDDz+s/pMj77//vpYsWaItW7bo5ptvrnG/eDyurKwszZo1Sz/99JP27duneDyue+65p873kQoKCrR06VIlEgm1bdtWvXv31vTp03X77bc36uONEkJqJlavXq1NmzZp3rx5tdZdCmnOnDlXfMVCeJzaNRPr16+v870jNA5CirCJEyeqsrJSpaWlisfjuu+++5I9UovFj78jrFWrVnr88ceVnp6ud955RykpKbW2ycvLU2FhoSRpwYIFKioqUk5OTmOP2uxxjQQYcGoHGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQY/B9SbFCn64VxagAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "sciq\n",
      "probe acc 0.7610000371932983\n",
      "probe sparse 201 8192\n",
      "model acc 0.8379999995231628\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO9UlEQVR4nO3deWwUhd/H8U+3BQHRUgQMihJOA+GoQiRGPKAooNEUSAxG8A7RhIgkBnyiIcFo0CIxXDHWGjVtuBIaDoU/fhHyAEqFhjtWxULEBdECpQhVju08f/BQf03LNf2022nfr7/YmenMd+O+3Znu0ZQgCAIBqJdYsgcAmgNCAgwICTAgJMCAkAADQgIMCAkwICTAIC3ZA4QxdOhQnTt3Tp07d072KGjmysrK1Lp1axUXF19xu0iGdPbsWSUSiWSPgRbgwoULupY3/0QypC5dukiSvvnmmyRPguYuKyvrmrbjGgkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDJpNSFVVVS3y2Gga0pI9gEssFtOKFStUVlbWqMft3LmznnrqqUY9JpqeZhOSJJWVlenIkSPJHgMtULM5tQOSiZAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMDAHtKpU6d09OhR926BJi10SNOmTatz+Z49ezRmzBjl5uaGHgqImtAh/frrr3UuHz58uDZv3qw1a9aEHgqImtAhpaSkXHZdZWWlzp49G3bXQOSkXc/GixYt0uLFi6tv9+vX77Lbjho1KvxUQMRcV0j33nuvJCkIAi1fvlwTJ06svcO0NHXr1k2PPvqoZ0IgAq47pEsxHThwQFOnTm2QoYCoCX2N9NFHH11x/datW8PuGoicBntBNicnp6F2DTQ513Vq99+OHz+uuXPnauvWrTp27JiqqqqccwGREjqkt956S6WlpRo5cqQyMjIUi/375HbplxFASxE6pB9++EFff/21brrppjrXx+Px0EMBURP6Gqlbt26XjUiSPvjgg7C7BiIndEhjxozR+vXrL7v+ueeeC7trIHJCn9qVlJQoLy9PH3/8sXr27Km2bdvWWP/jjz/WezggKkKHtHbtWnXp0kWnT5/Wnj17aq2vrKys12BAlIQOqXfv3lq1atVl12dnZ4fdNRA5oa+R3n777Suu5wVZtCShQxo6dOgV12/bti3sroHICX1qd+TIkSuuX7ZsmSZNmhR290CkhA5p5MiRV/xwH9CShA6pa9eueu2112osO3PmjEpLS7Vt2zZNnjy53sMBURE6pLFjx2rcuHF1ristLdWSJUtCDwVETehfNsyYMeOy63r16qVdu3aF3TUQOQ3yeaT9+/frxIkTDbFroEkKfWr37LPP1loWBIEqKip04MABPf300/UaDIiS0CHt3btXAwYMqLEsNTVVffr00eTJky97/QQ0R6FD6t69u/Lz852zAJEV+hrpiy++MI4BRFvoZ6QOHTpIkoqLi7V582aVl5crIyNDDzzwwFXfPgQ0N6FDSiQSmjFjhtatW6cgCKqX5+bm6rHHHtPcuXNrfI8D0JyFDumTTz7R9u3bNXPmTGVmZqpDhw46efKkdu3apc8//1y5ubl65ZVXnLMCTVbokNasWaP8/Hx17969xvLMzEyNGDFCU6ZMISS0GKHPvWKxWK2ILunevbtSU1NDDwVETeiQzp07p/Ly8jrXnThxgj/rghYldEhZWVl64YUXtHHjRh0/flwXLlzQ8ePHtWHDBr300kt65JFHnHMCTVroa6Rp06bpxRdf1Kuvvlrrc0mDBw+u9RELoDkLHVK7du1UUFCgtWvXasuWLdWvIw0fPlxPPPGE0tJC7xqInNCP9vLycu3YsUMdO3bUvHnzqpevW7dOp06dUseOHS0DAlEQ+hqpoKBA06ZN0+rVq2ss//7775Wdna1ffvml3sMBURH6GWnjxo367LPPNGzYsBrLZ8+erREjRignJ0e5ubn1HhCIgtDPSEEQ1IrokocfflhlZWWhhwKiJnRIFRUV9VoPNCehQ+rVq5cWLFigRCJRY3kikdD8+fPVs2fPeg8HREW9XkeaNGmSVqxYoX79+ik9PV0VFRUqKSnR6dOn+RYhtCihQxowYIDy8/OVk5Ojb7/9VlVVVYrFYhoyZIhmzJih/v37O+cEmrR6vWo6cOBA5efn659//lFFRYXS09PVpk0b12xAZFjeftCmTRsCQovGR1gBA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkBpYUFXVZI+dSOJsyTx2Q0hL9gDNXUospoNrcvT3sUONety2ne5UjydnXHGb1FhMc/73PzpUUd5IU110Z3qG/uehRxr1mA2NkBrB38cO6e8/SpM9Rp0OVZTrl+PHkj1G5HFqBxgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEpqkIAgidWy+shhNUkpKiv7880+dP3++UY/bqlUrdenS5bp/jpDQZJ0/f17nzp1L9hjXhFM7wICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDFKCZL7NNqSBAwcqkUioa9euNZafOXNGiUSiUWdJTU3VjTfeeMVtLlRWKEhcaKSJLkpJTVNau/Srbnfyn791oaqqESb6V1ospg5t2l51u8b+b3lJampq9b9///13paamau/evVf8mUi+afWGG26o882MV3tAJ8u1PKCT5Voe0Mny3w/oZElLS1Pr1q2vul0kn5GApoZrJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0KKiHg8roULFyoejyd7FNSBkCLi8OHDWrRokQ4fPpzsUVAHQgIM+BahCMjLy9Py5ct16NAh3XnnnWrXrp1Onz6tEydOqLKyUj179tSkSZP0zDPPqLi4WLNnz1ZpaakyMzM1cuRIFRYWqrS0VG+++aZKSkq0f/9+xeNxDRkyRLNmzdJtt91WfawgCPTll19q2bJlki7+HddRo0bp9ddfV9u2Tferu5IuQCQUFRUFffv2DYqKiqqX7d69O+jbt2+wYsWKGttu2rQpmDJlSvXt3377Lejbt28wbNiwYOfOnUEQBMHJkyeD7OzsYPTo0cH58+ert33vvfeCwYMHB7t37w6CIAjKysqCsWPHBi+//HID3rvo49QuwgYNGqQ+ffpo5cqVNZYXFhZqwoQJtbbPyspSZmamJCk9PV1Tp07VwYMHtWrVKknSoUOHlJ+fr3HjxmnQoEGSpE6dOmnKlCnatGmTiouLG/T+RBkhRdz48eO1c+dOHThwQJJUUVGhHTt2aMSIEbW2veuuu2rcvhTLzp07JUnfffedqqqqNGTIkDp/rqioyD5/cxHJryzGv5588knNmzdPhYWFeuONN/TVV19p9OjRatWqVa1t27dvX+N2evrFr1L+448/JEnl5eWSpPnz5+vTTz+t3q6qqkqdOnVSZWVlQ92NyCOkiOvUqZMefPBBrVq1StOnT9fKlSs1Z86cOrf966+/atw+efKkJOnWW2+VJGVkZEiSZs6cqVGjRjXc0M0Qp3YRkZZ28f95wf//knXfvn06ePCgJGnChAkqKytTXl6eYrFYrVO4S37++ecat/fs2SNJuvvuuyVJ999/v2KxmEpKSmr97Lvvvqvt27d77kwzREgR0a1bN6WkpOjo0aOSLj6wd+/eLUl66KGHdMstt2j+/PkaP378ZfexdetW7dq1S9LFa6nFixerR48eys7OliTdcccdev7551VQUKB9+/ZJuhju0qVLtWHDBvXv37/h7mDE8TpShCxcuFCFhYVq3769evTooQ8//LD6T468//77WrJkibZs2aKbb765xs/F43FlZWVp1qxZ+umnn7Rv3z7F43Hdc889db6OVFBQoKVLlyqRSKht27bq3bu3pk+frttvv71R72+UEFIzsXr1am3atEnz5s2rte5SSHPmzLniMxbC49SumVi/fn2drx2hcRBShE2cOFGVlZUqLS1VPB7Xfffdl+yRWix+/R1hrVq10uOPP6709HS98847SklJqbVNXl6eCgsLJUkLFixQUVGRcnJyGnvUZo9rJMCAUzvAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMDg/wAVL1Cj3T9ghQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "creak\n",
      "probe acc 0.7024070024490356\n",
      "probe sparse 3853 8192\n",
      "model acc 0.50255286693573\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO/klEQVR4nO3da2wUhf7G8afbgoBoAQGDooSrgXCpQiRGvEBRQKMpkBiM4D1EEyKSGPBEQ4LRoEViuMVYa9S04ZbQcFF4cSLkAEqFhnusioWIC6IFShGqtGzn/4I/PadpQZk+7Xba7+cVOzOd+W3crzvT7e6mBEEQCECDxJI9ANASEBJgQEiAASEBBoQEGBASYEBIgAEhAQZpyR4gjBEjRqiyslLdunVL9iho4UpLS9W2bVsVFRVddbtIhnThwgUlEolkj4FW4OLFi/onf/wTyZC6d+8uSfrqq6+SPAlauszMzH+0HddIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBgD+ns2bM6ceKEe7dAsxY6pJkzZ9a7fP/+/Ro/frxycnJCDwVETeiQfv7553qXjxo1Stu2bdP69etDDwVETeiQUlJSrriuoqJCFy5cCLtrIHLSrmXjpUuXatmyZTW3Bw4ceMVtx44dG34qIGKuKaS7775bkhQEgVatWqUpU6bU3WFamnr27KmHH37YMyEQAdcc0uWYDh8+rBkzZjTKUEDUhL5G+uCDD666fseOHWF3DUROo70gm52d3Vi7Bpqdazq1+1+nTp3SggULtGPHDp08eVLV1dXOuYBICR3SG2+8oZKSEo0ZM0adO3dWLPbfJ7fLv4wAWovQIX333Xf68ssvdcMNN9S7Ph6Phx4KiJrQ10g9e/a8YkSS9N5774XdNRA5oUMaP368Nm3adMX1zzzzTNhdA5ET+tSuuLhYubm5+vDDD9WnTx+1b9++1vrvv/++wcMBURE6pA0bNqh79+46d+6c9u/fX2d9RUVFgwYDoiR0SP369dPatWuvuD4rKyvsroHICX2N9Oabb151PS/IojUJHdKIESOuun7nzp1hdw1ETuhTu+PHj191/cqVKzV16tSwuwciJXRIY8aMueqb+4DWJHRIPXr00CuvvFJr2fnz51VSUqKdO3dq2rRpDR4OiIrQIU2YMEETJ06sd11JSYmWL18eeiggakL/smH27NlXXNe3b1/t3bs37K6ByGmU9yMdOnRIp0+fboxdA81S6FO7p59+us6yIAhUXl6uw4cP68knn2zQYECUhA7pwIEDGjx4cK1lqamp6t+/v6ZNm3bF6yegJQodUq9evZSXl+ecBYis0NdIn332mXEMINpCPyN16tRJklRUVKRt27aprKxMnTt31n333fe3fz4EtDShQ0okEpo9e7Y2btyoIAhqlufk5OiRRx7RggULan2OA9CShQ7po48+0q5duzRnzhxlZGSoU6dOOnPmjPbu3atPP/1UOTk5eumll5yzAs1W6JDWr1+vvLw89erVq9byjIwMjR49WtOnTycktBqhz71isVidiC7r1auXUlNTQw8FRE3okCorK1VWVlbvutOnT/O1LmhVQoeUmZmp5557Tlu2bNGpU6d08eJFnTp1Sps3b9YLL7yghx56yDkn0KyFvkaaOXOmnn/+eb388st13pc0bNiwOm+xAFqy0CF16NBB+fn52rBhg7Zv317zOtKoUaP02GOPKS0t9K6ByAn9aC8rK9Pu3bvVpUsXLVy4sGb5xo0bdfbsWXXp0sUyIBAFoa+R8vPzNXPmTK1bt67W8m+//VZZWVn66aefGjwcEBWhn5G2bNmiTz75RCNHjqy1fN68eRo9erSys7OVk5PT4AGBKAj9jBQEQZ2ILnvwwQdVWloaeiggakKHVF5e3qD1QEsSOqS+fftq8eLFSiQStZYnEgktWrRIffr0afBwQFQ06HWkqVOnavXq1Ro4cKDS09NVXl6u4uJinTt3jk8RQqsSOqTBgwcrLy9P2dnZ+vrrr1VdXa1YLKbhw4dr9uzZGjRokHNOoFlr0KumQ4YMUV5env766y+Vl5crPT1d7dq1c80GRIblzw/atWtHQGjVeAsrYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGLSYkKqrq1vlsdE8tJgPn4vFYlq9enWTf1ZEt27d9MQTTzTpMdH8tJiQJKm0tFTHjx9P9hhohVrMqR2QTIQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQUiMLkvglZMk8dmvTor4fqTlKicV0ZH22/jx5tEmP277r7er9+OwmPWZrRkhN4M+TR/XnbyXJHgONiFM7wICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAnNUhAEkTp2WiPMATRYSkqKfv/9d1VVVTXpcdu0aaPu3btf888REpqtqqoqVVZWJnuMf4RTO8CAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQmrFEtXVrfLYjYHP/m7FUmMxzf/Pv3W0vKxJj3t7emf964GHmvSYjY2QWrmj5WX66dTJZI8ReZzaAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYJASBEGQ7CGu1ZAhQ5RIJNSjR49ay8+fP69EItGks6Smpur666+/6jYXK8oVJC420USXpKSmKa1D+t9ud+avP3Wxib9hPC0WU6d27f92u6b+b3lZampqzb9//fVXpaam6sCBA1f9mUh+GfN1112nysrKOsv/7gGdLP/kAZ0s/+QBnSz/+4BOlrS0NLVt2/Zvt4vkMxLQ3HCNBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEgREY/HtWTJEsXj8WSPgnoQUkQcO3ZMS5cu1bFjx5I9CupBSIABnyIUAbm5uVq1apWOHj2q22+/XR06dNC5c+d0+vRpVVRUqE+fPpo6daqeeuopFRUVad68eSopKVFGRobGjBmjgoIClZSU6PXXX1dxcbEOHTqkeDyu4cOHa+7cubrllltqjhUEgT7//HOtXLlSklRVVaWxY8fq1VdfVfv2zfeju5IuQCQUFhYGAwYMCAoLC2uW7du3LxgwYECwevXqWttu3bo1mD59es3tX375JRgwYEAwcuTIYM+ePUEQBMGZM2eCrKysYNy4cUFVVVXNtu+8804wbNiwYN++fUEQBEFpaWkwYcKE4MUXX2zEexd9nNpF2NChQ9W/f3+tWbOm1vKCggJNnjy5zvaZmZnKyMiQJKWnp2vGjBk6cuSI1q5dK0k6evSo8vLyNHHiRA0dOlSS1LVrV02fPl1bt25VUVFRo96fKCOkiJs0aZL27Nmjw4cPS5LKy8u1e/dujR49us62d9xxR63bl2PZs2ePJOmbb75RdXW1hg8fXu/PFRYW2udvKSL5kcX4r8cff1wLFy5UQUGBXnvtNX3xxRcaN26c2rRpU2fbjh071rqdnn7po5R/++03SVJZWZkkadGiRfr4449rtquurlbXrl1VUVHRWHcj8ggp4rp27ar7779fa9eu1axZs7RmzRrNnz+/3m3/+OOPWrfPnDkjSbr55pslSZ07d5YkzZkzR2PHjm28oVsgTu0iIi3t0v/zgv//JevBgwd15MgRSdLkyZNVWlqq3NxcxWKxOqdwl/3444+1bu/fv1+SdOedd0qS7r33XsViMRUXF9f52bffflu7du3y3JkWiJAiomfPnkpJSdGJEyckXXpg79u3T5L0wAMP6KabbtKiRYs0adKkK+5jx44d2rt3r6RL11LLli1T7969lZWVJUm67bbb9Oyzzyo/P18HDx6UdCncFStWaPPmzRo0aFDj3cGI43WkCFmyZIkKCgrUsWNH9e7dW++//37NV468++67Wr58ubZv364bb7yx1s/F43FlZmZq7ty5+uGHH3Tw4EHF43Hddddd9b6OlJ+frxUrViiRSKh9+/bq16+fZs2apVtvvbVJ72+UEFILsW7dOm3dulULFy6ss+5ySPPnz7/qMxbC49Suhdi0aVO9rx2haRBShE2ZMkUVFRUqKSlRPB7XPffck+yRWi1+/R1hbdq00aOPPqr09HS99dZbSklJqbNNbm6uCgoKJEmLFy9WYWGhsrOzm3rUFo9rJMCAUzvAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMDg/wCDllCnRfwWvwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "creak\n",
      "probe acc 0.6710430383682251\n",
      "probe sparse 790 8192\n",
      "model acc 0.50255286693573\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO/0lEQVR4nO3da2wUBd+G8bvbgoBoAQGDooSjgXCoQiRGPEBRQKMpkBiM4DlEEyKSGPCNhgSjQYvEcIqx1qhpwymh4aDw4YmQF1AqNJxjVSxEXBAtUIpQpWU77wde6tO0oEzvdjvt9fvEzszO/Dfu5c50u9uUIAgCAWiQWLIHAFoCQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMWLECFVWVqpbt27JHgUtXGlpqdq2bauioqKrbhfJkC5cuKBEIpHsMdAKXLx4Uf/ml38iGVL37t0lSV999VWSJ0FLl5mZ+a+24xoJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgzsIZ09e1YnTpxw7xZo1kKHNHPmzHqX79+/X+PHj1dOTk7ooYCoCR3Szz//XO/yUaNGadu2bVq/fn3ooYCoCR1SSkrKFddVVFTowoULYXcNRE7atWy8dOlSLVu2rOb2wIEDr7jt2LFjw08FRMw1hXT33XdLkoIg0KpVqzRlypS6O0xLU8+ePfXwww97JgQi4JpDuhzT4cOHNWPGjEYZCoia0NdIH3zwwVXX79ixI+yugchptDdks7OzG2vXQLNzTad2/+3UqVNasGCBduzYoZMnT6q6uto5FxApoUN64403VFJSojFjxqhz586Kxf5+cbv8wwigtQgd0nfffacvv/xSN9xwQ73r4/F46KGAqAl9jdSzZ88rRiRJ7733XthdA5ETOqTx48dr06ZNV1z/zDPPhN01EDmhT+2Ki4uVm5urDz/8UH369FH79u1rrf/+++8bPBwQFaFD2rBhg7p3765z585p//79ddZXVFQ0aDAgSkKH1K9fP61du/aK67OyssLuGoic0NdIb7755lXX84YsWpPQIY0YMeKq63fu3Bl210DkhD61O378+FXXr1y5UlOnTg27eyBSQoc0ZsyYq364D2hNQofUo0cPvfLKK7WWnT9/XiUlJdq5c6emTZvW4OGAqAgd0oQJEzRx4sR615WUlGj58uWhhwKiJvQPG2bPnn3FdX379tXevXvD7hqInEb5PNKhQ4d0+vTpxtg10CyFPrV7+umn6ywLgkDl5eU6fPiwnnzyyQYNBkRJ6JAOHDigwYMH11qWmpqq/v37a9q0aVe8fgJaotAh9erVS3l5ec5ZgMgKfY302WefGccAoi30K1KnTp0kSUVFRdq2bZvKysrUuXNn3Xffff/460NASxM6pEQiodmzZ2vjxo0KgqBmeU5Ojh555BEtWLCg1vc4AC1Z6JA++ugj7dq1S3PmzFFGRoY6deqkM2fOaO/evfr000+Vk5Ojl156yTkr0GyFDmn9+vXKy8tTr169ai3PyMjQ6NGjNX36dEJCqxH63CsWi9WJ6LJevXopNTU19FBA1IQOqbKyUmVlZfWuO336NH/WBa1K6JAyMzP13HPPacuWLTp16pQuXryoU6dOafPmzXrhhRf00EMPOecEmrXQ10gzZ87U888/r5dffrnO55KGDRtW5yMWQEsWOqQOHTooPz9fGzZs0Pbt22veRxo1apQee+wxpaWF3jUQOaGf7WVlZdq9e7e6dOmihQsX1izfuHGjzp49qy5dulgGBKIg9DVSfn6+Zs6cqXXr1tVa/u233yorK0s//fRTg4cDoiL0K9KWLVv0ySefaOTIkbWWz5s3T6NHj1Z2drZycnIaPCAQBaFfkYIgqBPRZQ8++KBKS0tDDwVETeiQysvLG7QeaElCh9S3b18tXrxYiUSi1vJEIqFFixapT58+DR4OiIoGvY80depUrV69WgMHDlR6errKy8tVXFysc+fO8S1CaFVChzR48GDl5eUpOztbX3/9taqrqxWLxTR8+HDNnj1bgwYNcs4JNGsNetd0yJAhysvL019//aXy8nKlp6erXbt2rtmAyLD8+kG7du0ICK0aH2EFDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkACDFhNSdXV1qzw2mgfLX+xrDmKxmFavXq3S0tImPW63bt30xBNPNOkx0fy0mJAkqbS0VMePH0/2GGiFWsypHZBMhNTIgiRePyXz2K1Nizq1a45SYjEdWZ+tP08ebdLjtu96u3o/PrtJj9maEVIT+PPkUf35W0myx0Aj4tQOMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCCkVixRXd1sjx0EQRNN4jl2WiPMgYhIjcU0/3//o6PlZU163NvTO+t/HnjoqtukpKTo999/V1VVVRNNdUmbNm3UvXv3a74fIbVyR8vL9NOpk8keo15VVVWqrKxM9hj/Cqd2gAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYpARBECR7iGs1ZMgQJRIJ9ejRo9by8+fPK5FINOksqampuv7666+6zcWKcgWJi0000SUpqWlK65D+j9ud+etPXayuboKJ/pYWi6lTu/b/uF1T/7e8LDU1tebfv/76q1JTU3XgwIGr3ietsYdqDNddd50qKyvrLP+nJ3Sy/JsndLL8myd0svz3EzpZ0tLS1LZt23/cLpKvSEBzwzUSYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIUVEPB7XkiVLFI/Hkz0K6kFIEXHs2DEtXbpUx44dS/YoqAchAQZ8i1AE5ObmatWqVTp69Khuv/12dejQQefOndPp06dVUVGhPn36aOrUqXrqqadUVFSkefPmqaSkRBkZGRozZowKCgpUUlKi119/XcXFxTp06JDi8biGDx+uuXPn6pZbbqk5VhAE+vzzz7Vy5UpJUlVVlcaOHatXX31V7ds336/uSroAkVBYWBgMGDAgKCwsrFm2b9++YMCAAcHq1atrbbt169Zg+vTpNbd/+eWXYMCAAcHIkSODPXv2BEEQBGfOnAmysrKCcePGBVVVVTXbvvPOO8GwYcOCffv2BUEQBKWlpcGECROCF198sREfXfRxahdhQ4cOVf/+/bVmzZpaywsKCjR58uQ622dmZiojI0OSlJ6erhkzZujIkSNau3atJOno0aPKy8vTxIkTNXToUElS165dNX36dG3dulVFRUWN+niijJAibtKkSdqzZ48OHz4sSSovL9fu3bs1evToOtvecccdtW5fjmXPnj2SpG+++UbV1dUaPnx4vfcrLCy0z99SRPIri/G3xx9/XAsXLlRBQYFee+01ffHFFxo3bpzatGlTZ9uOHTvWup2efumrlH/77TdJUllZmSRp0aJF+vjjj2u2q66uVteuXVVRUdFYDyPyCCniunbtqvvvv19r167VrFmztGbNGs2fP7/ebf/4449at8+cOSNJuvnmmyVJnTt3liTNmTNHY8eObbyhWyBO7SIiLe3S//OC//8h68GDB3XkyBFJ0uTJk1VaWqrc3FzFYrE6p3CX/fjjj7Vu79+/X5J05513SpLuvfdexWIxFRcX17nv22+/rV27dnkeTAtESBHRs2dPpaSk6MSJE5IuPbH37dsnSXrggQd00003adGiRZo0adIV97Fjxw7t3btX0qVrqWXLlql3797KysqSJN1222169tlnlZ+fr4MHD0q6FO6KFSu0efNmDRo0qPEeYMTxPlKELFmyRAUFBerYsaN69+6t999/v+ZPjrz77rtavny5tm/frhtvvLHW/eLxuDIzMzV37lz98MMPOnjwoOLxuO66665630fKz8/XihUrlEgk1L59e/Xr10+zZs3Srbfe2qSPN0oIqYVYt26dtm7dqoULF9ZZdzmk+fPnX/UVC+FxatdCbNq0qd73jtA0CCnCpkyZooqKCpWUlCgej+uee+5J9kitFj/+jrA2bdro0UcfVXp6ut566y2lpKTU2SY3N1cFBQWSpMWLF6uwsFDZ2dlNPWqLxzUSYMCpHWBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYPB//uVQo8EaQwsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "creak\n",
      "probe acc 0.5973742008209229\n",
      "probe sparse 89 8192\n",
      "model acc 0.50255286693573\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO/klEQVR4nO3deWwUhd/H8U+3BQHRAgIGRQmngXBUIRIjHlAU0GgKJAYjeIdoQkQSAz7RkGA0aJEYrhhrjZo2XAkNh8Ifvwh5AKVCwx2rYiHigmiBUoQqLbvz/MFD/TUtKNNPj2nfr7/YmenMd+O+3Znd7jYlCIJAAOol1tQDAC0BIQEGhAQYEBJgQEiAASEBBoQEGBASYJDW1AOEMWLECFVWVqpbt25NPQpauNLSUrVt21ZFRUVX3S6SIV24cEGJRKKpx0ArcPHiRf2bX/6JZEjdu3eXJH311VdNPAlauszMzH+1HddIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBgD+ns2bM6ceKEe7dAsxY6pJkzZ9a5fP/+/Ro/frxycnJCDwVETeiQfv755zqXjxo1Stu2bdP69etDDwVETeiQUlJSrriuoqJCFy5cCLtrIHLSrmXjpUuXatmyZdW3Bw4ceMVtx44dG34qIGKuKaS7775bkhQEgVatWqUpU6bU3mFamnr27KmHH37YMyEQAdcc0uWYDh8+rBkzZjTIUEDUhL5G+uCDD666fseOHWF3DUROg70hm52d3VC7Bpqdazq1+2+nTp3SggULtGPHDp08eVLJZNI5FxApoUN64403VFJSojFjxqhz586Kxf5+crv8YgTQWoQO6bvvvtOXX36pG264oc718Xg89FBA1IS+RurZs+cVI5Kk9957L+yugcgJHdL48eO1adOmK65/5plnwu4aiJzQp3bFxcXKzc3Vhx9+qD59+qh9+/Y11n///ff1Hg6IitAhbdiwQd27d9e5c+e0f//+WusrKirqNRgQJaFD6tevn9auXXvF9VlZWWF3DURO6GukN99886rreUMWrUnokEaMGHHV9Tt37gy7ayByQp/aHT9+/KrrV65cqalTp4bdPRApoUMaM2bMVT/cB7QmoUPq0aOHXnnllRrLzp8/r5KSEu3cuVPTpk2r93BAVIQOacKECZo4cWKd60pKSrR8+fLQQwFRE/rFhtmzZ19xXd++fbV3796wuwYip0E+j3To0CGdPn26IXYNNEuhT+2efvrpWsuCIFB5ebkOHz6sJ598sl6DAVESOqQDBw5o8ODBNZalpqaqf//+mjZt2hWvn4CWKHRIvXr1Ul5ennMWILJCXyN99tlnxjGAaAv9jNSpUydJUlFRkbZt26aysjJ17txZ99133z/++hDQ0oQOKZFIaPbs2dq4caOCIKhenpOTo0ceeUQLFiyo8T0OQEsWOqSPPvpIu3bt0pw5c5SRkaFOnTrpzJkz2rt3rz799FPl5OTopZdecs4KNFuhQ1q/fr3y8vLUq1evGsszMjI0evRoTZ8+nZDQaoQ+94rFYrUiuqxXr15KTU0NPRQQNaFDqqysVFlZWZ3rTp8+zZ91QasSOqTMzEw999xz2rJli06dOqWLFy/q1KlT2rx5s1544QU99NBDzjmBZi30NdLMmTP1/PPP6+WXX671uaRhw4bV+ogF0JKFDqlDhw7Kz8/Xhg0btH379ur3kUaNGqXHHntMaWmhdw1ETuhHe1lZmXbv3q0uXbpo4cKF1cs3btyos2fPqkuXLpYBgSgIfY2Un5+vmTNnat26dTWWf/vtt8rKytJPP/1U7+GAqAj9jLRlyxZ98sknGjlyZI3l8+bN0+jRo5Wdna2cnJx6DwhEQehnpCAIakV02YMPPqjS0tLQQwFREzqk8vLyeq0HWpLQIfXt21eLFy9WIpGosTyRSGjRokXq06dPvYcDoqJe7yNNnTpVq1ev1sCBA5Wenq7y8nIVFxfr3LlzfIsQWpXQIQ0ePFh5eXnKzs7W119/rWQyqVgspuHDh2v27NkaNGiQc06gWavXu6ZDhgxRXl6e/vrrL5WXlys9PV3t2rVzzQZEhuXXD9q1a0dAaNX4CCtgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBi0mJCSyWSrPDaahxbzR4xisZhWr17d6N853q1bNz3xxBONekw0Py0mJEkqLS3V8ePHm3oMtEIt5tQOaEqEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBKapSAIInXstAaYA6i3lJQU/f7776qqqmrU47Zp00bdu3e/5p8jJDRbVVVVqqysbOox/hVO7QADQgIMCAkwIKQGFiSTrfLYrQ0vNjSwlFhMR9Zn68+TRxv1uO273q7ej89u1GO2ZoTUCP48eVR//lbS1GOgAXFqBxgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASK1YIplslcduCGlNPQCaTmospvn/+x8dLS9r1OPent5Z//PAQ416zIZGSK3c0fIy/XTqZFOPEXmc2gEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYJASBEHQ1ENcqyFDhiiRSKhHjx41lp8/f16JRKJRZ0lNTdX1119/1W0uVpQrSFxspIkuSUlNU1qH9H/c7sxff+piMtkIE/0tLRZTp3bt/3G7xv5veVlqamr1v3/99VelpqbqwIEDV/2ZtIYeqiFcd911qqysrLX8nx7QTeXfPKCbyr95QDeV/35AN5W0tDS1bdv2H7eL5DMS0NxwjQQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIERGPx7VkyRLF4/GmHgV1IKSIOHbsmJYuXapjx4419SioAyEBBnyLUATk5uZq1apVOnr0qG6//XZ16NBB586d0+nTp1VRUaE+ffpo6tSpeuqpp1RUVKR58+appKREGRkZGjNmjAoKClRSUqLXX39dxcXFOnTokOLxuIYPH665c+fqlltuqT5WEAT6/PPPtXLlSklSVVWVxo4dq1dffVXt2zffr+5qcgEiobCwMBgwYEBQWFhYvWzfvn3BgAEDgtWrV9fYduvWrcH06dOrb//yyy/BgAEDgpEjRwZ79uwJgiAIzpw5E2RlZQXjxo0Lqqqqqrd95513gmHDhgX79u0LgiAISktLgwkTJgQvvvhiA9676OPULsKGDh2q/v37a82aNTWWFxQUaPLkybW2z8zMVEZGhiQpPT1dM2bM0JEjR7R27VpJ0tGjR5WXl6eJEydq6NChkqSuXbtq+vTp2rp1q4qKihr0/kQZIUXcpEmTtGfPHh0+fFiSVF5ert27d2v06NG1tr3jjjtq3L4cy549eyRJ33zzjZLJpIYPH17nzxUWFtrnbyki+ZXF+Nvjjz+uhQsXqqCgQK+99pq++OILjRs3Tm3atKm1bceOHWvcTk+/9FXKv/32mySprKxMkrRo0SJ9/PHH1dslk0l17dpVFRUVDXU3Io+QIq5r1666//77tXbtWs2aNUtr1qzR/Pnz69z2jz/+qHH7zJkzkqSbb75ZktS5c2dJ0pw5czR27NiGG7oF4tQuItLSLv0/L/j/F1kPHjyoI0eOSJImT56s0tJS5ebmKhaL1TqFu+zHH3+scXv//v2SpDvvvFOSdO+99yoWi6m4uLjWz7799tvatWuX5860QIQUET179lRKSopOnDgh6dIDe9++fZKkBx54QDfddJMWLVqkSZMmXXEfO3bs0N69eyVdupZatmyZevfuraysLEnSbbfdpmeffVb5+fk6ePCgpEvhrlixQps3b9agQYMa7g5GHO8jRciSJUtUUFCgjh07qnfv3nr//fer/+TIu+++q+XLl2v79u268cYba/xcPB5XZmam5s6dqx9++EEHDx5UPB7XXXfdVef7SPn5+VqxYoUSiYTat2+vfv36adasWbr11lsb9f5GCSG1EOvWrdPWrVu1cOHCWusuhzR//vyrPmMhPE7tWohNmzbV+d4RGgchRdiUKVNUUVGhkpISxeNx3XPPPU09UqvFy98R1qZNGz366KNKT0/XW2+9pZSUlFrb5ObmqqCgQJK0ePFiFRYWKjs7u7FHbfG4RgIMOLUDDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIM/g+GpFCnNhkBXwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "creak\n",
      "probe acc 0.4945295453071594\n",
      "probe sparse 38 8192\n",
      "model acc 0.50255286693573\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO+ElEQVR4nO3deWwUhd/H8U+3BQHRUgQMihJOA+GoQiRGPKAooNEUSAxG8A7RhIgkBnyiIcFo0CIxXDHWGjVtuBIaDoU/fhHyAEqFhjtWxULEBdECpQhVju08f/BQf03LNf2022nfr7/cmdnZ78Z9szO73d2UIAgCAaiXWLIHAJoDQgIMCAkwICTAgJAAA0ICDAgJMCAkwCAt2QOEMXToUJ07d06dO3dO9iho5srKytS6dWsVFxdfcbtIhnT27FklEolkj4EW4MKFC7qWP/6JZEhdunSRJH3zzTdJngTNXVZW1jVtxzkSYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBjYQzp16pSOHj3q3i3QpIUOadq0aXUu37Nnj8aMGaPc3NzQQwFREzqkX3/9tc7lw4cP1+bNm7VmzZrQQwFREzqklJSUy66rrKzU2bNnw+4aiJy069l40aJFWrx4cfXlfv36XXbbUaNGhZ8KiJjrCunee++VJAVBoOXLl2vixIm1d5iWpm7duunRRx/1TAhEwHWHdCmmAwcOaOrUqQ0yFBA1oc+RPvrooyuu37p1a9hdA5HTYG/I5uTkNNSugSbnug7t/tvx48c1d+5cbd26VceOHVNVVZVzLiBSQof01ltvqbS0VCNHjlRGRoZisX+f3C69GAG0FKFD+uGHH/T111/rpptuqnN9PB4PPRQQNaHPkbp163bZiCTpgw8+CLtrIHJChzRmzBitX7/+suufe+65sLsGIif0oV1JSYny8vL08ccfq2fPnmrbtm2N9T/++GO9hwOiInRIa9euVZcuXXT69Gnt2bOn1vrKysp6DQZESeiQevfurVWrVl12fXZ2dthdA5ET+hzp7bffvuJ63pBFSxI6pKFDh15x/bZt28LuGoic0Id2R44cueL6ZcuWadKkSWF3D0RK6JBGjhx5xQ/3AS1J6JC6du2q1157rcayM2fOqLS0VNu2bdPkyZPrPRwQFaFDGjt2rMaNG1fnutLSUi1ZsiT0UEDUhH6xYcaMGZdd16tXL+3atSvsroHIaZDPI+3fv18nTpxoiF0DTVLoQ7tnn3221rIgCFRRUaEDBw7o6aefrtdgQJSEDmnv3r0aMGBAjWWpqanq06ePJk+efNnzJ6A5Ch1S9+7dlZ+f75wFiKzQ50hffPGFcYzmK0jiR/CTedstTehnpA4dOkiSiouLtXnzZpWXlysjI0MPPPDAVf98qCVJicV0cE2O/j52qFFvt22nO9Xjycu/sgqv0CElEgnNmDFD69atUxAE1ctzc3P12GOPae7cuTW+x6El+/vYIf39R2myx0ADCh3SJ598ou3bt2vmzJnKzMxUhw4ddPLkSe3atUuff/65cnNz9corrzhnBZqs0CGtWbNG+fn56t69e43lmZmZGjFihKZMmUJIaDFCH3vFYrFaEV3SvXt3paamhh4KiJrQIZ07d07l5eV1rjtx4gQ/64IWJXRIWVlZeuGFF7Rx40YdP35cFy5c0PHjx7Vhwwa99NJLeuSRR5xzAk1a6HOkadOm6cUXX9Srr75a63NJgwcPrvURC6A5Cx1Su3btVFBQoLVr12rLli3V7yMNHz5cTzzxhNLSQu8aiJzQj/by8nLt2LFDHTt21Lx586qXr1u3TqdOnVLHjh0tAwJREPocqaCgQNOmTdPq1atrLP/++++VnZ2tX375pd7DAVER+hlp48aN+uyzzzRs2LAay2fPnq0RI0YoJydHubm59R4QiILQz0hBENSK6JKHH35YZWVloYcCoiZ0SBUVFfVaDzQnoUPq1auXFixYoEQiUWN5IpHQ/Pnz1bNnz3oPB0RFvd5HmjRpklasWKF+/fopPT1dFRUVKikp0enTp/kWIbQooUMaMGCA8vPzlZOTo2+//VZVVVWKxWIaMmSIZsyYof79+zvnBJq0er1rOnDgQOXn5+uff/5RRUWF0tPT1aZNG9dsQGRY/vygTZs2BIQWjY+wAgaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQUguWSOIv+iXzthsCX4fagqXGYprzv//RoYq6fwyhodyZnqH/eah5fTc8IbVwhyrK9cvxY8keI/I4tAMMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDJpNSFVVVS3yttE0pCV7AJdYLKYVK1aorKysUW+3c+fOeuqppxr1NtH0NJuQJKmsrExHjhxJ9hhogZrNoR2QTIQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBKapCAIInXbzeo3ZNF8pKSk6M8//9T58+cb9XZbtWqlLl26XPf1CAlN1vnz53Xu3Llkj3FNUoJkPoeGNHDgQCUSCXXt2rXG8jNnziiRSDTqLKmpqbrxxhuvuM2FygoFiQuNNNFFKalpSmuXftXtTv7zty5UVTXCRP9Ki8XUoU3bq27X2P8vL0lNTa3+799//12pqanau3fvFa8TyWekG264oc5/qa72gE6Wa3lAJ8u1PKCT5b8f0MmSlpam1q1bX3W7SD4jAU0Nr9oBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQUkTE43EtXLhQ8Xg82aOgDoQUEYcPH9aiRYt0+PDhZI+COhASYMC3CEVAXl6eli9frkOHDunOO+9Uu3btdPr0aZ04cUKVlZXq2bOnJk2apGeeeUbFxcWaPXu2SktLlZmZqZEjR6qwsFClpaV68803VVJSov379ysej2vIkCGaNWuWbrvtturbCoJAX375pZYtWybp4pc0jho1Sq+//rratm26X92VdAEioaioKOjbt29QVFRUvWz37t1B3759gxUrVtTYdtOmTcGUKVOqL//2229B3759g2HDhgU7d+4MgiAITp48GWRnZwejR48Ozp8/X73te++9FwwePDjYvXt3EARBUFZWFowdOzZ4+eWXG/DeRR+HdhE2aNAg9enTRytXrqyxvLCwUBMmTKi1fVZWljIzMyVJ6enpmjp1qg4ePKhVq1ZJkg4dOqT8/HyNGzdOgwYNkiR16tRJU6ZM0aZNm1RcXNyg9yfKCCnixo8fr507d+rAgQOSpIqKCu3YsUMjRoyote1dd91V4/KlWHbu3ClJ+u6771RVVaUhQ4bUeb2ioiL7/M1FJL+yGP968sknNW/ePBUWFuqNN97QV199pdGjR6tVq1a1tm3fvn2Ny+npF79K+Y8//pAklZeXS5Lmz5+vTz/9tHq7qqoqderUSZWVlQ11NyKPkCKuU6dOevDBB7Vq1SpNnz5dK1eu1Jw5c+rc9q+//qpx+eTJk5KkW2+9VZKUkZEhSZo5c6ZGjRrVcEM3QxzaRURa2sV/84L/f5F13759OnjwoCRpwoQJKisrU15enmKxWK1DuEt+/vnnGpf37NkjSbr77rslSffff79isZhKSkpqXffdd9/V9u3bPXemGSKkiOjWrZtSUlJ09OhRSRcf2Lt375YkPfTQQ7rllls0f/58jR8//rL72Lp1q3bt2iXp4rnU4sWL1aNHD2VnZ0uS7rjjDj3//PMqKCjQvn37JF0Md+nSpdqwYYP69+/fcHcw4ngfKUIWLlyowsJCtW/fXj169NCHH35Y/ZMj77//vpYsWaItW7bo5ptvrnG9eDyurKwszZo1Sz/99JP27duneDyue+65p873kQoKCrR06VIlEgm1bdtWvXv31vTp03X77bc36v2NEkJqJlavXq1NmzZp3rx5tdZdCmnOnDlXfMZCeBzaNRPr16+v870jNA5CirCJEyeqsrJSpaWlisfjuu+++5I9UovFy98R1qpVKz3++ONKT0/XO++8o5SUlFrb5OXlqbCwUJK0YMECFRUVKScnp7FHbfY4RwIMOLQDDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIM/g+Yu1CjRssAugAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "creak\n",
      "probe acc 0.5193289518356323\n",
      "probe sparse 90 8192\n",
      "model acc 0.50255286693573\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANIAAAGpCAYAAAD4C2j0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAO7UlEQVR4nO3daWwVBd+G8bsLSBEtxYJBEcJqICxViMSICxQFNJoCicEI7iGaEJHEgE80JBgNWiSGLcZao6YNW0LDovDhjZAXUCo07LEqFiIeFC1bEaq0nM7zgZc+T9+yTu/2dNrr94kzM535n3guz5w9KQiCQAAaJDnRAwAtASEBBoQEGBASYEBIgAEhAQaEBBgQEmCQmugBwhg2bJiqqqrUuXPnRI+CFq68vFxt27ZVSUnJFbeLZEjnzp1TPB5P9BhoBc6fP69refNPJEPq0qWLJOnrr79O8CRo6bKzs69pOx4jAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASICBPaTTp0/r6NGj7t0CzVrokKZPn37J5Xv37tXYsWOVl5cXeiggakKH9Msvv1xy+YgRI7RlyxatXbs29FBA1IQOKSkp6bLrKisrde7cubC7BiIn9Xo2Xrx4sZYsWVJ7uX///pfddvTo0eGnAiLmukK65557JElBEGjFihWaNGlS/R2mpqpbt2565JFHPBMCEXDdIV2M6eDBg5o2bVqjDAVETejHSB9++OEV12/bti3sroHIabQXZHNzcxtr10Czc12ndv/t+PHjmjdvnrZt26Zjx46ppqbGORcQKaFDevPNN1VWVqZRo0YpIyNDycn/uXO7+GQE0FqEDun777/XV199pZtuuumS62OxWOihgKgJ/RipW7dul41Ikt5///2wuwYiJ3RIY8eO1YYNGy67/tlnnw27ayByQp/alZaWKj8/Xx999JF69eqltLS0Out/+OGHBg8HREXokNatW6cuXbrozJkz2rt3b731lZWVDRoMiJLQIfXp00erV6++7PqcnJywuwYiJ/RjpLfeeuuK63lBFq1J6JCGDRt2xfXbt28Pu2sgckKf2v32229XXL98+XJNnjw57O6BSAkd0qhRo6744T6gNQkdUteuXfXqq6/WWXb27FmVlZVp+/btmjJlSoOHA6IidEjjxo3T+PHjL7murKxMS5cuDT0UEDWhn2yYOXPmZdf17t1bu3fvDrtrIHIa5fNIBw4c0IkTJxpj10CzFPrU7plnnqm3LAgCVVRU6ODBg3rqqacaNBgQJaFD2rdvnwYOHFhnWUpKivr27aspU6Zc9vET0BKFDqlHjx4qKChwzgJEVujHSJ9//rlxDCDaQt8jdezYUZJUUlKiLVu26OTJk8rIyND9999/1bcPAS1N6JDi8bhmzpyp9evXKwiC2uV5eXl69NFHNW/evDrf4wC0ZKFD+vjjj7Vjxw7NmjVLWVlZ6tixo06dOqXdu3frs88+U15enl5++WXnrECzFTqktWvXqqCgQD169KizPCsrSyNHjtTUqVObNKSampqE3QMm8thoHkKHlJycXC+ii3r06KGUlJTQQ4WdZ+XKlSovL2/S43bu3FlPPvlkkx4TzU/okKqqqmqfYPj/Tpw4kZCfdSkvL7/qxzuAxhD6fCQ7O1vPP/+8Nm3apOPHj+v8+fM6fvy4Nm7cqBdffFEPP/ywc06gWQt9jzR9+nS98MILeuWVV+p9LmnIkCH1PmIBtGShQ2rfvr0KCwu1bt06bd26tfY0b8SIEXr88ceVmhp610DkhL61nzx5Ujt37lSnTp00f/782uXr16/X6dOn1alTJ8uAQBSEfoxUWFio6dOna82aNXWWf/fdd8rJydHPP//c4OGAqAh9j7Rp0yZ9+umnGj58eJ3lc+bM0ciRI5Wbm6u8vLwGDwhEQeh7pCAI6kV00UMPPdTkr+cAiRQ6pIqKigatB1qS0CH17t1bCxcuVDwer7M8Ho9rwYIF6tWrV4OHA6KiQa8jTZ48WStXrlT//v2Vnp6uiooKlZaW6syZM3yLEFqV0CENHDhQBQUFys3N1TfffFP7xs2hQ4dq5syZGjBggHNOoFlr0KumgwYNUkFBgf755x9VVFQoPT1d7dq1c80GRIbl7Qft2rUjILRqfIgGMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICQ0S0EQROrY/BoYmqWkpCT9+eefqq6ubtLjtmnTRl26dLnuvyMkNFvV1dWqqqpK9BjXhFM7wICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgJMCAkAADQmpkQU1Nqzx2a5Oa6AFauqTkZB1am6u/jx1u0uOmZXZXzydmNukxWzNCagJ/Hzusv/8oS/QYaESc2gEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIgAEhAQaEBBgQEmBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYEBIrVg8gb/ol8hjNwZ+aKwVS0lO1tz//R8drjjZpMftnp6hfz34cJMes7ERUit3uOKkfj5+LNFjRB6ndoABIQEGhAQYEBJgQEiAASEBBoQEGCQFQRAkeojrNWjQIMXjcXXt2rXO8rNnzyoejzfpLCkpKbrxxhuvuM35ygoF8fNNNNEFSSmpSm2fftXtTv3zt8438bsMUpOT1bFd2lW3a+r/lhelpKTU/vv3339XSkqK9u3bd8W/ieQLsjfccIOqqqrqLb/aDTpRruUGnSjXcoNOlP++QSdKamqq2rZte9XtInmPBDQ3PEYCDAgJMCAkwICQAANCAgwICTAgJMCAkAADQgIMCAkwICTAgJAAA0ICDAgJMCAkwICQAANCAgwICTAgpIiIxWJatGiRYrFYokfBJRBSRBw5ckSLFy/WkSNHEj0KLoGQAAO+RSgC8vPztWLFCh0+fFjdu3dX+/btdebMGZ04cUKVlZXq1auXJk+erKefflolJSWaM2eOysrKlJWVpVGjRqmoqEhlZWV64403VFpaqgMHDigWi2no0KGaPXu2brvtttpjBUGgL774QsuXL5ckVVdXa/To0XrttdeUltZ8v7or4QJEQnFxcdCvX7+guLi4dtmePXuCfv36BStXrqyz7ebNm4OpU6fWXv7111+Dfv36BcOHDw927doVBEEQnDp1KsjJyQnGjBkTVFdX12777rvvBkOGDAn27NkTBEEQlJeXB+PGjQteeumlRrx20cepXYQNHjxYffv21apVq+osLyoq0sSJE+ttn52draysLElSenq6pk2bpkOHDmn16tWSpMOHD6ugoEDjx4/X4MGDJUmZmZmaOnWqNm/erJKSkka9PlFGSBE3YcIE7dq1SwcPHpQkVVRUaOfOnRo5cmS9be+88846ly/GsmvXLknSt99+q5qaGg0dOvSSf1dcXGyfv6WI5FcW4z+eeOIJzZ8/X0VFRXr99df15ZdfasyYMWrTpk29bTt06FDncnr6ha9S/uOPPyRJJ09e+FHmBQsW6JNPPqndrqamRpmZmaqsrGysqxF5hBRxmZmZeuCBB7R69WrNmDFDq1at0ty5cy+57V9//VXn8qlTpyRJt956qyQpIyNDkjRr1iyNHj268YZugTi1i4jU1Av/zwv+70nW/fv369ChQ5KkiRMnqry8XPn5+UpOTq53CnfRTz/9VOfy3r17JUl33XWXJOm+++5TcnKySktL6/3tO++8ox07dniuTAtESBHRrVs3JSUl6ejRo5Iu3LD37NkjSXrwwQd1yy23aMGCBZowYcJl97Ft2zbt3r1b0oXHUkuWLFHPnj2Vk5MjSbrjjjv03HPPqbCwUPv375d0Idxly5Zp48aNGjBgQONdwYjjdaQIWbRokYqKitShQwf17NlTH3zwQe1Pjrz33ntaunSptm7dqptvvrnO38ViMWVnZ2v27Nn68ccftX//fsViMd19992XfB2psLBQy5YtUzweV1pamvr06aMZM2bo9ttvb9LrGyWE1EKsWbNGmzdv1vz58+utuxjS3Llzr3iPhfA4tWshNmzYcMnXjtA0CCnCJk2apMrKSpWVlSkWi+nee+9N9EitFk9/R1ibNm302GOPKT09XW+//baSkpLqbZOfn6+ioiJJ0sKFC1VcXKzc3NymHrXF4zESYMCpHWBASIABIQEGhAQYEBJgQEiAASEBBoQEGBASYPBvm9VQ5hV6pDwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 200x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for dataset in data:\n",
    "    for l1 in [0, 0.01, 0.03, 0.1, 0.3]:\n",
    "        n_train, model_dim = data_processed[dataset][\"train\"][\"good_repr\"].shape\n",
    "        probe = train_probe(data_processed[dataset][\"train\"], data_processed[dataset][\"valid\"], l1=l1)\n",
    "        probe.layer.weight.data[probe.layer.weight.data < 1e-3] = 0\n",
    "        probe_preds = torch.sigmoid(\n",
    "            probe(data_processed[dataset][\"valid\"][\"good_repr\"])[:, 1]\n",
    "            - probe(data_processed[dataset][\"valid\"][\"bad_repr\"])[:, 1]\n",
    "        ).detach().cpu().numpy()\n",
    "        lm_preds = torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()).detach().cpu().numpy()\n",
    "        lm_correct = lm_preds > 0.499\n",
    "        probe_correct = probe_preds > 0.499\n",
    "        \n",
    "        lm_probe = lm_correct & probe_correct\n",
    "        lm_noprobe = lm_correct & ~probe_correct\n",
    "        nolm_probe = ~lm_correct & probe_correct\n",
    "        nolm_noprobe = ~lm_correct & ~probe_correct\n",
    "        cat_labels = 0 * lm_probe + 1 * nolm_probe + 2 * lm_noprobe + 3 * nolm_noprobe\n",
    "        type_order = [\"both right\", \"probe correct\", \"pred correct\", \"both wrong\"]\n",
    "        #palette = dict(zip(type_order, sns.husl_palette(5, h=0.1)))\n",
    "        names = dict(enumerate(type_order))\n",
    "        base_palette = sns.husl_palette(5, h=0.1)\n",
    "        neutral = (.48, .48, .48)\n",
    "        neutral_light = (0.87, 0.87, 0.87)\n",
    "        palette = {\"both right\": neutral, \"probe correct\": base_palette[0], \"pred correct\": base_palette[2], \"both wrong\": neutral_light}\n",
    "        named_labels = [names[c] for c in cat_labels]\n",
    "        df = pd.DataFrame({\n",
    "            'lm': lm_preds,\n",
    "            'probe': probe_preds,\n",
    "            'type': named_labels,\n",
    "        }).sort_values(by=\"type\")\n",
    "        \n",
    "        print(dataset)\n",
    "        print(\"probe acc\", eval_probe(probe, data_processed[dataset][\"valid\"]))\n",
    "        print(\"probe sparse\", (probe.layer.weight.abs() != 0).int().sum().item(), probe.layer.weight.shape[0] * probe.layer.weight.shape[1])\n",
    "        print(\"model acc\", data_processed[dataset][\"valid\"][\"correct\"].float().mean().item())\n",
    "\n",
    "        fig, ax = plt.subplots(figsize=(2, 4.8))\n",
    "        sns.countplot(ax=ax, data=df, x=\"type\", order=type_order, palette=palette)\n",
    "        ax.xaxis.set_ticklabels([])\n",
    "        ax.yaxis.set_ticklabels([])\n",
    "        plt.ylim(0, len(df))\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "        \n",
    "        print()\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Probe superiority seems mainly attributable to probe calibration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def plot_calibration(probe_preds):\n",
    "    p_true = probe_preds\n",
    "    p_false = 1 - p_true\n",
    "    calib_range = []\n",
    "    calib_acc = []\n",
    "    calib_size = []\n",
    "    step = 0.1\n",
    "    for start in np.arange(0, 1 + step, step):\n",
    "        end = start + step\n",
    "        pred_true = ((start <= p_true) & (p_true < end)).sum()\n",
    "        pred_false = ((start <= p_false) & (p_false < end)).sum()\n",
    "        if pred_true + pred_false == 0:\n",
    "            continue\n",
    "        frac_true = pred_true / (pred_true + pred_false)\n",
    "        calib_range.append(start)\n",
    "        calib_acc.append(frac_true)\n",
    "        calib_size.append(pred_true + pred_false)\n",
    "\n",
    "    data = pd.DataFrame({\"range\": calib_range, \"acc\": calib_acc, \"size\": calib_size})\n",
    "    fig, ax = plt.subplots(figsize=(3, 3))\n",
    "    sns.scatterplot(ax=ax, data=data, x=\"range\", y=\"acc\", size=\"size\")\n",
    "    ax.get_legend().remove()\n",
    "\n",
    "    plt.xlim(0, 1)\n",
    "    plt.ylim(0, 1)\n",
    "    plt.show()\n",
    "\n",
    "for dataset in data:\n",
    "    print(dataset)\n",
    "    plot_calibration(probe_preds[dataset])\n",
    "    plot_calibration(torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()).detach().cpu().numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models are not strictly better than probes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for dataset in data:\n",
    "    lm_dev_preds = torch.sigmoid(data_processed[dataset][\"train\"][\"ratio\"][-N_DEV:].float()).detach().cpu().numpy()\n",
    "    lm_preds = torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()).detach().cpu().numpy()\n",
    "    lm_correct = lm_preds > 0.499\n",
    "    probe_correct = probe_preds[dataset] > 0.499\n",
    "    \n",
    "    ensemble_scores = []\n",
    "    for d in np.arange(0, 1, 0.05):\n",
    "        dev_acc = ((d * lm_dev_preds + (1-d) * probe_dev_preds[dataset]) > 0.5).mean()\n",
    "        acc = ((d * lm_preds + (1-d) * probe_preds[dataset]) > 0.5).mean()\n",
    "        ensemble_scores.append((dev_acc, acc))\n",
    "    ensemble_corrects = max(ensemble_scores)[1]\n",
    "        \n",
    "    lm_probe = lm_correct & probe_correct\n",
    "    lm_noprobe = lm_correct & ~probe_correct\n",
    "    nolm_probe = ~lm_correct & probe_correct\n",
    "    nolm_noprobe = ~lm_correct & ~probe_correct\n",
    "    cat_labels = 0 * lm_probe + 1 * nolm_probe + 2 * lm_noprobe + 3 * nolm_noprobe\n",
    "    type_order = [\"both right\", \"probe correct\", \"pred correct\", \"both wrong\"]\n",
    "    #palette = dict(zip(type_order, sns.husl_palette(5, h=0.1)))\n",
    "    names = dict(enumerate(type_order))\n",
    "    base_palette = sns.husl_palette(5, h=0.1)\n",
    "    neutral = (.48, .48, .48)\n",
    "    neutral_light = (0.87, 0.87, 0.87)\n",
    "    palette = {\"both right\": neutral, \"probe correct\": base_palette[0], \"pred correct\": base_palette[2], \"both wrong\": neutral_light}\n",
    "\n",
    "    #names = {0: \"both right\", 1: \"pred correct\", 2: \"probe_correct\", 3: \"both wrong\"}\n",
    "    named_labels = [names[c] for c in cat_labels]\n",
    "    \n",
    "    print(dataset)\n",
    "    \n",
    "    df = pd.DataFrame({\n",
    "        'lm': lm_preds,\n",
    "        'probe': probe_preds[dataset],\n",
    "        'type': named_labels,\n",
    "    }).sort_values(by=\"type\")\n",
    "    \n",
    "    g = sns.JointGrid(data=df, x=\"lm\", y=\"probe\", hue=\"type\", xlim=(0, 1), ylim=(0, 1), space=0, palette=palette)\n",
    "    g.plot_joint(\n",
    "        sns.histplot, #discrete=(True, False),\n",
    "        binrange=[0, 1],\n",
    "        bins=20,\n",
    "        #cmap=\"gist_gray_r\",\n",
    "    )\n",
    "    g.plot_marginals(\n",
    "        sns.histplot, multiple=\"stack\",\n",
    "        binrange=[0, 1],\n",
    "        bins=20,\n",
    "        #color=\"gray\",\n",
    "\n",
    "    )\n",
    "    g.figure.axes[0].set_xticks([0, 0.5, 1])\n",
    "    g.figure.axes[0].set_yticks([0, 0.5, 1])\n",
    "    g.figure.axes[0].set_axisbelow(False)\n",
    "    g.figure.axes[0].grid(color=\"black\")\n",
    "    g.figure.axes[0].get_legend().remove()\n",
    "    #g.plot_joint(sns.scatterplot, sns.histplot)\n",
    "    #g = sns.jointplot(data=df, x=\"lm\", y=\"probe\")\n",
    "    plt.show()\n",
    "    \n",
    "    #sns.relplot(data=df, x='lm', y='probe', hue='type')\n",
    "    #plt.xlim(0, 1)\n",
    "    #plt.ylim(0, 1)\n",
    "    #plt.show()\n",
    "    \n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(2, 4.8))\n",
    "    sns.countplot(ax=ax, data=df, x=\"type\", order=type_order, palette=palette)\n",
    "    ax.xaxis.set_ticklabels([])\n",
    "    ax.yaxis.set_ticklabels([])\n",
    "    plt.ylim(0, len(df))\n",
    "    plt.show()\n",
    "    \n",
    "    #print(\"lm & !probe\", (lm_correct & (~probe_correct)).mean())\n",
    "    #print(\"!lm & probe\", ((~lm_correct) & probe_correct).mean())\n",
    "    #print()\n",
    "    print(\"lm acc      \", lm_correct.mean())\n",
    "    print(\"probe acc   \", probe_correct.mean())\n",
    "    print(\"ensemble acc\", ensemble_corrects)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## When probe and model disagree, probe is just uncertain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in data:\n",
    "    print(dataset)\n",
    "    print()\n",
    "    lm_preds = torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()).detach().cpu().numpy()\n",
    "    disagreements = (lm_preds > 0.5) ^ (probe_preds[dataset] > 0.5)\n",
    "    disagree_prob_lm = abs(lm_preds[disagreements]-.5)\n",
    "    disagree_prob_probe = abs(probe_preds[dataset][disagreements]-.5)\n",
    "    disagree_acc_lm = (lm_preds[disagreements] > 0.5)\n",
    "    disagree_acc_probe = (probe_preds[dataset][disagreements] > 0.5)\n",
    "    agree_prob_lm = abs(lm_preds[~disagreements]-.5)\n",
    "    agree_prob_probe = abs(probe_preds[dataset][~disagreements]-.5)\n",
    "    print(\"lm disagree conf\", disagree_prob_lm.mean(), disagree_prob_lm.std())\n",
    "    print(\"probe disagree conf\", disagree_prob_probe.mean(), disagree_prob_probe.std())\n",
    "    print(\"lm agree conf\", agree_prob_lm.mean(), agree_prob_lm.std())\n",
    "    print(\"probe agree conf\", agree_prob_probe.mean(), agree_prob_probe.std())\n",
    "    print()\n",
    "    \n",
    "    print(\"lm disagree acc\", disagree_acc_lm.mean())\n",
    "    print(\"probe disagree acc\", disagree_acc_probe.mean())\n",
    "    print()\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [\"deception\", \"heterogeneity (probe)\", \"agreement (correct)\", \"confabulation\", \"agreement (uncertain)\", \"heterogeneity (query)\", \"agreement (incorrect)\", \"probe error (query uncertain)\", \"probe error (query correct)\"]\n",
    "dfs = []\n",
    "for dataset in data:\n",
    "    lm_preds = torch.sigmoid(data_processed[dataset][\"valid\"][\"ratio\"].float()).detach().cpu().numpy()\n",
    "    \n",
    "    lm_lc = lm_preds < 0.333\n",
    "    lm_mc = (lm_preds >= 0.333) & (lm_preds < 0.667) #(0.333 <= lm_preds < 0.667)\n",
    "    lm_hc = 0.667 <= lm_preds\n",
    "    probe_lc = probe_preds[dataset] < 0.333\n",
    "    probe_mc = (probe_preds[dataset] >= 0.333) & (probe_preds[dataset] < 0.667) # 0.333 <= probe_preds[dataset] < 0.667\n",
    "    probe_hc = 0.667 <= probe_preds[dataset]\n",
    "    \n",
    "    deception = lm_lc & probe_hc\n",
    "    het_p = lm_mc & probe_hc\n",
    "    agr_c = lm_hc & probe_hc\n",
    "    confab = lm_lc & probe_mc\n",
    "    agr_u = lm_mc & probe_mc\n",
    "    het_l = lm_hc & probe_mc\n",
    "    agr_i = lm_lc & probe_lc\n",
    "    probe_err_u = lm_mc & probe_lc\n",
    "    probe_err_c = lm_hc & probe_lc\n",
    "    \n",
    "    cat_labels = (\n",
    "        0 * deception\n",
    "        + 1 * het_p\n",
    "        + 2 * agr_c\n",
    "        + 3 * confab\n",
    "        + 4 * agr_u\n",
    "        + 5 * het_l\n",
    "        + 6 * agr_i\n",
    "        + 7 * probe_err_u\n",
    "        + 8 * probe_err_c\n",
    "    )\n",
    "    named_labels = [labels[c] for c in cat_labels]\n",
    "    \n",
    "    df = pd.DataFrame({\n",
    "        'lm': [\"gpt2-xl\"] * len(lm_preds),\n",
    "        'dataset': [dataset] * len(lm_preds),\n",
    "        'type': named_labels,\n",
    "    }).groupby(\"dataset\")[\"type\"].value_counts(normalize=True).rename(\"percent\").reset_index()\n",
    "    dfs.append(df)\n",
    "    \n",
    "palette = {\n",
    "    \"deception\": \"#E9A365\",\n",
    "    \"heterogeneity (probe)\": \"#E9D165\",\n",
    "    \"agreement (correct)\": \"#7A7A7A\",\n",
    "    \"confabulation\": \"#F4D1B2\",\n",
    "    \"agreement (uncertain)\": \"#ABABAB\",\n",
    "    \"heterogeneity (query)\": \"#6EC168\",\n",
    "    \"agreement (incorrect)\": \"#DEDEDE\",\n",
    "    \"probe error (query uncertain)\": \"#CDEBE8\",\n",
    "    \"probe error (query correct)\": \"#68C1B8\",\n",
    "}\n",
    "#palette = sns.color_palette(palette)\n",
    "\n",
    "    \n",
    "g = sns.catplot(x=\"type\", y=\"percent\", col=\"dataset\",\n",
    "                data=pd.concat(dfs), kind=\"bar\", palette=palette, order=labels)\n",
    "g.set(xticklabels=[])\n",
    "plt.ylim(0, 0.7)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}