Setup.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "from torch import load, save\n",
    "from numpy import sort\n",
    "from os.path import join\n",
    "from random import shuffle\n",
    "from PIL import Image\n",
    "from matplotlib import pylab as pl\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set user-defined parameters\n",
    "\n",
    "inatpath = '.'             # The location of your downloaded 'train_val_images' folder. \n",
    "annopath = '.'             # The location of your downloaded 'train_2017_bboxes.json' file. \n",
    "datapath = '.'             # Wherever you want your new dataset to appear. \n",
    "catsize_min = 50           # The smallest category allowed in our data set\n",
    "catsize_max = 1000         # The largest category allowed in our data set\n",
    "is_tiered = False          # Create meta-iNat or tiered meta-iNat? Overrides random_assign.\n",
    "random_assign = False      # Split categories randomly over the representation and evaluation sets, \n",
    "                           # or use the splits from the paper? \n",
    "\n",
    "if not is_tiered and not random_assign:\n",
    "    assert catsize_min==50 and catsize_max==1000, 'The provided splits work only for category sizes between 50 and 1000.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Built annotation dictionaries\n"
     ]
    }
   ],
   "source": [
    "# Compile bounding box annotations for each image id\n",
    "\n",
    "with open(join(annopath,'train_2017_bboxes.json')) as f:\n",
    "    allinfo = json.load(f)\n",
    "annolist = allinfo['annotations']\n",
    "\n",
    "annodict = dict() # im_id to list of box_ids\n",
    "boxdict = dict() # box_id to box coords\n",
    "catdict = dict() # dict of numerical category codes / labels to corresponding list of image ids\n",
    "for d in annolist:\n",
    "    im = d['image_id']\n",
    "    boxid = d['id']\n",
    "    cat = d['category_id']\n",
    "    \n",
    "    # Add box_id to image entry\n",
    "    if im in annodict:\n",
    "        annodict[im].append(boxid)\n",
    "    else:\n",
    "        annodict[im] = [boxid]\n",
    "        \n",
    "    # Add mapping from box_id to box\n",
    "    boxdict[boxid] = d['bbox']\n",
    "    \n",
    "    # Add image to category set\n",
    "    if cat in catdict:\n",
    "        catdict[cat].add(im)\n",
    "    else:\n",
    "        catdict[cat] = set([im])\n",
    "    \n",
    "print(\"Built annotation dictionaries\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Built path dictionary\n"
     ]
    }
   ],
   "source": [
    "# assemble im_id -> filepath dictionary\n",
    "namelist = allinfo['images']\n",
    "keys = []\n",
    "vals = []\n",
    "for d in namelist:\n",
    "    keys.append(d['id'])\n",
    "    vals.append(join(inatpath,d['file_name']))\n",
    "pather = dict(zip(keys,vals))\n",
    "\n",
    "print(\"Built path dictionary\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Detected 1135 categories of the desired size\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAwDElEQVR4nO3deXxdVbn/8c+TeWyTNEPbtGlaOgAtQ0tBKDNFAVGLXlCuooADV/QKDvengF7H672oXCcUvagMgoAMKoggIDJThhRK6UBpS9M2nZK0TZukTZrh+f2xd8ohzdQmJzsn+b5fr/0656w9PStpz5O11t5rm7sjIiLSk6SoAxARkaFPyUJERHqlZCEiIr1SshARkV4pWYiISK+ULEREpFdKFiJxZmbXmNlvB/F86Wa23MzGDtY5D4SZfdvMbg/fl5jZCjNLjzou6ZmShcSVmX3UzCrMrMHMNpvZw2Z2Uh/3dTObGu8YB4KZLTCzxWa2y8xqzexxMysHcPf/dvdPD2I4lwFPu/uWMLYJZnZfGNdOM3vdzC452IObWaWZnTkQgbr7VuCJMGYZwpQsJG7M7MvAT4H/BkqAMuAGYEGEYfXKzFIOcPupwO+BrwCjgckE9Wwf+Oj65N+A22I+3wZsACYBY4BPAFsP9KAH+nM5AH8giFmGMnfXomXAF4IvzQbggh62OQ5YCNQBm4FfAGnhuqcBBxrD43wkLH8fsDjc53ngyJjjzQFeBeqBe4A/Av8Vs/4zwGpgO/AAMD5mnQOfB1YBa4FfAv/bKd6/Al/soh7nA4t7qOe3gdvD978I69OxtALfDteNB+4DasIYruj0s6oAdhF80f+4m3OVAXuAlJiyBuDoHuL7ALAs/Jk+CRwWs64S+BqwBGgG7iRIgnvC43413O748PdRB7wGnBZzjMnAU+Hv5bHwZ3B7zPoUYDcwKep/t1q6XyIPQMvwXICzwy/ClB62OSb8kkkByoEVsV/G4Rf41JjPc4Bq4F1AMnBx+GWWDqQB64ArgVTgQ8DejmQBnAHUhsdIB64n6KqJPddjQAGQGX45bwKSwvWF4RdaSRf1mAI0AT8BTgdyOq3flyw6lR8dJobZBK38RcA3w7pMAd4Czgq3XQh8PHyfAxzfzc/0XGBZp7J/AM8BFwJlndZNJ0jI7w5/bl8lSKgdSbuSIDlPBDJjys6MOUYpsA14b1iPd4efi2Ji/3H4cz8lTBq3d4pjCfCBqP/daul+UTeUxMsYoNbdW7vbwN0XufsL7t7q7pXA/wGn9nDMzwD/5+4vunubu99K8Nfu8byddH7u7i3u/ifgpZh9Pwbc5O6vuHszcDVwQse4Quh/3H27u+9x95eAncD8cN2FwJMe9LF3rsdbwGkEX5p3A7VmdouZ5XRXETMrAv4CfMHdXwWOJfhy/a677w2P+ZvwvAAtwFQzK3T3Bnd/oZtD5xF8Gce6AHgG+E9gbTi2cmy47iPA39z9MXdvAa4jSJbzYvb/ubtvcPc93ZzzIuAhd3/I3dvd/TGCVtB7zawsrNt/unuzuz9N0ELrrD6MXYYoJQuJl21AYU/93GY23cweNLMtZraLYGyjsIdjTgK+YmZ1HQvBX7zjw2Wju8fOjLkh5v14gpYHAO7eEMZY2s32ALcSfBESvt5GN8Kk92F3LwJOJvgL+utdbWtmqcC9wB3ufldM3cZ3qts1BGM9AJ8iaAW8YWYvm9n7ugllB5DbKbYd7n6Vu88Mj7cY+IuZGfv/XNrDn0NPP5fOJgEXdIr9JGBcePwd7t4Ys/26Lo6RS9CFJUOUkoXEy0KCrpnzetjmV8AbwDR3H0Xw5Wg9bL8B+L6758UsWe5+J8GYR2n4BdhhYsz7TQRfagCYWTZB62djzDadp2C+HVhgZkcBhxG0BHrl7i8DfwJmdbPJ9QR/SX+jU93Wdqpbrru/NzzmKnf/V6AY+AFwb1iHzpYAU7pL0u5eS9B6GE/Q5db552IEP7eefi6dP28AbusUe7a7X0vwe8nvFGtZ7M5hrFMJxjpkiFKykLhw950E/e+/NLPzzCzLzFLN7Bwz+2G4WS7BgG2DmR0KXN7pMFsJ+u47/Ab4rJm9ywLZZnaumeUSJKc24N/NLMXMFhCMO3S4A7jUzI4Or+n/b+DFsPuruzpUAS8TtCju664bxsxOMrPPmFlx+PlQgkHj/bqKzOzfCLraPhr+Fd/hJWCXmX3NzDLNLNnMZnV0F5nZRWZWFO5TF+7T1k3Mq2LrbmY/CI+VEv6sLgdWu/s2gm6zc81sftji+QpB197z3f1c2P/3cjvwfjM7K4w7w8xOM7MJ7r6OoEvqO2aWFl42/f5OxzsOqAy3laEq6kETLcN7IRgrqCAYRN0C/A2YF647haBl0UDQp/5d4NmYfT9L8JdpHfDhsOxsgi/wunDdPUBuuG4uQRdLQ1j+J4K+8tjjrSG4GupBYELMuncMpseUXxSuO72HOs4i6IffGp67kuCv/9Rw/bd5+2qoJwm+jGOviLomXDee4GqjLQTdSS8QDiQTfCFXh9svA87rIZ7PA7+K+Xw9QQJpIBhQf5B3XvH0QWA5wRjNU8DMmHWVxAxmh2ULgPXh7+A/wrJ3hftuD8/xN8LBdILE8kx4/q6uhvolMVd+aRmai4W/LJFhx8xeBH7t7jf34xinEHxRl/s7WwJDVthyehWY7+6bo46nJ2Fr7Clgtrs3RR2PdE/JQoYNMzsVWElwiezHgF8DUw72CzPslrkLeM3dvztggYokoHjdkSkShRkEffA5BN1N5/cjURxG0H32GnDpgEUokqDUshARkV7paigREenVsO2GKiws9PLy8qjDEBFJKIsWLar14ObSdxi2yaK8vJyKioqowxARSShm1uX9LuqGEhGRXilZiIhIr5QsRESkV3FLFmZ2k5lVm9nSmLICM3vMzFaFr/kx6642s9VmttLMzoopPyZ8DORqM/t5p4niRERkEMSzZXELwTw+sa4CHnf3acDj4WfM7HCCeftnhvvcYGbJ4T6/Ing+77Rw6XxMERGJs7glCw8ecrK9U/ECgmcEEL6eF1N+lwcPR1lL8KSu48xsHDDK3Rd6cPfg7+l5ymsREYmDwR6zKOmYfiF8LQ7LS3nnA1aqwrLS8H3n8i6Z2WVmVmFmFTU1NQMauIjISDZUBri7GofwHsq75O43uvtcd59bVLTfPSV9cuvzlfz1tU0Hta+IyHA12Mlia9i1RPhaHZZX8c6nmk0geIJXVfi+c3nc3PnSeiULEZFOBjtZPABcHL6/GLg/pvxCM0s3s8kEA9kvhV1V9WZ2fHgV1Cdi9omLURmp7GpqiecpREQSTtym+zCzO4HTgEIzqwK+BVwL3G1mnyJ40tYFAO6+zMzuJnhaVyvweXfveGTk5QRXVmUCD4dL3ORmpLBll57BIiISK27JwoOHy3dlfjfbfx/4fhflFXT/4PsBNyozlVXVDYN1OhGRhDBUBriHjPysNNZv382muj1RhyIiMmQoWXRy0fFlANy/WIPcIiIdlCw6mVKUQ2FOGuu2NUYdiojIkKFk0YUJ+Vls2LE76jBERIYMJYsuTMjPpGqHxixERDooWXShrCCLjTv20NTS1vvGIiIjgJJFF46dXEBru1NRuSPqUEREhgQliy4cUToagNeq6qINRERkiFCy6MKY7DRK8zJ5ZNmWqEMRERkSlCy6YGacPK2QTXWa9kNEBJQsulWYk872xmba27udEV1EZMRQsuhGUW467Y7utxARQcmiWydNKwRg4ZptEUciIhI9JYtuTCrIIjnJWFuraT9ERJQsupGSnMS04hweXLI56lBERCKnZNGDC+ZOZGPdHrbqYUgiMsIpWfRgSmE2ABv1bAsRGeGULHpQlJsOQE19c8SRiIhES8miBx3JQt1QIjLSKVn0oDg3ncKcNBavr4s6FBGRSClZ9MDMmFacS6WemiciI5ySRS/Gjs5g6y6NWYjIyKZk0YviUelU1zfhrjmiRGTkUrLoRX5WGi1tzu69emqeiIxcSha9yMtMBaBuT0vEkYiIREfJohd5WWkAPP1mTcSRiIhER8miFzPHjyI9JYlbn6+MOhQRkcgoWfRiYkEWnz31EFZurae+SV1RIjIyKVn0wZxJ+bjDkqqdUYciIhIJJYs+OHpiHgCvrNsRbSAiIhFRsuiD0ZmpTCvO4ZX1ShYiMjIpWfTRrNLRvLm1IeowREQioWTRR8G0H020t+tObhEZeSJJFmb2JTNbZmZLzexOM8swswIze8zMVoWv+THbX21mq81spZmdFUXM40dn0NruLNmoQW4RGXkGPVmYWSlwBTDX3WcBycCFwFXA4+4+DXg8/IyZHR6unwmcDdxgZsmDHfdZs8ZSmJPOVfctGexTi4hELqpuqBQg08xSgCxgE7AAuDVcfytwXvh+AXCXuze7+1pgNXDc4IYLxbkZXHpiOW9sqdfDkERkxBn0ZOHuG4HrgPXAZmCnuz8KlLj75nCbzUBxuEspsCHmEFVh2X7M7DIzqzCzipqagZ+e46SphQBc+/AbA35sEZGhLIpuqHyC1sJkYDyQbWYX9bRLF2VdjjK7+43uPtfd5xYVFfU/2E6OmpjHmYcV81pV3YAfW0RkKIuiG+pMYK2717h7C/AnYB6w1czGAYSv1eH2VcDEmP0nEHRbReLQsaOorG1ky051RYnIyBFFslgPHG9mWWZmwHxgBfAAcHG4zcXA/eH7B4ALzSzdzCYD04CXBjnmfc44rJh2R60LERlRUgb7hO7+opndC7wCtAKvAjcCOcDdZvYpgoRyQbj9MjO7G1gebv95d4/sSUSleZkAVNfrUasiMnIMerIAcPdvAd/qVNxM0MroavvvA9+Pd1x9MSY7jeQk44HFG7noXWUEjSMRkeFNd3AfoJTkJI6fUsDLlTvUuhCREUPJ4iBcfEI5ANW7lCxEZGRQsjgIxaMyALhn0YZethQRGR6ULA7CzPGjSEkyHngtsit4RUQGlZLFQUhNTuKK+dOo291Cc2tkF2aJiAwaJYuDND68hHapZqEVkRFAyeIgnTh1DACv67ncIjICKFkcpPysNAB2t6gbSkSGPyWLg5SekoQZ7NmrZCEiw5+SxUEyM7JSk9mtZCEiI4CSRT9kpqWwe29r1GGIiMSdkkU/ZKerZSEiI4OSRT+UjMqgsrYx6jBEROJOyaIf5pTls2zTLlrb2qMORUQkrpQs+mFKYTat7c6mOj01T0SGNyWLfpg0JguAym3qihKR4U3Joh/KC7MBeHV9XbSBiIjEmZJFPxTnpjOlKJt/rqyOOhQRkbhSsugHM+PYSQVs3LEn6lBEROJKyaKfphRlU9vQzKPLtkQdiohI3ChZ9NMnTiinrCCLHz2yMupQRETiRsminzLTkjl71lgqtzXS3u5RhyMiEhdKFgNgRkkuLW3Ojc+8FXUoIiJxoWQxAM6bXcrcSfnc+nwlTXq+hYgMQ0oWAyA5yfjc6YeweWcTd720PupwREQGnJLFADllWhHHTMrn+w+toLpe03+IyPCiZDFAUpKT+P4HZ9HS5jy5sibqcEREBpSSxQCaUZLLmOw0nltdG3UoIiIDSsliAJkZ7z68hIde30x9U0vU4YiIDBgliwF2zhHjaGlzXnhre9ShiIgMGCWLAfauyQUUZKfx0Oubow5FRGTA9JoszCzbzJLC99PN7ANmlhr/0BJTRmoyR0/M45lVNayuro86HBGRAdGXlsXTQIaZlQKPA5cCt8QzqET35XdPZ29rOz95bFXUoYiIDIi+JAtz993Ah4Dr3f2DwOH9OamZ5ZnZvWb2hpmtMLMTzKzAzB4zs1Xha37M9leb2WozW2lmZ/Xn3INhVulo5pYX8FatnqAnIsNDn5KFmZ0AfAz4W1iW0s/z/gz4u7sfChwFrACuAh5392kELZirwpMfDlwIzATOBm4ws+R+nj/uygqy2LB9N+6aXFBEEl9fksUXgauBP7v7MjObAjxxsCc0s1HAKcDvANx9r7vXAQuAW8PNbgXOC98vAO5y92Z3XwusBo472PMPlrKCLBqaW9nWuDfqUERE+q3XZOHuT7n7B4BfhJ/fcvcr+nHOKUANcLOZvWpmvzWzbKDE3TeH59gMFIfblwIbYvavCsv2Y2aXmVmFmVXU1ER7F/WRE0YD8IU7XqW5VZMLikhi68vVUCeY2XKCriLM7Cgzu6Ef50wB5gC/cvfZQCNhl1N3IXRR1mXfjrvf6O5z3X1uUVFRP0Lsv7nlBVz7oSNY+NY2fvO0pi4XkcTWl26onwJnAdsA3P01gm6kg1UFVLn7i+HnewmSx1YzGwcQvlbHbD8xZv8JwKZ+nH/QXHhcGYeOzeW6R9/U1OUiktD6dFOeu2/oVHTQ33zuvgXYYGYzwqL5wHLgAeDisOxi4P7w/QPAhWaWbmaTgWnASwd7/sF23uygx+y6R1ZqsFtEElZfrmraYGbzADezNOAKwi6pfvgC8IfweG8R3LuRBNxtZp8C1gMXAISD6ncTJJRW4PPunjB/pl928hQqaxv57bNr2da4l+suOIrkpK561kREhi7r7a9dMyskuNT1TILxg0eBK919W/zDO3hz5871ioqKqMMAoL3dueKuV3lwyWbu/ewJzC0viDokEZEumdkid5/bubwvV0PVuvvH3L3E3Yvd/aKhniiGmqQk42tnHwpAxbodEUcjInLgeu2GMrOfd1G8E6hw9/u7WCddKM3LZHJhNg8s3sRnTz0k6nBERA5IXwa4M4CjgVXhciRQAHzKzH4at8iGmaQkY8HR41m+eRePLtsSdTgiIgekL8liKnCGu1/v7tcTjF0cBnwQeE88gxtuPn3yFMaNzuCGJ9foyigRSSh9SRalQHbM52xgfHhFUnNcohqmctJTuHheOYs31PGHF9dHHY6ISJ/1JVn8EFhsZjeb2S3Aq8B14RQd/4hncMPRpSeWU5iTxjf+spSn3ox2ShIRkb7qy9VQvwPmAX8Jl5Pc/bfu3uju/y++4Q0/6SnJ3Hf5PDJSk7h3UVXU4YiI9ElfH6vaBGwGtgNTzaw/032MeJPGZPPB2RN46PXNPLe6NupwRER61ZeJBD9N8LS8R4DvhK/fjm9Yw99X3jOdifmZfPrWChbp3gsRGeL60rK4EjgWWOfupwOzCaYYl34ozEnnOwtmkZxknP/r59XCEJEhrS/JosndmwDMLN3d3wBm9LKP9MGp04v453+cSkluBh/77Yts2L476pBERLrUl2RRZWZ5BIPbj5nZ/STIFOGJoDg3g+s/OhuA6/+5KuJoRES61peroT7o7nXu/m3gPwkeh3penOMaUY4tL+BDs0t5cMlm2tp1s56IDD19uhrKzPLN7EignuBhRLPiGtUINGdSPrv3tlFd3xR1KCIi++nLRILfAy4heO5Ee1jswBnxC2vkKSvIAmBtTSPjRmdGHI2IyDv15eFHHwYOcfe98Q5mJJtdlkdaShKPLt/KvKmFUYcjIvIOfemGWgrkxTmOES83I5UzZhRzy/OVuipKRIacviSL/wFeNbNHzOyBjiXegY1EF88rB2D+j59i3bbGaIMREYnRl2RxK/AD4Frgf2MWGWAnHDKGH/7LkextbefdP3maHY3q+RORoaEvYxa17t7V0/IkDj587ERa251r/vw6T6+qYcHRpVGHJCLSp5bFIjP7HzM7wczmdCxxj2wEu2DuBDJSk/j7Uj1RT0SGhr60LGaHr8fHlOnS2ThKTU5i1vjRPLx0C/cv3qjWhYhErtdkEU4eKIPs2x+Yycd/9yJX3rWYJ1fW8MPzjyQ1ua8zyouIDKxuk4WZfbmnHd39xwMfjnSYVTqahVfP5xM3vcSfX93IebNLOXV6UdRhicgI1dOfqrm9LBJnGanJ3HzJsaQkGU+t1KzwIhKdblsW7v6dwQxEupadnsLxU8bwx5fXc9U5h5KWoq4oERl8+uZJAP96XBmNe9u47YV1uGtWWhEZfEoWCWD+YcWcOr2I7z24nEeW6XJaERl8ShYJICM1mZsuOZaygiy++9flrN+muaNEZHD1mizM7Bsx79PjG450JznJuPqcQ9m0s4m/L9scdTgiMsJ0myzM7KtmdgJwfkzxwviHJN0554hx5GWlsl6z0orIIOvppryVwAXAFDN7BlgBjDGzGe6+clCik/2UFWSxTt1QIjLIeuqG2gFcA6wGTgM6JhO8ysyej3Nc0o2JBVlqWYjIoOspWZwN/A04BPgxcBzQ6O6Xuvu8/p7YzJLN7FUzezD8XGBmj5nZqvA1P2bbq81stZmtNLOz+nvuRDa1KId123bz4lvbdBmtiAyabpOFu1/j7vOBSuB2gi6rIjN71sz+OgDnvpKga6vDVcDj7j4NeDz8jJkdDlwIzCRIYDeYWfIAnD8hnRg+cvUjN77Aadc9ydNv6s5uEYm/vlw6+4i7v+zuNwJV7n4ScGl/TmpmE4Bzgd/GFC8geNAS4et5MeV3uXuzu68l6BY7rj/nT2THTS7gn185la+dfSht7c4nbnqJn/1jlVoZIhJXvSYLd/9qzMdLwrLafp73p8BXgfaYshJ33xwefzNQHJaXAhtitqsKy/ZjZpeZWYWZVdTUDN+/uKcU5XD5aYdw/+dP5PQZRfzkH2/yu2fXRh2WiAxjB3RTnru/1t8Tmtn7gGp3X9TXXboKpasN3f1Gd5/r7nOLiob/DK1jctK58RNzKcpN59aFlbS0tfe+k4jIQYjiDu4TgQ+YWSVwF3CGmd0ObDWzcQDha3W4fRUwMWb/CcCmwQt3aEtNTuLaDx3Bhu17+OIfF1Pf1BJ1SCIyDA16snD3q919gruXEwxc/9PdLwIeAC4ON7sYuD98/wBwoZmlm9lkYBrw0iCHPaSdcWgx8w8t5m9LNjP7u4/x66fWaAxDRAZUXx6rOliuBe42s08B6wluCMTdl5nZ3cByoBX4vLu3RRfm0GNm3HDRHP65opqbn6/k2offIMngslMOiTo0ERkmbLj+BTp37lyvqKiIOoxB19TSxpzvPcbe1nZ+8C9H8i/HTIg6JBFJIGa2yN3ndi7XrLPDTEZqMrd96jiKc9P5yj2vsXhDXdQhicgwoGQxDB0zqYA7LzuetOQkzvvlc/zfU2uiDklEEpySxTA1aUw2910+j0ljsvifh9+geldT1CGJSAJTshjGjpgwmh9/+GgAvnT3Ytrbh+f4lIjEn5LFMHfMpHwuO2UKz63extfuWxJ1OCKSoJQsRoCrzzmUC4+dyD2Lqnh+TX9nahGRkUjJYgQws32X0H7hjldZU9MQcUQikmiULEaIY8sLuO/yedQ3tXL2T5/mO39dpru8RaTPlCxGkGMm5fPgFScxe2I+Nz9Xyef+8Aqrqxs08C0ivVKyGGGml+Ry52XHc+mJ5Ty2fCtn/vgpjvrOozzwmuZmFJHuDaW5oWSQJCcZ33r/TD56XBnPrq7ljhfX8+U/LmZ7QzOXnDg56vBEZAhSy2IEm1aSy6UnTubey+cxZ1I+3/7rcj722xdYtG5H1KGJyBCjZCGMzkzl9588jo/Mnchzq7dx4Y0LWV2tK6ZE5G1KFgIEExD+4Pwj+dPn5tHu8IFfPMs371/Kxro9UYcmIkOAkoW8w5yyfB664mTmlOXz+4Xr+LfbKmhsbo06LBGJmJKF7GfG2Fxu//S7+K/zZrF04y5Ou+5JHnp9M616xrfIiKVkId362LvKuPmSY2luaeNzf3iFd//kaZ5fU6ukITICKVlIt8yM0w8t5umvns7Xzj6UjTv28NHfvMjp//skL7y1LerwRGQQ6bGq0mfVu5q4Z1EVP3t8FXtb2znzsBJOm1HEew4voXhURtThicgA6O6xqkoWcsD27G3jR4+s5PYX1rE37JK6/l9n8/6jxkccmYj0l57BLQMmMy2Zb77/cJZ+5yxuvuRYDh2by5fvXsyTK6ujDk1E4kTJQg5aWkoSpx9azJ2fOZ7SvEwuufllvvvX5VGHJSJxoGQh/Zafncb9nz+Jwpx0fr+wksraxqhDEpEBpmQhA2J0Vio3XTKX5CTjtOue5Jo/v86uppaowxKRAaJkIQPmyAl53Hf5PM6eOZY7XlzPnO8+xvceXM66bWppiCQ6XQ0lcfHY8q385um3eKlyOwDzDy3m38+Yyuyy/IgjE5GedHc1lJ5nIXHx7sNLePfhJbyxZRe3v7COP768gcffqOaEKWM454ixvOfwsYwdrXszRBKFWhYyKHY07uWXT6zm3leqqNsdjGXc+snjOHV6UcSRiUgs3ZQnQ0J7u7PwrW188paXaW5t5xMnTOKa9x5GRmpy1KGJCLopT4aIpCTjxKmF/PM/TuPcI8bx+4XrOPI7j/LJW15mb6smKBQZqpQsJBKleZn88mNz+OVH5zCnLI9/vlHN2T97mqodu6MOTUS6oGQhkTr3yHHc+Znj+ca5h1FZ28gpP3yC3z27luHaPSqSqDRmIUPGsk07+eHfV/LUmzVkpSVz9syx/PsZU5lSlBN1aCIjxpAZszCziWb2hJmtMLNlZnZlWF5gZo+Z2arwNT9mn6vNbLWZrTSzswY7ZhkcM8eP5uZLjuVH5x/J1OIc/vTqRs7436f43B8WsXTjTrU2RCI06C0LMxsHjHP3V8wsF1gEnAdcAmx392vN7Cog392/ZmaHA3cCxwHjgX8A0929rafzqGWR+DZs3833/7aCvy/bAsD0khw+PHciM8bmMrkwmwn5WRFHKDL8DJmb8tx9M7A5fF9vZiuAUmABcFq42a3Ak8DXwvK73L0ZWGtmqwkSx8LBjVwG28SCLH798WN4c2s991Rs4I4X1/Nff1uxb/0VZ0zlkydNJi8rLcIoRUaGSMcszKwceBqYBax397yYdTvcPd/MfgG84O63h+W/Ax5293u7ON5lwGUAZWVlx6xbty7+lZBB09rWzrrtu3nijWpufq6SjXV7ACjMSWPB0aXMKh3FtOJcRmemMiE/EzOLOGKRxDNkWhYdzCwHuA/4orvv6uE/dlcrusxw7n4jcCME3VADEacMHSnJSRxSlMMhRTl8+uQpVFRu5+lVtTyzqobfPbv2HdumpyRx8rRCPnJsGXPK8hiTkx5R1CLDQyTJwsxSCRLFH9z9T2HxVjMb5+6bw3GNjseuVQETY3afAGwavGhlqJpbXsDc8gK+/O7p7Gjcy9ptjWzYvputu5qo3LabexdV8Y8V1aQlJ3HWrLFcOX8aU4t1ZZXIwYhigNsIxiS2u/sXY8p/BGyLGeAucPevmtlM4A7eHuB+HJimAW7pTfWuJlZsqee+RVU8uGQT7Q5zyvI4fUYxZ88ay9TiHHVViXQyZOaGMrOTgGeA14GO+R2uAV4E7gbKgPXABe6+Pdzn68AngVaCbquHezuPkoXE2rB9N7e/sI4Hl2zeN9aRm5HCMZPyuea9hzG9JDfiCEWGhiGTLAaLkoV057UNdbxcuZ1nV9fy5Moa8rNS+e8PHkFmWjInHDKG9BRNaigjl5KFSBeeX13Lx296ibb24P9BcW468w8rYdKYLGaMzeXkqYWkJGtWHBk5htzVUCJDwbyphTz7tdPZ1rCXN7fWc9sL67i7YsO+5FEyKp13TR7D7LI8Tp5WpAFyGbHUshDpZG9rOzv3tPDo8i3cv3gTyzftoqG5FYCJBZmcMaOYqSW5HFk6mqMm5kUbrMgAUzeUyEFqa3eWbdrJM6tq+curG1lb20hr2PI4onQ0J04t5KgJoxmXl8nkwmxGZ6ZGHLHIwVOyEBkgLW3tVNY28udXN/L3ZVt4q6Zx3zozOOvwsZxzxFhOm1HMqIwUXZ4rCUXJQiROauqbqdqxm1XVDVRUbufeRVWEDQ/GZKdx9MQ85k0tZFpxDmNHZzBN93fIEKZkITJImlraeOKNalZurWfZpl0sXLNt35hHhymF2cwsHc2Y7DSmFAVdV8eWFzA+LzOiqEUCuhpKZJBkpCZzzhHjOOeIcUAwYL5lZxOra+qp2rGHtbWNLNu4i2dX1dDQ3EpL29t/sI0fncHxU8YwPi+TCfmZnDi1kIkFmopdoqdkIRJnaSlJlI3JomzM/l/6rW3tVO3YQ21DM4+/Uc2r63fw0NLNNLW079umNC+Tk6YWUl6YTXFuOidOLaQwJ033f8igUrIQiVBKchLlhdmUF2Yzt7xgX3lbu1NRuZ3n1mzjudW13PdK1b4rsABGZ6Zy2SlT+OSJk8lM0x3nEn8asxBJAC1t7extbeeV9TtYvmkXDy3dwmsb6gBITjJOmVbIcZPHMP+wYs1zJf2iAW6RYcTdeerNGl5ZX8emuj08snQL9eEgek56CqV5mcwsHcWJhxRybHkBEwv0MCjpGyULkWHM3VlV3cDfl25h884mVmzexZKqun2X8I7KSOGoiXnMKcsnIzWZ1GRjekkus8vyyM3QTYTyNl0NJTKMmQVf/rFdUHtb23lp7XaWbKxj6cadvLR2O8+sqt1v3ymF2YzLy6A0L5OpxTlMLsyhZFQ6RbnpjButS3kloGQhMkylpSRx0rRCTppWCAStj+bW4CqrmvpmllTtZOFbtWzZ2czq6noWr6+jce87nylWmpfJrNJRlOZlMbU4h1GZKUwtziEjJZmygiySktS1NVIoWYiMEGZGRmpw5dTEgiwmFmRx7pHj9q1va3e27mpiVXUDzS1tLN24k1fW17FoXR3/WFG9bybeDqnJRmFOOtNLcplanENmajJTirI5onQ0kwuzdWnvMKNkISJAcFXV+LzMfXeRv2fm2H3rmlra2LKzic07m6iub2J7417WbdvNmpoG1tY28uzq2nckk6y0ZI6ZlM+s0tHkZaYytTiHklEZTCnKJi05SYkkASlZiEivMlKT990P0p2mljaWVO3k9Y07qajczmsb6rocI8lMTeawcblMLswhMy2JmeNHM70kl4LsNCb3cHyJlq6GEpG4aWppY+eeFt6qaWRNTQO7mlpYtbWB1zfuZM/eNrbsanpHiyQ9JYkzDi2mZFQGhTlpHD9lDAXZaaQkJeny30Giq6FEZNBlpCaTkZpMyagMTjhkzH7rm1raWLmlnpr6ZtbWNlKxbjsvrd1OU0vbfoPtozNTOXHqGCYXZlOUk874vExSU5KYXpJLbkYKo3QJcFwpWYhIZDJSk9/xtMHPMGXf+7dqGli6aRfuTvWuZp5bU8vTb9by0OtbujxWWUHw3PRpxTnkZqSSn5XKmYeXkJWWTGZqslol/aRuKBFJKO7OmpoGmlra2VS3h+r6ZtZta2Tpxl2s29bIpp1N++1TmJNGflYaJaMyGDs6AwOmFOWQk55M2Zhs8rNSmZCfRUF22uBXaIhRN5SIDAtmxtTi4ObDWaWj91vf1NKGO1Ss287yTbto3NvGmprgcuBV1cHVWzv3tOz3jBGAlCQL7iMJLwPOy0wLLgtOSyIzNbjHpOPWkuQko6wga8S0WJQsRGRY6biX5ORpRZw8rajLbdrbne2797JnbxurqxvY0xK8bt7ZxMa6Peza08LjK6ppaG7d7/6SWGnJSeRmBEkkLeXty4GLctIZl5cBwNhRGZSMCt6PykxlSnjFV1pKEnlZidOSUbIQkREnKSm4oRDo8eFSHfeXOLC2toH6prdbIzX1zWys28Omuj3U1DfT0hbcHd/S5lRU7qClrf0d08p3pTAnndTkt1sm2ekpHFKUTZIFLZystOAr2gwOKcohOz1IhIYxrSSHrHB6+oyU5LjfTa9kISLSjY77S4CDugekvd1ZXdPA3tZ23GF1TT27w6u8qnbsYVtD875t3WFNTQOVtbvZsXsvDy/teiC/K3lZqRSFyQ/gwStOIj1lYJ9zomQhIhInSUn2jskdj5iw/xhLV2Ln8QKob2plTU3Dvs+79rSwtrYRgKaWdlbXNNDW/vb2xsC3MpQsRESGmNh5vCBo4RTlpvewR/xpghYREemVkoWIiPRKyUJERHqlZCEiIr1SshARkV4pWYiISK+ULEREpFdKFiIi0qthO0W5mdUA6w5y90Jg/+dBJr7hWK/hWCdQvRLJcKvTJHffbwbGYZss+sPMKrqazz3RDcd6Dcc6geqVSIZjnbqibigREemVkoWIiPRKyaJrN0YdQJwMx3oNxzqB6pVIhmOd9qMxCxER6ZVaFiIi0islCxER6ZWSRQwzO9vMVprZajO7Kup4DoSZTTSzJ8xshZktM7Mrw/ICM3vMzFaFr/kx+1wd1nWlmZ0VXfQ9M7NkM3vVzB4MPw+HOuWZ2b1m9kb4OzthmNTrS+G/v6VmdqeZZSRivczsJjOrNrOlMWUHXA8zO8bMXg/X/dzM4vug7Hhydy3BuE0ysAaYAqQBrwGHRx3XAcQ/DpgTvs8F3gQOB34IXBWWXwX8IHx/eFjHdGByWPfkqOvRTd2+DNwBPBh+Hg51uhX4dPg+DchL9HoBpcBaIDP8fDdwSSLWCzgFmAMsjSk74HoALwEnAAY8DJwTdd0OdlHL4m3HAavd/S133wvcBSyIOKY+c/fN7v5K+L4eWEHwn3cBwRcT4et54fsFwF3u3uzua4HVBD+DIcXMJgDnAr+NKU70Oo0i+DL6HYC773X3OhK8XqEUINPMUoAsYBMJWC93fxrY3qn4gOphZuOAUe6+0IPM8fuYfRKOksXbSoENMZ+rwrKEY2blwGzgRaDE3TdDkFCA4nCzRKnvT4GvAu0xZYlepylADXBz2L32WzPLJsHr5e4bgeuA9cBmYKe7P0qC1yvGgdajNHzfuTwhKVm8rau+xIS7rtjMcoD7gC+6+66eNu2ibEjV18zeB1S7+6K+7tJF2ZCqUyiFoIvjV+4+G2gk6NboTkLUK+zDX0DQFTMeyDazi3rapYuyIVevPuiuHsOlfoCSRawqYGLM5wkETeiEYWapBIniD+7+p7B4a9gcJnytDssTob4nAh8ws0qCbsEzzOx2ErtOEMRZ5e4vhp/vJUgeiV6vM4G17l7j7i3An4B5JH69OhxoParC953LE5KSxdteBqaZ2WQzSwMuBB6IOKY+C6+y+B2wwt1/HLPqAeDi8P3FwP0x5ReaWbqZTQamEQzGDRnufrW7T3D3coLfxz/d/SISuE4A7r4F2GBmM8Ki+cByErxeBN1Px5tZVvjvcT7B2Fmi16vDAdUj7KqqN7Pjw5/HJ2L2STxRj7APpQV4L8FVRGuAr0cdzwHGfhJBE3cJsDhc3guMAR4HVoWvBTH7fD2s60qG+FUawGm8fTVUwtcJOBqoCH9ffwHyh0m9vgO8ASwFbiO4Qijh6gXcSTDu0kLQQvjUwdQDmBv+LNYAvyCcNSMRF033ISIivVI3lIiI9ErJQkREeqVkISIivVKyEBGRXilZiIhIr5QsJOGZ2dfDmU6XmNliM3vXAe5/iZmNP8B9ymNnJO20brqZPRTONLrCzO42s5JejvXRAzn/wTKz75rZmYNxLhleUqIOQKQ/zOwE4H0EM+42m1khwSyufd0/mWBm1KUMwN21ZpYB/A34srv/NSw7HSgCtnazWznwUYKZdePGzJLd/ZvxPIcMX2pZSKIbB9S6ezOAu9e6+yYAM5sfTtT3evh8gvSwvNLMvmlmzwL/SnDj1B/CVklm+AyCp8xskZk9EjPFwzFm9pqZLQQ+3008HwUWdiSKMKYn3H1p2IJ4xsxeCZd54SbXAieH5/+SBc/v+JGZvRy2lv4tPH+Smd0QtqIeDFsv5x9AXS8ws1ti9umunleY2fLw3HcN1C9KElzUdwVq0dKfBcghuFv9TeAG4NSwPINgJtDp4effE0yuCFAJfDXmGE8Cc8P3qcDzQFH4+SPATeH7JTHH/xExzzqIOdaPgSu7iTULyAjfTwMqwvenEd6dHn6+DPhG+D6d4E7vycD5wEMEf+SNBXaEZQdS11vCfXqq5yYgPXyfF/XvWMvQWNSykITm7g3AMQRfsDXAH83sEmAGwaR2b4ab3krwDIkOf+zmkDOAWcBjZrYY+AYwwcxGE3xxPhVud9tBhJsK/MbMXgfuIXhoTlfeA3wiPP+LBNNMTCOY0uUed2/3YH6pJ2JiPtC6dlnPcN0SgpbWRUDrgVZShieNWUjCc/c2gtbBk+EX8cUErY2eNHZTbsAydz/hHYVmefRteullwKndrPsSwbjFUQStg6YeYviCuz/SKYZze9i+J13Vtct6hs4lSDYfAP7TzGa6u5LGCKeWhSQ0M5thZtNiio4G1hFMZlduZlPD8o8DT9G1eoJH0UIwEVxROHCOmaWGX5Z1wE4zOync7mPdHOsOYF7sF7sFz3Y/AhgNbHb39jCe5C7OD/AIcLkFU853XF2VDTwL/Es4dlFC0H3FAda1Q5f1NLMkYKK7P0Hw0Kk8gq4+GeHUspBElwNcH/7l30rwSMvL3L3JzC4F7rHgEZ8vA7/u5hi3AL82sz0Ez0s+H/h52PWUQvC0vmXApcBNZrab4At9P+6+x4KHNv3UzH5KMGvpEuBKgjGV+8zsAoIupI6/+JcArWb2WhjLzwiukHolnNq6huBxnPcRTPu9lGCM5kWCp9EdSF074twbDnR3ruebwO1hmQE/CROljHCadVYkgZhZjrs3mNkYgmc/nBiOX4jElVoWIonlwbAVlQZ8T4lCBotaFiIi0isNcIuISK+ULEREpFdKFiIi0islCxER6ZWShYiI9Or/A7YobXwhck4JAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize the categories that meet the given size requirements\n",
    "\n",
    "catsizes = sort([(len(catdict[c])) for c in catdict if len(catdict[c]) >= catsize_min and len(catdict[c]) <= catsize_max])\n",
    "print('Detected %d categories of the desired size' % len(catsizes))\n",
    "pl.figure()\n",
    "pl.plot(catsizes[::-1])\n",
    "pl.title('Category Sizes (Sorted)')\n",
    "pl.ylabel('# Images')\n",
    "pl.xlabel('Sorted Categories')\n",
    "pl.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial size: 2854 categories\n",
      "Final size: 1135 categories\n",
      "\n",
      "Supercategory distributions:\n",
      "Insecta: 332\n",
      "Animalia: 22\n",
      "Aves: 478\n",
      "Reptilia: 124\n",
      "Amphibia: 50\n",
      "Mammalia: 67\n",
      "Actinopterygii: 7\n",
      "Arachnida: 22\n",
      "Mollusca: 33\n"
     ]
    }
   ],
   "source": [
    "# Pare down the category dictionary to the desired size\n",
    "\n",
    "print('Initial size: %d categories' % len(list(catdict.keys())))\n",
    "clist = list(catdict.keys())\n",
    "for c in clist:\n",
    "    if len(catdict[c]) < catsize_min or len(catdict[c]) > catsize_max:\n",
    "        catdict.pop(c)\n",
    "print('Final size: %d categories' % len(list(catdict.keys())))\n",
    "\n",
    "supercat = dict()\n",
    "for d in allinfo['categories']:\n",
    "    catid = d['id']\n",
    "    if catid in catdict:\n",
    "        sc = d['supercategory']\n",
    "        if sc in supercat:\n",
    "            supercat[sc].append(catid)\n",
    "        else:\n",
    "            supercat[sc] = [catid,]\n",
    "print('\\nSupercategory distributions:')\n",
    "for sc in supercat:\n",
    "    print(sc+':', len(supercat[sc]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Category splits assigned. \n",
      "Representation set has 908 of 1135 concepts, evaluation has 227.\n"
     ]
    }
   ],
   "source": [
    "# Create category splits\n",
    "\n",
    "if is_tiered:\n",
    "    traincatlist = (supercat['Animalia']+supercat['Aves']+supercat['Reptilia']+supercat['Amphibia']\n",
    "                   +supercat['Mammalia']+supercat['Actinopterygii']+supercat['Mollusca'])\n",
    "    testcatlist = supercat['Insecta']+supercat['Arachnida']\n",
    "elif random_assign:\n",
    "    catlist = list(catdict.keys())\n",
    "    shuffle(catlist)\n",
    "    testlen = len(catlist)//5\n",
    "    testcatlist = catlist[:testlen]\n",
    "    traincatlist = catlist[testlen:]\n",
    "else:\n",
    "    traincatlist = load(join('helpful_files','traincatlist.pth'))\n",
    "    testcatlist = load(join('helpful_files','testcatlist.pth'))\n",
    "\n",
    "print('Category splits assigned. \\nRepresentation set has %d of %d concepts, evaluation has %d.' \n",
    "      % (len(traincatlist), len(list(catdict.keys())), len(testcatlist)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 908/908 [00:28<00:00, 32.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Representation set complete!\n"
     ]
    }
   ],
   "source": [
    "# Build representation set\n",
    "\n",
    "boxdict_smaller = dict()\n",
    "catlist = traincatlist\n",
    "for c in tqdm(catlist):\n",
    "    # For each category:\n",
    "    if not os.path.exists(join(datapath,'train',str(c))):\n",
    "        os.makedirs(join(datapath,'train',str(c)))\n",
    "    ims = catdict[c]\n",
    "    for imkey in ims:\n",
    "        # For each image:\n",
    "        path = pather[imkey]\n",
    "        filename = path[path.rfind(os.path.sep)+1:path.rfind('.')]+'.png'\n",
    "        file = join(str(c),filename)\n",
    "        newpath = join(datapath,'train',file)\n",
    "        # Downsize the image to 84x84\n",
    "        with open(path, 'rb') as f:\n",
    "            p = Image.open(f)\n",
    "            w,h = p.size\n",
    "            p = p.convert('RGB')\n",
    "        p = p.resize((84, 84), Image.BILINEAR)\n",
    "        p.save(newpath)\n",
    "        # Downsize the bounding box annotations to 10x10\n",
    "        boxes = annodict[imkey]\n",
    "        boxdict_smaller[str(c)+'/'+filename] = []\n",
    "        for boxcode in boxes:\n",
    "            box = boxdict[boxcode]\n",
    "            xmin = box[0]\n",
    "            xmax = box[2]+xmin\n",
    "            ymin = box[1]\n",
    "            ymax = box[3]+ymin\n",
    "            boxdict_smaller[str(c)+'/'+filename].append([xmin*10/w, ymin*10/h, xmax*10/w, ymax*10/h])\n",
    "\n",
    "time.sleep(.1)\n",
    "print(\"Representation set complete!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 227/227 [00:06<00:00, 33.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation set complete!\n"
     ]
    }
   ],
   "source": [
    "# Build evaluation set\n",
    "\n",
    "catlist = testcatlist\n",
    "for c in tqdm(catlist):\n",
    "    # For each category:\n",
    "    if not os.path.exists(join(datapath,'test',str(c))):\n",
    "        os.makedirs(join(datapath,'test',str(c)))\n",
    "    ims = catdict[c]\n",
    "    for imkey in ims:\n",
    "        # For each image:\n",
    "        path = pather[imkey]\n",
    "        filename = path[path.rfind(os.path.sep)+1:path.rfind('.')]+'.png'\n",
    "        file = join(str(c),filename)\n",
    "        newpath = join(datapath,'test',file)\n",
    "        # Downsize the image to 84x84\n",
    "        with open(path, 'rb') as f:\n",
    "            p = Image.open(f)\n",
    "            w,h = p.size\n",
    "            p = p.convert('RGB')\n",
    "        p = p.resize((84, 84), Image.BILINEAR)\n",
    "        p.save(newpath)\n",
    "        # Downsize the bounding box annotations to 10x10\n",
    "        boxes = annodict[imkey]\n",
    "        boxdict_smaller[str(c)+'/'+filename] = []\n",
    "        for boxcode in boxes:\n",
    "            box = boxdict[boxcode]\n",
    "            xmin = box[0]\n",
    "            xmax = box[2]+xmin\n",
    "            ymin = box[1]\n",
    "            ymax = box[3]+ymin\n",
    "            boxdict_smaller[str(c)+'/'+filename].append([xmin*10/w, ymin*10/h, xmax*10/w, ymax*10/h])\n",
    "\n",
    "save(boxdict_smaller, join(datapath,'box_coords.pth'))\n",
    "\n",
    "time.sleep(.1)\n",
    "print(\"Evaluation set complete!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 227/227 [00:00<00:00, 529.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reference images compiled!\n"
     ]
    }
   ],
   "source": [
    "# Compile reference images within the evaluation set\n",
    "\n",
    "catlist = testcatlist\n",
    "for c in tqdm(catlist):\n",
    "    # For each category:\n",
    "    if not os.path.exists(join(datapath,'refr',str(c))):\n",
    "        os.makedirs(join(datapath,'refr',str(c)))\n",
    "    ims = list(catdict[c])\n",
    "    ims = ims[:len(ims)//5]\n",
    "    for imkey in ims:\n",
    "        # For each image:\n",
    "        path = pather[imkey]\n",
    "        oldpath = join(datapath,'test',str(c),path[path.rfind(os.path.sep)+1:path.rfind('.')]+'.png')\n",
    "        newpath = join(datapath,'refr',str(c),path[path.rfind(os.path.sep)+1:path.rfind('.')]+'.png')\n",
    "        # Create a softlink to the corresponding evaluation set image\n",
    "        os.symlink(oldpath, newpath)\n",
    "        \n",
    "time.sleep(.1)\n",
    "print(\"Reference images compiled!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 227/227 [00:01<00:00, 184.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query images compiled!\n"
     ]
    }
   ],
   "source": [
    "# Compile query images within the evaluation set\n",
    "\n",
    "catlist = testcatlist\n",
    "for c in tqdm(catlist):\n",
    "    # For each category:\n",
    "    if not os.path.exists(join(datapath,'query',str(c))):\n",
    "        os.makedirs(join(datapath,'query',str(c)))\n",
    "    ims = list(catdict[c])\n",
    "    ims = ims[len(ims)//5:]\n",
    "    for imkey in ims:\n",
    "        # For each image:\n",
    "        path = pather[imkey]\n",
    "        oldpath = join(datapath,'test',str(c),path[path.rfind(os.path.sep)+1:path.rfind('.')]+'.png')\n",
    "        newpath = join(datapath,'query',str(c),path[path.rfind(os.path.sep)+1:path.rfind('.')]+'.png')\n",
    "        # Create a softlink to the corresponding evaluation set image\n",
    "        os.symlink(oldpath, newpath)\n",
    "        \n",
    "time.sleep(.1)\n",
    "print(\"Query images compiled!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your md5 hash is: fcc785ea360b904674cac7184de3c4a7\n",
      "\n",
      "Congratulations! Your dataset appears to be a faithful reproduction.\n"
     ]
    }
   ],
   "source": [
    "# Use a recursive md5 checksum to verify that the constructed dataset reproduces the original\n",
    "\n",
    "# NOTE: this code only checks the assignments of photos to categories. \n",
    "# Thus, changing any file or folder names WILL cause the check to fail, even if the images themselves are still correct.\n",
    "\n",
    "import hashlib\n",
    "\n",
    "# Get hashes of relative locations for each photo in the dataset\n",
    "hashlist = []\n",
    "subdirs = ['train', 'test', 'refr', 'query']\n",
    "for subdir in subdirs:\n",
    "    for cat in os.listdir(join(datapath,subdir)):\n",
    "        hashlist = hashlist + [hashlib.md5(join(subdir,cat,file).encode()).hexdigest() \n",
    "                               for file in os.listdir(join(datapath,subdir,cat))]\n",
    "# Get a hash for the sorted list of hashes\n",
    "hashlist.sort()\n",
    "md5 = hashlib.md5(\"\".join(hashlist).encode()).hexdigest()\n",
    "# Compare\n",
    "print(\"Your md5 hash is:\", md5)\n",
    "print()\n",
    "if not random_assign:\n",
    "    if md5!=(\"fcc785ea360b904674cac7184de3c4a7\" if not is_tiered else \"ba43c60ae907ac9c06b961ea7110f534\"):\n",
    "        print(\"ALERT: Something went wrong. Your dataset does not match the original.\")\n",
    "    else:\n",
    "        print(\"Congratulations! Your dataset appears to be a faithful reproduction.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shut down the notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "Jupyter.notebook.session.delete();\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%javascript\n",
    "Jupyter.notebook.session.delete();"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}