train_jupyter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse, yaml\n",
    "from openvqa.models.model_loader import CfgLoader\n",
    "from utils1.exec1 import Execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, copy\n",
    "import sys\n",
    "from openvqa.datasets.dataset_loader import DatasetLoader\n",
    "\n",
    "class Execution:\n",
    "    def __init__(self, __C):\n",
    "        self.__C = __C\n",
    "\n",
    "        print('Loading dataset........')\n",
    "        self.dataset = DatasetLoader(__C).DataSet()\n",
    "\n",
    "        # If trigger the evaluation after every epoch\n",
    "        # Will create a new cfgs with RUN_MODE = 'val'\n",
    "        self.dataset_eval = None\n",
    "        if __C.EVAL_EVERY_EPOCH:\n",
    "            __C_eval = copy.deepcopy(__C)\n",
    "            setattr(__C_eval, 'RUN_MODE', 'val')\n",
    "\n",
    "            print('Loading validation set for per-epoch evaluation........')\n",
    "            self.dataset_eval = DatasetLoader(__C_eval).DataSet()\n",
    "\n",
    "\n",
    "    def run(self, run_mode):\n",
    "        \n",
    "        if run_mode == 'train':\n",
    "            if self.__C.RESUME is False:\n",
    "                self.empty_log(self.__C.VERSION)\n",
    "            train_engine(self.__C, self.dataset, self.dataset_eval)\n",
    "\n",
    "        elif run_mode == 'val':\n",
    "            test_engine(self.__C, self.dataset, validation=True)\n",
    "\n",
    "        elif run_mode == 'test':\n",
    "            test_engine(self.__C, self.dataset)\n",
    "\n",
    "        else:\n",
    "            exit(-1)\n",
    "\n",
    "\n",
    "    def empty_log(self, version):\n",
    "        print('Initializing log file........')\n",
    "        if (os.path.exists(self.__C.LOG_PATH + '/log_run_' + version + '.txt')):\n",
    "            os.remove(self.__C.LOG_PATH + '/log_run_' + version + '.txt')\n",
    "        print('Finished!')\n",
    "        print('')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_args():\n",
    "    '''\n",
    "    Parse input arguments\n",
    "    '''\n",
    "    parser = argparse.ArgumentParser(description='OpenVQA Args')\n",
    "\n",
    "    parser.add_argument('--RUN', dest='RUN_MODE',\n",
    "                      choices=['train', 'val', 'test'],\n",
    "                      help='{train, val, test}',\n",
    "                      default='train',\n",
    "                      type=str, required=False)\n",
    "\n",
    "    parser.add_argument('--MODEL', dest='MODEL',\n",
    "                      choices=[\n",
    "                           'mcan_small',\n",
    "                           'mcan_small_wa',\n",
    "                           'mcan_large',\n",
    "                           'ban_4',\n",
    "                           #Edits\n",
    "                           'ban_8_wa',\n",
    "                           'baseline_wa',\n",
    "                           #End of Edits\n",
    "                           'ban_8',\n",
    "                           'mfb',\n",
    "                           'mfb_wa',\n",
    "                           'mfh',\n",
    "                           'mfh_wa',\n",
    "                           'mem',\n",
    "                           'butd',\n",
    "                           'butd_wa',\n",
    "                           'baseline',\n",
    "                           'baseline_wa_no_fusion',\n",
    "                           'positional',\n",
    "                           'mcan_large_wa',\n",
    "                           'mcan_small_augmented',\n",
    "                           'mcan_small_without_a'\n",
    "                           ]\n",
    "                        ,\n",
    "                      help='{'\n",
    "                           'mcan_small,'\n",
    "                           'mcan_small_wa,'\n",
    "                           'mcan_large,'\n",
    "                            #Edits\n",
    "                           'ban_wa,'\n",
    "                           'baseline_wa,'\n",
    "                           #End of Edits\n",
    "                           'ban_4,'\n",
    "                           'ban_8,'\n",
    "                           'mfb,'\n",
    "                           'mfb_wa,'\n",
    "                           'mfh,'\n",
    "                           'mfh_wa,'\n",
    "                           'butd,'\n",
    "                           'butd_wa,'\n",
    "                           'baseline,'\n",
    "                           'baseline_wa_no_fusion,'\n",
    "                           'positional,'\n",
    "                           '}'\n",
    "                        ,\n",
    "                      type=str, required=True)\n",
    "\n",
    "    parser.add_argument('--DATASET', dest='DATASET',\n",
    "                      choices=['vqa', 'gqa', 'clevr'],\n",
    "                      help='{'\n",
    "                           'vqa,'\n",
    "                           'gqa,'\n",
    "                           'clevr,'\n",
    "                           '}'\n",
    "                        ,\n",
    "                      default='vqa',  \n",
    "                      type=str, required=False)\n",
    "\n",
    "    parser.add_argument('--SPLIT', dest='TRAIN_SPLIT',\n",
    "                      choices=['train', 'train+val', 'train+val+vg'],\n",
    "                      help=\"set training split, \"\n",
    "                           \"vqa: {'train', 'train+val', 'train+val+vg'}\"\n",
    "                           \"gqa: {'train', 'train+val'}\"\n",
    "                           \"clevr: {'train', 'train+val'}\"\n",
    "                        ,\n",
    "                        default='train', required=False,\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--EVAL_EE', dest='EVAL_EVERY_EPOCH',\n",
    "                      choices=['True', 'False'],\n",
    "                      help='True: evaluate the val split when an epoch finished,'\n",
    "                           'False: do not evaluate on local',\n",
    "                           default='True',\n",
    "                           required=False,\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--SAVE_PRED', dest='TEST_SAVE_PRED',\n",
    "                      choices=['True', 'False'],\n",
    "                      help='True: save the prediction vectors,'\n",
    "                           'False: do not save the prediction vectors',\n",
    "                      default='True',\n",
    "                      required=False,\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--BS', dest='BATCH_SIZE',\n",
    "                      help='batch size in training',\n",
    "                      type=int)\n",
    "\n",
    "    parser.add_argument('--GPU', dest='GPU',\n",
    "                      help=\"gpu choose, eg.'0, 1, 2, ...'\",\n",
    "                      default='0, 1',\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--SEED', dest='SEED',\n",
    "                      help='fix random seed',\n",
    "                      type=int)\n",
    "\n",
    "    parser.add_argument('--VERSION', dest='VERSION',\n",
    "                      help='Enter descriptive name here (eg baseline_wa_gru), will be used for WANDB and for version',\n",
    "                      required=True,\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--RESUME', dest='RESUME',\n",
    "                      choices=['True', 'False'],\n",
    "                      help='True: use checkpoint to resume training,'\n",
    "                           'False: start training with random init',\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--CKPT_V', dest='CKPT_VERSION',\n",
    "                      help='checkpoint version',\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--CKPT_E', dest='CKPT_EPOCH',\n",
    "                      help='checkpoint epoch',\n",
    "                      type=int)\n",
    "\n",
    "    parser.add_argument('--CKPT_PATH', dest='CKPT_PATH',\n",
    "                      help='load checkpoint path, we '\n",
    "                           'recommend that you use '\n",
    "                           'CKPT_VERSION and CKPT_EPOCH '\n",
    "                           'instead, it will override'\n",
    "                           'CKPT_VERSION and CKPT_EPOCH',\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--ACCU', dest='GRAD_ACCU_STEPS',\n",
    "                      help='split batch to reduce gpu memory usage',\n",
    "                      type=int)\n",
    "\n",
    "    parser.add_argument('--NW', dest='NUM_WORKERS',\n",
    "                      help='multithreaded loading to accelerate IO',\n",
    "                      type=int)\n",
    "\n",
    "    parser.add_argument('--PINM', dest='PIN_MEM',\n",
    "                      choices=['True', 'False'],\n",
    "                      help='True: use pin memory, False: not use pin memory',\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--VERB', dest='VERBOSE',\n",
    "                      choices=['True', 'False'],\n",
    "                      help='True: verbose print, False: simple print',\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--USE_NEW_QUESTION', dest='USE_NEW_QUESTION',\n",
    "                      choices=['True', 'False'],\n",
    "                      help='whether to use new question while testing',\n",
    "                      default='False',\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--NEW_QUESTION', dest='NEW_QUESTION',\n",
    "                      help='the new question to be asked while testing',\n",
    "                      type=str)\n",
    "\n",
    "    parser.add_argument('--IMAGE_ID', dest='IMAGE_ID',\n",
    "                      help='image id on which the questions to be asked',\n",
    "                      type=str)\n",
    "    \n",
    "    ######################################################\n",
    "    #########  CHANGE MODEL AND VERSION HERE #############\n",
    "    ######################################################\n",
    "    args = parser.parse_args(args=['--MODEL', 'mfb_wa', '--VERSION', 'fixed_decoder', '--GPU', '0', '--DATASET', 'vqa', '--CKPT_V', 'mfb_ans_img_only', '--CKPT_E', '13'])\n",
    "    return args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Namespace(BATCH_SIZE=None, CKPT_EPOCH=13, CKPT_PATH=None, CKPT_VERSION='mfb_ans_img_only', DATASET='vqa', EVAL_EVERY_EPOCH='True', GPU='0', GRAD_ACCU_STEPS=None, IMAGE_ID=None, MODEL='mfb_wa', NEW_QUESTION=None, NUM_WORKERS=None, PIN_MEM=None, RESUME=None, RUN_MODE='train', SEED=None, TEST_SAVE_PRED='True', TRAIN_SPLIT='train', USE_NEW_QUESTION='False', VERBOSE=None, VERSION='fixed_decoder')\n"
     ]
    }
   ],
   "source": [
    "args = parse_args()\n",
    "print(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking dataset ........\n",
      "Finished!\n",
      "\n",
      "Hyper Parameters:\n",
      "{ ALPHA             }->1\n",
      "{ ANS_STDDEV        }->0.1\n",
      "{ AUGMENTED_ANSWER  }->True\n",
      "{ BATCH_SIZE        }->64\n",
      "{ BETA              }->30.0\n",
      "{ CACHE_PATH        }->./results/cache\n",
      "{ CAP_DIST          }->0.3\n",
      "{ CKPTS_PATH        }->./ckpts\n",
      "{ CKPT_EPOCH        }->13\n",
      "{ CKPT_PATH         }->None\n",
      "{ CKPT_VERSION      }->mfb_ans_img_only\n",
      "{ DATASET           }->vqa\n",
      "{ DATA_PATH         }->{'vqa': './data/vqa', 'gqa': './data/gqa', 'clevr': './data/clevr'}\n",
      "{ DATA_ROOT         }->./data\n",
      "{ DEVICES           }->[0]\n",
      "{ DROPOUT_R         }->0.1\n",
      "{ EVAL_BATCH_SIZE   }->32\n",
      "{ EVAL_EVERY_EPOCH  }->True\n",
      "{ FEATS_PATH        }->{'vqa': {'train': './data/vqa/feats/train2014', 'val': './data/vqa/feats/val2014', 'test': './data/vqa/feats/test2015'}, 'gqa': {'default-frcn': './data/gqa/feats/gqa-frcn', 'default-grid': './data/gqa/feats/gqa-grid'}, 'clevr': {'train': './data/clevr/feats/train', 'val': './data/clevr/feats/val', 'test': './data/clevr/feats/test'}}\n",
      "{ FEAT_SIZE         }->{'vqa': {'FRCN_FEAT_SIZE': (100, 2048), 'BBOX_FEAT_SIZE': (100, 5)}, 'gqa': {'FRCN_FEAT_SIZE': (100, 2048), 'GRID_FEAT_SIZE': (49, 2048), 'BBOX_FEAT_SIZE': (100, 5)}, 'clevr': {'GRID_FEAT_SIZE': (196, 1024)}}\n",
      "{ GPU               }->0\n",
      "{ GRAD_ACCU_STEPS   }->1\n",
      "{ GRAD_NORM_CLIP    }->-1\n",
      "{ HIDDEN_SIZE       }->512\n",
      "{ HIGH_ORDER        }->False\n",
      "{ I_GLIMPSES        }->2\n",
      "{ LOAD_PRETRAINED   }->True\n",
      "{ LOG_PATH          }->./results/log\n",
      "{ LOSS_FUNC         }->kld\n",
      "{ LOSS_FUNC_NAME_DICT }->{'ce': 'CrossEntropyLoss', 'bce': 'BCEWithLogitsLoss', 'kld': 'KLDivLoss', 'mse': 'MSELoss'}\n",
      "{ LOSS_FUNC_NONLINEAR }->{'ce': [None, 'flat'], 'bce': [None, None], 'kld': ['log_softmax', None], 'mse': [None, None]}\n",
      "{ LOSS_REDUCTION    }->sum\n",
      "{ LR_BASE           }->0.0007\n",
      "{ LR_DECAY_LIST     }->[6, 12]\n",
      "{ LR_DECAY_R        }->0.5\n",
      "{ LSTM_LAYERS       }->1\n",
      "{ LSTM_NUM_DIRECTIONS }->1\n",
      "{ LSTM_OUT_SIZE     }->1024\n",
      "{ MAX_EPOCH         }->22\n",
      "{ MFB_K             }->5\n",
      "{ MFB_O             }->1000\n",
      "{ MODEL             }->mfb_wa\n",
      "{ MODEL_USE         }->mfb\n",
      "{ NUM_WORKERS       }->8\n",
      "{ N_GPU             }->1\n",
      "{ OPT               }->Adam\n",
      "{ OPT_PARAMS        }->{'betas': (0.9, 0.99), 'eps': 1e-09, 'weight_decay': 0, 'amsgrad': False}\n",
      "{ PIN_MEM           }->True\n",
      "{ PRED_PATH         }->./results/pred\n",
      "{ PROJ_STDDEV       }->0.1\n",
      "{ Q_GLIMPSES        }->2\n",
      "{ RAW_PATH          }->{'vqa': {'train': './data/vqa/raw/v2_OpenEnded_mscoco_train2014_questions.json', 'train-anno': './data/vqa/raw/v2_mscoco_train2014_annotations.json', 'val': './data/vqa/raw/v2_OpenEnded_mscoco_val2014_questions.json', 'val-anno': './data/vqa/raw/v2_mscoco_val2014_annotations.json', 'vg': './data/vqa/raw/VG_questions.json', 'vg-anno': './data/vqa/raw/VG_annotations.json', 'test': './data/vqa/raw/v2_OpenEnded_mscoco_test2015_questions.json'}, 'gqa': {'train': './data/gqa/raw/questions1.2/train_balanced_questions.json', 'val': './data/gqa/raw/questions1.2/val_balanced_questions.json', 'testdev': './data/gqa/raw/questions1.2/testdev_balanced_questions.json', 'test': './data/gqa/raw/questions1.2/submission_all_questions.json', 'val_all': './data/gqa/raw/questions1.2/val_all_questions.json', 'testdev_all': './data/gqa/raw/questions1.2/testdev_all_questions.json', 'train_choices': './data/gqa/raw/eval/train_choices', 'val_choices': './data/gqa/raw/eval/val_choices.json'}, 'clevr': {'train': './data/clevr/raw/questions/CLEVR_train_questions.json', 'val': './data/clevr/raw/questions/CLEVR_val_questions.json', 'test': './data/clevr/raw/questions/CLEVR_test_questions.json'}}\n",
      "{ RESULT_PATH       }->./results/result_test\n",
      "{ RESUME            }->False\n",
      "{ RUN_MODE          }->train\n",
      "{ SAVED_PATH        }->./saved\n",
      "{ SEED              }->5964730\n",
      "{ SPLIT             }->{'train': 'train', 'val': 'val', 'test': 'test'}\n",
      "{ SPLITS            }->{'vqa': {'train': 'train', 'val': 'val', 'test': 'test'}, 'gqa': {'train': '', 'val': 'testdev', 'test': 'test'}, 'clevr': {'train': '', 'val': 'val', 'test': 'test'}}\n",
      "{ SUB_BATCH_SIZE    }->64\n",
      "{ TASK_LOSS_CHECK   }->{'vqa': ['bce', 'kld'], 'gqa': ['ce'], 'clevr': ['ce']}\n",
      "{ TEST_SAVE_PRED    }->False\n",
      "{ TRAIN_SPLIT       }->train\n",
      "{ USE_GLOVE         }->True\n",
      "{ USE_NEW_QUESTION  }->False\n",
      "{ VERBOSE           }->True\n",
      "{ VERSION           }->fixed_decoder\n",
      "{ WARMUP_EPOCH      }->3\n",
      "{ WITH_ANSWER       }->True\n",
      "{ WITH_FUSION_LOSS  }->True\n",
      "{ WORD_EMBED_SIZE   }->300\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anaconda2/envs/frost/lib/python3.6/site-packages/ipykernel_launcher.py:5: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "cfg_file = \"configs/{}/{}.yml\".format(args.DATASET, args.MODEL)\n",
    "with open(cfg_file, 'r') as f:\n",
    "\n",
    "    # Loads the yaml file\n",
    "    yaml_dict = yaml.load(f)\n",
    "\n",
    "# Loads the model_cfgs + base_cfgs\n",
    "__C = CfgLoader(yaml_dict['MODEL_USE']).load()\n",
    "\n",
    "# Loads the command line cfgs\n",
    "args = __C.str_to_bool(args)\n",
    "args_dict = __C.parse_to_dict(args)\n",
    "\n",
    "# {**dict1, **dict2} creates a new dictionary by merging dict1 and dict2, using dict2 for key clashes\n",
    "args_dict = {**yaml_dict, **args_dict}\n",
    "__C.add_args(args_dict)\n",
    "__C.proc()\n",
    "\n",
    "# FINAL PREFERENCE OF CFGS:\n",
    "# COMMAND LINE > YAML FILE > MODEL CFGS > BASE CFGS\n",
    "\n",
    "print('Hyper Parameters:')\n",
    "print(__C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset........\n",
      "Loading all questions (for statistics)\n",
      "Loading all image features\n",
      "Loading split questions and answers\n",
      " ========== Dataset size: 443757\n",
      "\n",
      "Tokenising questions\n",
      " ========== Question token vocab size: 20573\n",
      "Tokenising answers\n",
      " ========== Answer token vocab size:  16596\n",
      " ========== Answer token vocab size (occur more than 8 times): 3129\n",
      "Finished!\n",
      "\n",
      "Loading validation set for per-epoch evaluation........\n",
      "Loading all questions (for statistics)\n",
      "Loading all image features\n",
      "Loading split questions and answers\n",
      " ========== Dataset size: 214354\n",
      "\n",
      "Tokenising questions\n",
      " ========== Question token vocab size: 20573\n",
      "Tokenising answers\n",
      " ========== Answer token vocab size:  16596\n",
      " ========== Answer token vocab size (occur more than 8 times): 3129\n",
      "Finished!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "execution = Execution(__C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --------------------------------------------------------\n",
    "# OpenVQA\n",
    "# Written by Yuhao Cui https://github.com/cuiyuhao1996\n",
    "# --------------------------------------------------------\n",
    "\n",
    "import os, torch, datetime, shutil, time\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.utils.data as Data\n",
    "import wandb\n",
    "from openvqa.models.model_loader import ModelLoader\n",
    "from openvqa.utils.optim import get_optim, adjust_lr\n",
    "from utils1.test_engine import test_engine, ckpt_proc\n",
    "from vis import plotter, vis_func\n",
    "from multiprocessing import Pool\n",
    "import multiprocessing\n",
    "import sys\n",
    "\n",
    "def train_engine(__C, dataset, dataset_eval=None):\n",
    "\n",
    "    data_size = dataset.data_size\n",
    "    token_size = dataset.token_size\n",
    "    ans_size = dataset.ans_size\n",
    "    pretrained_emb = dataset.pretrained_emb\n",
    "\n",
    "    #Edits\n",
    "    pretrained_emb_ans = dataset.pretrained_emb_ans\n",
    "    token_size_ans = dataset.token_size_ans #End of Edits\n",
    "\n",
    "    print(\"Model being used is {}\".format(__C.MODEL_USE))\n",
    "\n",
    "    net = ModelLoader(__C).Net(\n",
    "        __C,\n",
    "        pretrained_emb,\n",
    "        token_size,\n",
    "        ans_size,\n",
    "        pretrained_emb_ans,\n",
    "        token_size_ans\n",
    "    )\n",
    "\n",
    "    net.cuda()\n",
    "    net.train()\n",
    "\n",
    "    if __C.N_GPU > 1:\n",
    "        net = nn.DataParallel(net, device_ids=__C.DEVICES)\n",
    "\n",
    "    # Define Loss Function\n",
    "    loss_fn = eval('torch.nn.' + __C.LOSS_FUNC_NAME_DICT[__C.LOSS_FUNC] + \"(reduction='\" + __C.LOSS_REDUCTION + \"').cuda()\")\n",
    "\n",
    "\n",
    "    # creating a folder for saving the numpy visualization arrays\n",
    "    if (__C.WITH_ANSWER and ((__C.VERSION) not in os.listdir(__C.SAVED_PATH))):\n",
    "        os.mkdir(__C.SAVED_PATH + '/' + __C.VERSION)\n",
    "\n",
    "    ###############################################################\n",
    "    ######## Load the pretrained ans+img only model ###############\n",
    "    ###############################################################\n",
    "\n",
    "    if __C.LOAD_PRETRAINED:\n",
    "        print(\"using the pretrained model for ans+img encoder and decoder both parts\")\n",
    "\n",
    "        path = __C.CKPTS_PATH + \\\n",
    "               '/ckpt_' + __C.CKPT_VERSION + \\\n",
    "               '/epoch' + str(__C.CKPT_EPOCH) + '.pkl'\n",
    "        \n",
    "        print('Loading ckpt from {}'.format(path))\n",
    "        ckpt = torch.load(path)\n",
    "        print('Finish!')\n",
    "\n",
    "        pretrained_state_dict = torch.load(path)['state_dict']\n",
    "        \n",
    "        net_state_dict = net.state_dict()\n",
    "\n",
    "        #print(\"filtering keys from pretrained state dict\")\n",
    "        #pretrained_state_dict_updated = {k: v for k, v in pretrained_state_dict.items() if k in net_state_dict} \n",
    "    \n",
    "        print(\"updating keys in net_state_dict form pretrained state dict\")\n",
    "        net_state_dict.update(pretrained_state_dict)\n",
    "\n",
    "        print(\"loading this state dict in the net\")\n",
    "        '''\n",
    "        if __C.N_GPU > 1:\n",
    "            net.load_state_dict(ckpt_proc(net_state_dict))\n",
    "        else:\n",
    "        '''\n",
    "        net.load_state_dict(net_state_dict)\n",
    "        print(\"loaded net state dict succesfully\")\n",
    "        \n",
    "\n",
    "    # Load checkpoint if resume training\n",
    "    if __C.RESUME:\n",
    "        print(' ========== Resume training')\n",
    "\n",
    "        if __C.CKPT_PATH is not None:\n",
    "            print('Warning: Now using CKPT_PATH args, '\n",
    "                  'CKPT_VERSION and CKPT_EPOCH will not work')\n",
    "            path = __C.CKPT_PATH\n",
    "        else:\n",
    "            path = __C.CKPTS_PATH + \\\n",
    "                   '/ckpt_' + __C.CKPT_VERSION + \\\n",
    "                   '/epoch' + str(__C.CKPT_EPOCH) + '.pkl'\n",
    "\n",
    "        # Load the network parameters\n",
    "        print('Loading ckpt from {}'.format(path))\n",
    "        ckpt = torch.load(path)\n",
    "        print('Finish!')\n",
    "\n",
    "        if __C.N_GPU > 1:\n",
    "            net.load_state_dict(ckpt_proc(ckpt['state_dict']))\n",
    "        else:\n",
    "            net.load_state_dict(ckpt['state_dict'])\n",
    "        start_epoch = ckpt['epoch']\n",
    "\n",
    "        # Load the optimizer paramters\n",
    "        optim = get_optim(__C, net, data_size, ckpt['lr_base'])\n",
    "        optim._step = int(data_size / __C.BATCH_SIZE * start_epoch)\n",
    "        optim.optimizer.load_state_dict(ckpt['optimizer'])\n",
    "        \n",
    "        if ('ckpt_' + __C.VERSION) not in os.listdir(__C.CKPTS_PATH):\n",
    "            os.mkdir(__C.CKPTS_PATH + '/ckpt_' + __C.VERSION)\n",
    "\n",
    "    else:\n",
    "        if ('ckpt_' + __C.VERSION) not in os.listdir(__C.CKPTS_PATH):\n",
    "            #shutil.rmtree(__C.CKPTS_PATH + '/ckpt_' + __C.VERSION)\n",
    "            os.mkdir(__C.CKPTS_PATH + '/ckpt_' + __C.VERSION)\n",
    "\n",
    "        optim = get_optim(__C, net, data_size)\n",
    "        start_epoch = 0\n",
    "\n",
    "    loss_sum = 0\n",
    "    named_params = list(net.named_parameters())\n",
    "    grad_norm = np.zeros(len(named_params))\n",
    "\n",
    "    # Define multi-thread dataloader\n",
    "    # if __C.SHUFFLE_MODE in ['external']:\n",
    "    #     dataloader = Data.DataLoader(\n",
    "    #         dataset,\n",
    "    #         batch_size=__C.BATCH_SIZE,\n",
    "    #         shuffle=False,\n",
    "    #         num_workers=__C.NUM_WORKERS,\n",
    "    #         pin_memory=__C.PIN_MEM,\n",
    "    #         drop_last=True\n",
    "    #     )\n",
    "    # else:\n",
    "    dataloader = Data.DataLoader(\n",
    "        dataset,\n",
    "        batch_size=__C.BATCH_SIZE,\n",
    "        shuffle=True,\n",
    "        num_workers=__C.NUM_WORKERS,\n",
    "        pin_memory=__C.PIN_MEM,\n",
    "        drop_last=True\n",
    "    )\n",
    "\n",
    "    logfile = open(\n",
    "        __C.LOG_PATH +\n",
    "        '/log_run_' + __C.VERSION + '.txt',\n",
    "        'a+'\n",
    "    )\n",
    "    logfile.write(str(__C))\n",
    "    logfile.close()\n",
    "\n",
    "    # For dry runs\n",
    "    # os.environ['WANDB_MODE'] = 'dryrun' \n",
    "\n",
    "    # initializing the wandb project\n",
    "    # TODO to change the name of project later, once the proper coding starts\n",
    "    wandb.init(project=\"openvqa\", name=__C.VERSION, config=__C)\n",
    "\n",
    "    # obtain histogram of each gradients in network as it trains\n",
    "    wandb.watch(net, log=\"all\")\n",
    "\n",
    "    wandb.save(\"./openvqa/models/\" + str(__C.MODEL_USE) + \"/net.py\")\n",
    "    wandb.save(\"./utils1/train_engine.py\")\n",
    "\n",
    "    # Training script\n",
    "    for epoch in range(start_epoch, __C.MAX_EPOCH):\n",
    "\n",
    "        # Save log to file\n",
    "        logfile = open(\n",
    "            __C.LOG_PATH +\n",
    "            '/log_run_' + __C.VERSION + '.txt',\n",
    "            'a+'\n",
    "        )\n",
    "        logfile.write(\n",
    "            '=====================================\\nnowTime: ' +\n",
    "            datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') +\n",
    "            '\\n'\n",
    "        )\n",
    "        logfile.close()\n",
    "\n",
    "        # Learning Rate Decay\n",
    "        if epoch in __C.LR_DECAY_LIST:\n",
    "            adjust_lr(optim, __C.LR_DECAY_R)\n",
    "\n",
    "        # Externally shuffle data list\n",
    "        # if __C.SHUFFLE_MODE == 'external':\n",
    "        #     dataset.shuffle_list(dataset.ans_list)\n",
    "\n",
    "        time_start = time.time()\n",
    "        # Iteration\n",
    "        for step, (\n",
    "                frcn_feat_iter,\n",
    "                grid_feat_iter,\n",
    "                bbox_feat_iter,\n",
    "                ques_ix_iter,\n",
    "\n",
    "                #Edits\n",
    "                ans_ix_iter,\n",
    "                #End of Edits\n",
    "\n",
    "                ans_iter,\n",
    "                ques_type\n",
    "\n",
    "        ) in enumerate(dataloader):\n",
    "\n",
    "            optim.zero_grad()\n",
    "\n",
    "            frcn_feat_iter = frcn_feat_iter.cuda()\n",
    "            grid_feat_iter = grid_feat_iter.cuda()\n",
    "            bbox_feat_iter = bbox_feat_iter.cuda()\n",
    "            ques_ix_iter = ques_ix_iter.cuda()\n",
    "            #Edits\n",
    "            ans_ix_iter = ans_ix_iter.cuda()\n",
    "            #End of Edits\n",
    "            ans_iter = ans_iter.cuda()\n",
    "\n",
    "            loss_tmp = 0\n",
    "\n",
    "            loss_img_ques_tmp = 0\n",
    "            loss_ans_tmp = 0\n",
    "            loss_interp_tmp = 0\n",
    "            loss_fusion_tmp = 0\n",
    "\n",
    "            for accu_step in range(__C.GRAD_ACCU_STEPS):\n",
    "                loss_tmp = 0\n",
    "                loss_img_ques_tmp = 0\n",
    "                loss_ans_tmp = 0\n",
    "                loss_interp_tmp = 0\n",
    "                loss_fusion_tmp = 0\n",
    "\n",
    "                sub_frcn_feat_iter = \\\n",
    "                    frcn_feat_iter[accu_step * __C.SUB_BATCH_SIZE:\n",
    "                                  (accu_step + 1) * __C.SUB_BATCH_SIZE]\n",
    "                sub_grid_feat_iter = \\\n",
    "                    grid_feat_iter[accu_step * __C.SUB_BATCH_SIZE:\n",
    "                                  (accu_step + 1) * __C.SUB_BATCH_SIZE]\n",
    "                sub_bbox_feat_iter = \\\n",
    "                    bbox_feat_iter[accu_step * __C.SUB_BATCH_SIZE:\n",
    "                                  (accu_step + 1) * __C.SUB_BATCH_SIZE]\n",
    "                sub_ques_ix_iter = \\\n",
    "                    ques_ix_iter[accu_step * __C.SUB_BATCH_SIZE:\n",
    "                                 (accu_step + 1) * __C.SUB_BATCH_SIZE]\n",
    "                #Edits\n",
    "                sub_ans_ix_iter = \\\n",
    "                    ans_ix_iter[accu_step * __C.SUB_BATCH_SIZE:\n",
    "                                 (accu_step + 1) * __C.SUB_BATCH_SIZE]\n",
    "                #End of Edits\n",
    "\n",
    "                sub_ans_iter = \\\n",
    "                    ans_iter[accu_step * __C.SUB_BATCH_SIZE:\n",
    "                             (accu_step + 1) * __C.SUB_BATCH_SIZE]\n",
    "\n",
    "                \n",
    "                # when making predictions also pass the ans_iter which is a dictionary from which you\n",
    "                # can extract answers and pass them through decoders\n",
    "\n",
    "                if (__C.WITH_ANSWER):\n",
    "                    pred_img_ques, pred_ans, pred_fused, z_img_ques, z_ans, z_fused = net(\n",
    "                        sub_frcn_feat_iter,\n",
    "                        sub_grid_feat_iter,\n",
    "                        sub_bbox_feat_iter,\n",
    "                        sub_ques_ix_iter,\n",
    "                        sub_ans_ix_iter,\n",
    "                        step,\n",
    "                        epoch\n",
    "                    )\n",
    "                else:\n",
    "                     pred_img_ques = net(\n",
    "                        sub_frcn_feat_iter,\n",
    "                        sub_grid_feat_iter,\n",
    "                        sub_bbox_feat_iter,\n",
    "                        sub_ques_ix_iter,\n",
    "                        sub_ans_ix_iter,\n",
    "                        step,\n",
    "                        epoch\n",
    "                    )\n",
    "                   \n",
    "                # we need to change the loss terms accordingly\n",
    "                # now we need to modify the loss terms for the same\n",
    "                \n",
    "                #Edits: creating the loss items for each of the prediction vector\n",
    "\n",
    "                loss_item_img_ques = [pred_img_ques, sub_ans_iter]\n",
    "\n",
    "                # only calculate the ans and interp loss in case of WITH_ANSWER\n",
    "                if (__C.WITH_ANSWER):\n",
    "                    loss_item_ans = [pred_ans, sub_ans_iter]\n",
    "                    loss_item_interp = [pred_fused, sub_ans_iter]\n",
    "\n",
    "                \n",
    "                loss_nonlinear_list = __C.LOSS_FUNC_NONLINEAR[__C.LOSS_FUNC]\n",
    "                \n",
    "                # applying the same transformation on the all three\n",
    "                # althought for 'bce' loss the following does nothing\n",
    "                for item_ix, loss_nonlinear in enumerate(loss_nonlinear_list):\n",
    "                    if loss_nonlinear in ['flat']:\n",
    "                        loss_item_img_ques[item_ix] = loss_item_img_ques[item_ix].view(-1)\n",
    "                    elif loss_nonlinear:\n",
    "                        loss_item_img_ques[item_ix] = eval('F.' + loss_nonlinear + '(loss_item_img_ques[item_ix], dim=1)')\n",
    "\n",
    "                for item_ix, loss_nonlinear in enumerate(loss_nonlinear_list):\n",
    "                    if loss_nonlinear in ['flat'] and __C.WITH_ANSWER:\n",
    "                        loss_item_ans[item_ix] = loss_item_ans[item_ix].view(-1)\n",
    "                    elif loss_nonlinear and __C.WITH_ANSWER:\n",
    "                        loss_item_ans[item_ix] = eval('F.' + loss_nonlinear + '(loss_item_ans[item_ix], dim=1)')\n",
    "\n",
    "                for item_ix, loss_nonlinear in enumerate(loss_nonlinear_list):\n",
    "                    if loss_nonlinear in ['flat'] and __C.WITH_ANSWER:\n",
    "                        loss_item_interp[item_ix] = loss_item_interp[item_ix].view(-1)\n",
    "                    elif loss_nonlinear and __C.WITH_ANSWER:\n",
    "                        loss_item_interp[item_ix] = eval('F.' + loss_nonlinear + '(loss_item_interp[item_ix], dim=1)')\n",
    "\n",
    "\n",
    "                # Now we create all the four losses and then add them\n",
    "                #print(\"shape of loss_item_img_ques[0] is {} and of loss_item_img_ques[1] is {}\".format(loss_item_img_ques[0],loss_item_img_ques[1]))\n",
    "                loss_img_ques = loss_fn(loss_item_img_ques[0], loss_item_img_ques[1])\n",
    "\n",
    "                loss = 0\n",
    "                loss += loss_img_ques\n",
    "                \n",
    "                if (__C.WITH_ANSWER):\n",
    "\n",
    "                    # loss for the prediction from the answer\n",
    "                    #print(\"shape of loss_item_ans[0] is {} and of loss_item_ans[1] is {}\".format(loss_item_ans[0],loss_item_ans[1]))\n",
    "                    loss_ans = loss_fn(loss_item_ans[0], loss_item_ans[1])\n",
    "                \n",
    "                    # Loss for the prediction from the fused vector\n",
    "                    # I am keeping the loss same as bce but we can change it later for more predictions\n",
    "                    # loss_fused = interpolation loss\n",
    "                    #print(\"shape of loss_item_interp[0] is {} and of loss_item_interp[1] is {}\".format(loss_item_interp[0],loss_item_interp[1]))\n",
    "                    loss_interp = loss_fn(loss_item_interp[0], loss_item_interp[1])\n",
    "                    \n",
    "                    # we also need to multiply this fused loss by a hyperparameter alpha\n",
    "                    # put the alpha in the config and uncomment the following line\n",
    "                    loss_interp *= __C.ALPHA\n",
    "                    loss += loss_ans + loss_interp\n",
    "\n",
    "                    if (__C.WITH_FUSION_LOSS):\n",
    "\n",
    "                        # Now calculate the fusion loss\n",
    "                        #1. Higher loss for higher distance between vectors predicted\n",
    "                        # by different models for same example\n",
    "\n",
    "                        dist_calc = (z_img_ques - z_ans).pow(2).sum(1).sqrt()\n",
    "                        #print(\"Count of distances being clipped (true is clipped): \", np.unique((dist_calc > __C.CAP_DIST).cpu().numpy(), return_counts=True))\n",
    "\n",
    "                        '''\n",
    "                        loss_fusion = torch.min(\n",
    "                                torch.tensor(__C.CAP_DIST).cuda(),\n",
    "                                dist_calc\n",
    "                                ).mean()\n",
    "\n",
    "                        #2. Lower loss for more distance between two pred vectors of same model\n",
    "                        loss_fusion -= torch.min(\n",
    "                                torch.tensor(__C.CAP_DIST).cuda(), \n",
    "                                torch.pdist(z_img_ques, 2)\n",
    "                                ).mean() \n",
    "\n",
    "                        loss_fusion -= torch.min(\n",
    "                                torch.tensor(__C.CAP_DIST).cuda(), \n",
    "                                torch.pdist(z_ans, 2)\n",
    "                                ).mean() \n",
    "                        '''\n",
    "\n",
    "                        loss_fusion = dist_calc.mean()\n",
    "\n",
    "                        #2. Lower loss for more distance between two pred vectors of same model\n",
    "                        '''\n",
    "                        calculating pairwise intra distance on same type questions\n",
    "                        '''\n",
    "                        '''\n",
    "                        types = ['other', 'yes/no', 'number']\n",
    "                        for i in range(3):\n",
    "                            j = (i+1)%3\n",
    "                            indices_i = [k for k, val in enumerate(ques_type) if val == types[i]]\n",
    "                            indices_j = [k for k, val in enumerate(ques_type) if val == types[j]]\n",
    "                            if ((indices_i != []) and (indices_j != [])):\n",
    "                                loss_fusion -= torch.cdist(z_img_ques[indices_i], z_img_ques[indices_j]).mean()\n",
    "                                loss_fusion -= torch.cdist(z_ans[indices_i], z_ans[indices_j]).mean()\n",
    "                            if (indices_i != []):\n",
    "                                loss_fusion += torch.pdist(z_img_ques[indices_i], 2).mean()\n",
    "                                loss_fusion += torch.pdist(z_ans[indices_i], 2).mean()\n",
    "                        '''\n",
    "                        loss_fusion -= torch.pdist(z_img_ques, 2).mean() \n",
    "\n",
    "                        loss_fusion -= torch.pdist(z_ans, 2).mean() \n",
    "\n",
    "\n",
    "                        # Multiply the loss fusion with hyperparameter beta\n",
    "                        loss_fusion *= __C.BETA\n",
    "\n",
    "                        #print('fusion loss is : {}'.format(loss_fusion))\n",
    "\n",
    "                        loss += loss_fusion\n",
    "\n",
    "                \n",
    "                loss /= __C.GRAD_ACCU_STEPS\n",
    "                loss.backward()\n",
    "\n",
    "                loss_tmp += loss.cpu().data.numpy() * __C.GRAD_ACCU_STEPS\n",
    "                loss_sum += loss.cpu().data.numpy() * __C.GRAD_ACCU_STEPS\n",
    "\n",
    "                # calculating temp loss of each type\n",
    "                if __C.WITH_ANSWER:\n",
    "                    loss_img_ques_tmp += loss_img_ques.cpu().data.numpy() * __C.GRAD_ACCU_STEPS\n",
    "                    loss_ans_tmp += loss_ans.cpu().data.numpy() * __C.GRAD_ACCU_STEPS\n",
    "                    loss_interp_tmp += loss_interp.cpu().data.numpy() * __C.GRAD_ACCU_STEPS\n",
    "                    if (__C.WITH_FUSION_LOSS):\n",
    "                        loss_fusion_tmp += loss_fusion.cpu().data.numpy() * __C.GRAD_ACCU_STEPS\n",
    "\n",
    "\n",
    "            if __C.VERBOSE:\n",
    "                if dataset_eval is not None:\n",
    "                    mode_str = __C.SPLIT['train'] + '->' + __C.SPLIT['val']\n",
    "                else:\n",
    "                    mode_str = __C.SPLIT['train'] + '->' + __C.SPLIT['test']\n",
    "\n",
    "                print(\"\\r[Version %s][Epoch %2d][Step %4d/%4d] Loss: %.4f [iq: %.4f,ans: %.4f,interp: %.4f,fusion: %.4f]\" % (\n",
    "                    __C.VERSION,\n",
    "                    epoch + 1,\n",
    "                    step,\n",
    "                    int(data_size / __C.BATCH_SIZE),\n",
    "                    loss_tmp / __C.SUB_BATCH_SIZE,\n",
    "                    loss_img_ques_tmp / __C.SUB_BATCH_SIZE,\n",
    "                    loss_ans_tmp / __C.SUB_BATCH_SIZE,\n",
    "                    loss_interp_tmp / __C.SUB_BATCH_SIZE,\n",
    "                    loss_fusion_tmp / __C.SUB_BATCH_SIZE\n",
    "                ), end = '          ')\n",
    "\n",
    "            # Gradient norm clipping\n",
    "            if __C.GRAD_NORM_CLIP > 0:\n",
    "                nn.utils.clip_grad_norm_(\n",
    "                    net.parameters(),\n",
    "                    __C.GRAD_NORM_CLIP\n",
    "                )\n",
    "\n",
    "            # Save the gradient information\n",
    "            for name in range(len(named_params)):\n",
    "                norm_v = torch.norm(named_params[name][1].grad).cpu().data.numpy() \\\n",
    "                    if named_params[name][1].grad is not None else 0\n",
    "                grad_norm[name] += norm_v * __C.GRAD_ACCU_STEPS\n",
    "                # print('Param %-3s Name %-80s Grad_Norm %-20s'%\n",
    "                #       (str(grad_wt),\n",
    "                #        params[grad_wt][0],\n",
    "                #        str(norm_v)))\n",
    "\n",
    "            optim.step()\n",
    "\n",
    "        time_end = time.time()\n",
    "        elapse_time = time_end-time_start\n",
    "        print('Finished in {}s'.format(int(elapse_time)))\n",
    "        epoch_finish = epoch + 1\n",
    "\n",
    "        # Save checkpoint\n",
    "        if __C.N_GPU > 1:\n",
    "            state = {\n",
    "                'state_dict': net.module.state_dict(),\n",
    "                'optimizer': optim.optimizer.state_dict(),\n",
    "                'lr_base': optim.lr_base,\n",
    "                'epoch': epoch_finish\n",
    "            }\n",
    "        else:\n",
    "            state = {\n",
    "                'state_dict': net.state_dict(),\n",
    "                'optimizer': optim.optimizer.state_dict(),\n",
    "                'lr_base': optim.lr_base,\n",
    "                'epoch': epoch_finish\n",
    "            }\n",
    "        torch.save(\n",
    "            state,\n",
    "            __C.CKPTS_PATH +\n",
    "            '/ckpt_' + __C.VERSION +\n",
    "            '/epoch' + str(epoch_finish) +\n",
    "            '.pkl'\n",
    "        )\n",
    "\n",
    "        wandb.save(\n",
    "            __C.CKPTS_PATH +\n",
    "            '/ckpt_' + __C.VERSION +\n",
    "            '/epoch' + str(epoch_finish) +\n",
    "            '.h5'\n",
    "        )\n",
    "        \n",
    "        # Logging\n",
    "        logfile = open(\n",
    "            __C.LOG_PATH +\n",
    "            '/log_run_' + __C.VERSION + '.txt',\n",
    "            'a+'\n",
    "        )\n",
    "        logfile.write(\n",
    "            'Epoch: ' + str(epoch_finish) +\n",
    "            ', Loss: ' + str(loss_sum / data_size) +\n",
    "            ', Lr: ' + str(optim._rate) + '\\n' +\n",
    "            'Elapsed time: ' + str(int(elapse_time)) + \n",
    "            ', Speed(s/batch): ' + str(elapse_time / step) +\n",
    "            '\\n\\n'\n",
    "        )\n",
    "        logfile.close()\n",
    "\n",
    "        wandb.log({\n",
    "            'Loss': float(loss_sum / data_size),\n",
    "            'Learning Rate': optim._rate,\n",
    "            'Elapsed time': int(elapse_time) \n",
    "            })\n",
    "\n",
    "        # ---------------------------------------------- #\n",
    "        # ---- Create visualizations in new processes----#\n",
    "        # ---------------------------------------------- #\n",
    "        dic = {}\n",
    "        dic['version'] = __C.VERSION\n",
    "        dic['epoch'] = epoch \n",
    "        dic['num_samples'] = 1000\n",
    "\n",
    "        p = Pool(processes= 1)\n",
    "        p.map_async(vis_func, (dic, ))\n",
    "        p.close()\n",
    "\n",
    "        # Eval after every epoch\n",
    "        epoch_dict = {\n",
    "                'current_epoch': epoch\n",
    "                }\n",
    "        __C.add_args(epoch_dict)\n",
    "        if dataset_eval is not None:\n",
    "            test_engine(\n",
    "                __C,\n",
    "                dataset_eval,\n",
    "                state_dict=net.state_dict(),\n",
    "                validation=True,\n",
    "                epoch = 0\n",
    "            )\n",
    "        p.join()\n",
    "\n",
    "        # if self.__C.VERBOSE:\n",
    "        #     logfile = open(\n",
    "        #         self.__C.LOG_PATH +\n",
    "        #         '/log_run_' + self.__C.VERSION + '.txt',\n",
    "        #         'a+'\n",
    "        #     )\n",
    "        #     for name in range(len(named_params)):\n",
    "        #         logfile.write(\n",
    "        #             'Param %-3s Name %-80s Grad_Norm %-25s\\n' % (\n",
    "        #                 str(name),\n",
    "        #                 named_params[name][0],\n",
    "        #                 str(grad_norm[name] / data_size * self.__C.BATCH_SIZE)\n",
    "        #             )\n",
    "        #         )\n",
    "        #     logfile.write('\\n')\n",
    "        #     logfile.close()\n",
    "\n",
    "        loss_sum = 0\n",
    "        grad_norm = np.zeros(len(named_params))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing log file........\n",
      "Finished!\n",
      "\n",
      "Model being used is mfb\n",
      "Training________________________________\n",
      "using the pretrained model for ans+img encoder and decoder both parts\n",
      "Loading ckpt from ./ckpts/ckpt_mfb_ans_img_only/epoch13.pkl\n",
      "Finish!\n",
      "updating keys in net_state_dict form pretrained state dict\n",
      "loading this state dict in the net\n",
      "loaded net state dict succesfully\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://app.wandb.ai/frostlabs/openvqa\" target=\"_blank\">https://app.wandb.ai/frostlabs/openvqa</a><br/>\n",
       "                Run page: <a href=\"https://app.wandb.ai/frostlabs/openvqa/runs/fawbavqr\" target=\"_blank\">https://app.wandb.ai/frostlabs/openvqa/runs/fawbavqr</a><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to query for notebook name, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable\n",
      "wandb: Wandb version 0.8.28 is available!  To upgrade, please run:\n",
      "wandb:  $ pip install wandb --upgrade\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Version fixed_decoder][Epoch  1][Step  165/6933] Loss: 8.7682 [iq: 2.4543,ans: 3.8242,interp: 2.3413,fusion: 0.1484]             "
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-acd60a429d90>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mexecution\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m__C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRUN_MODE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-3-b0a0a4f52cc9>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, run_mode)\u001b[0m\n\u001b[1;32m     26\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRESUME\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty_log\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mVERSION\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m             \u001b[0mtrain_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__C\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset_eval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mrun_mode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'val'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-e71080b72a38>\u001b[0m in \u001b[0;36mtrain_engine\u001b[0;34m(__C, dataset, dataset_eval)\u001b[0m\n\u001b[1;32m    407\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    408\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[0;34m/=\u001b[0m \u001b[0m__C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGRAD_ACCU_STEPS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 409\u001b[0;31m                 \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    410\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    411\u001b[0m                 \u001b[0mloss_tmp\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0m__C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGRAD_ACCU_STEPS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anaconda2/envs/frost/lib/python3.6/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    193\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    194\u001b[0m         \"\"\"\n\u001b[0;32m--> 195\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    196\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anaconda2/envs/frost/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     97\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     98\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "execution.run(__C.RUN_MODE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}