From 78aaa379265fc151ec1a7c4b5e4945ad7a6fb83c Mon Sep 17 00:00:00 2001 From: Zhanwen Chen Date: Fri, 4 Mar 2022 16:00:05 -0500 Subject: [PATCH] Update 0011.ipynb Sync with server --- ipynb/eval_predcls/0011.ipynb | 6868 ++++++++++++++++++++++++++++++++- 1 file changed, 6740 insertions(+), 128 deletions(-) diff --git a/ipynb/eval_predcls/0011.ipynb b/ipynb/eval_predcls/0011.ipynb index cafc99c..b8157f6 100644 --- a/ipynb/eval_predcls/0011.ipynb +++ b/ipynb/eval_predcls/0011.ipynb @@ -22,7 +22,7 @@ { "data": { "text/plain": [ - "'0.3.0.post4'" + "'1.9.0'" ] }, "execution_count": 2, @@ -42,16 +42,27 @@ "outputs": [], "source": [ "exp_name = 'exp_045_rep'\n", - "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"2\"" + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'/home/zhanwen/kangaroo_meta/kangaroo_gbnet_all_plus_wikidata_51_with_emb_txt_lrga_bpl_no_sa_1_20220302_mine/ipynb/train_predcls'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "eval_epoch = 11" + "os.getcwd()" ] }, { @@ -67,8 +78,6 @@ "import pandas as pd\n", "import time\n", "import os\n", - "from tqdm import tqdm\n", - "import pickle\n", "\n", "from config import ModelConfig, BOX_SCALE, IM_SCALE\n", "from torch.nn import functional as F\n", @@ -84,27 +93,63 @@ "cell_type": "code", "execution_count": 6, "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "def set_seed(seed):\n", + " torch.backends.cudnn.deterministic = True\n", + " torch.backends.cudnn.benchmark = False\n", + " torch.manual_seed(seed)\n", + " torch.cuda.manual_seed_all(seed)\n", + " np.random.seed(seed)\n", + " random.seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "MODEL_NAME = 'lrga_bpl_no_sa_1'\n", + "def send_simple_message(epoch, body):\n", + " return requests.post(\n", + " \"https://api.mailgun.net/v3/mg.zhanwenchen.com/messages\",\n", + " auth=(\"api\", \"1f6fcd31dc6e4ce871093bac8dcfda1b-2bf328a5-80c22d94\"),\n", + " data={\"from\": \"Zhanwen's Code Notifications \",\n", + " \"to\": [\"phil.zhanwen.chen@gmail.com\"],\n", + " \"subject\": f\"Finshed Training Epoch {epoch} for Model {MODEL_NAME}\",\n", + " \"text\": f\"Finshed Training Epoch {epoch} for Model {MODEL_NAME}. The mean recall is {body}\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "~~~~~~~~ Hyperparameters used: ~~~~~~~\n", - "ckpt : checkpoints/kern_predcls/exp_045_rep/vgrel-11.tar\n", - "save_dir : \n", + "ckpt : checkpoints/vgdet/vgrel-12.tar\n", + "save_dir : checkpoints/kern_predcls/exp_045_rep\n", "num_gpus : 1\n", "num_workers : 1\n", - "lr : 0.001\n", - "batch_size : 1\n", + "lr : 0.0001\n", + "batch_size : 3\n", "val_size : 5000\n", "l2 : 0.0001\n", "adamwd : 0.0\n", "clip : 5.0\n", "print_interval : 1000\n", "mode : predcls\n", - "cache : caches/exp_045_rep/kern_predcls-11.pkl\n", - "adam : False\n", - "test : True\n", + "cache : \n", + "adam : True\n", + "test : False\n", "num_epochs : 50\n", "use_resnet : False\n", "use_proposals : False\n", @@ -121,48 +166,94 @@ "ggnn_rel_output_dim : 512\n", "use_rel_knowledge : False\n", "rel_knowledge : \n", - "tb_log_dir : \n", - "save_rel_recall : results/exp_045_rep/kern_rel_recall_predcls-11.pkl\n" + "tb_log_dir : summaries/kern_predcls/exp_045_rep\n", + "save_rel_recall : \n" ] } ], "source": [ "conf = ModelConfig(f'''\n", "-m predcls -p 1000 -clip 5 \n", - "-ckpt checkpoints/kern_predcls/{exp_name}/vgrel-{eval_epoch}.tar \n", - "-test\n", - "-b 1\n", + "-tb_log_dir summaries/kern_predcls/{exp_name} \n", + "-save_dir checkpoints/kern_predcls/{exp_name}\n", + "-ckpt checkpoints/vgdet/vgrel-12.tar \n", + "-val_size 5000 \n", + "-adam \n", + "-b 3\n", "-ngpu 1\n", - "-cache caches/{exp_name}/kern_predcls-{eval_epoch}.pkl \\\n", - "-save_rel_recall results/{exp_name}/kern_rel_recall_predcls-{eval_epoch}.pkl\n", + "-lr 1e-4 \n", "''')" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - "train, val, test = VG.splits(num_val_im=conf.val_size, filter_duplicate_rels=True,\n", + "conf.MODEL.CONF_MAT_FREQ_TRAIN = '../../../vgmeta/conf_mat_freq_train.npy'\n", + "conf.MODEL.LRGA.USE_LRGA = True\n", + "conf.MODEL.LRGA.K = 50\n", + "conf.MODEL.LRGA.DROPOUT = 0.5\n", + "conf.MODEL.GN.NUM_GROUPS = 1024//8" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'/home/zhanwen/kangaroo_meta/kangaroo_gbnet_all_plus_wikidata_51_with_emb_txt_lrga_bpl_no_sa_1_20220302_mine/ipynb/train_predcls'" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "os.getcwd()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataloader using BPL\n", + "Dataloader using BPL\n", + "Dataloader using BPL\n" + ] + } + ], + "source": [ + "set_seed(0)\n", + "train, val, _ = VG.splits(num_val_im=conf.val_size, filter_duplicate_rels=True,\n", " use_proposals=conf.use_proposals,\n", - " filter_non_overlap=conf.mode == 'sgdet')\n" + " filter_non_overlap=conf.mode == 'sgdet', with_clean_classifier=True, get_state=False)\n" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ - "ind_to_predicates = train.ind_to_predicates # ind_to_predicates[0] means no relationship\n", - "if conf.test:\n", - " val = test" + "ind_to_predicates = train.ind_to_predicates # ind_to_predicates[0] means no relationship" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -174,51 +265,396 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "!!!!!!!!!With Confusion Matrix Channel!!!!!\n", + "No SA: Not using adj_normalize.self.sa=False\n" + ] + } + ], "source": [ "detector = KERN(classes=train.ind_to_classes, rel_classes=train.ind_to_predicates,\n", " num_gpus=conf.num_gpus, mode=conf.mode, require_overlap_det=True,\n", " use_resnet=conf.use_resnet, use_proposals=conf.use_proposals, pooling_dim=conf.pooling_dim,\n", " ggnn_rel_time_step_num=3, ggnn_rel_hidden_dim=1024, ggnn_rel_output_dim=None,\n", - " graph_path=os.path.join(codebase, 'graphs/005/all_edges.pkl'), \n", - " emb_path=os.path.join(codebase, 'graphs/001/emb_mtx.pkl'), \n", + " graph_path=os.path.join(codebase, 'graphs/005/edge_dict_all_plus_wikidata_177_20220208.pkl'), \n", + " emb_path=os.path.join(codebase, 'graphs/001/emb_mtx_wiki_51.pkl'), \n", " rel_counts_path=os.path.join(codebase, 'graphs/001/pred_counts.pkl'), \n", " use_knowledge=True, use_embedding=True, refine_obj_cls=False,\n", - " class_volume=1.0\n", + " class_volume=1.0, with_clean_classifier=True, with_transfer=True, sa=False, config=conf,\n", " )\n", "\n" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ - "detector.cuda();\n" + "# Freeze the detector\n", + "for n, param in detector.detector.named_parameters():\n", + " param.requires_grad = False" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 16, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " 454.8M total parameters \n", + " ----- \n", + " \n", + "detector.roi_fmap.0.weight : [4096,25088] (102760448) ( )\n", + "roi_fmap.1.0.weight : [4096,25088] (102760448) (grad)\n", + "roi_fmap_obj.0.weight : [4096,25088] (102760448) (grad)\n", + "detector.roi_fmap.3.weight : [4096,4096] (16777216) ( )\n", + "roi_fmap.1.3.weight : [4096,4096] (16777216) (grad)\n", + "roi_fmap_obj.3.weight : [4096,4096] (16777216) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: [3328,3328] (11075584) (grad)\n", + "ggnn_rel_reason.obj_proj.weight : [1024,4096] ( 4194304) (grad)\n", + "ggnn_rel_reason.rel_proj.weight : [1024,4096] ( 4194304) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: [2048,2048] ( 4194304) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: [1024,3328] ( 3407872) (grad)\n", + "detector.bbox_fc.weight : [708,4096] ( 2899968) ( )\n", + "detector.features.19.weight : [512,512,3,3] ( 2359296) ( )\n", + "detector.features.21.weight : [512,512,3,3] ( 2359296) ( )\n", + "detector.features.24.weight : [512,512,3,3] ( 2359296) ( )\n", + "detector.features.26.weight : [512,512,3,3] ( 2359296) ( )\n", + "detector.features.28.weight : [512,512,3,3] ( 2359296) ( )\n", + "detector.rpn_head.conv.0.weight : [512,512,3,3] ( 2359296) ( )\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: [1024,2048] ( 2097152) (grad)\n", + "detector.features.17.weight : [512,256,3,3] ( 1179648) ( )\n", + "union_boxes.conv.4.weight : [512,256,3,3] ( 1179648) (grad)\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : [1024,1124] ( 1150976) (grad)\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : [1024,1124] ( 1150976) (grad)\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : [1024,1124] ( 1150976) (grad)\n", + "ggnn_rel_reason.ggnn.dimension_reduce.3.0.weight : [1024,1124] ( 1150976) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred_clean.model.0.linear.weight: [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred_clean.model.2.linear.weight: [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: [1024,1024] ( 1048576) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: [1024,768] ( 786432) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: [1024,768] ( 786432) (grad)\n", + "detector.score_fc.weight : [177,4096] ( 724992) ( )\n", + "detector.features.12.weight : [256,256,3,3] ( 589824) ( )\n", + "detector.features.14.weight : [256,256,3,3] ( 589824) ( )\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: [768,768] ( 589824) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: [768,768] ( 589824) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: [512,1024] ( 524288) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: [512,1024] ( 524288) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: [512,1024] ( 524288) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: [512,1024] ( 524288) (grad)\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : [1024,300] ( 307200) (grad)\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : [1024,300] ( 307200) (grad)\n", + "detector.features.10.weight : [256,128,3,3] ( 294912) ( )\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : [200,1024] ( 204800) (grad)\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : [200,1024] ( 204800) (grad)\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : [200,1024] ( 204800) (grad)\n", + "ggnn_rel_reason.ggnn.attention.3.w.0.weight : [200,1024] ( 204800) (grad)\n", + "detector.features.7.weight : [128,128,3,3] ( 147456) ( )\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: [256,512] ( 131072) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: [256,512] ( 131072) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: [256,512] ( 131072) (grad)\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: [256,512] ( 131072) (grad)\n", + "detector.features.5.weight : [128,64,3,3] ( 73728) ( )\n", + "detector.rpn_head.conv.2.weight : [120,512,1,1] ( 61440) ( )\n", + "detector.features.2.weight : [64,64,3,3] ( 36864) ( )\n", + "union_boxes.conv.0.weight : [256,2,7,7] ( 25088) (grad)\n", + "detector.features.0.weight : [64,3,3,3] ( 1728) ( )\n", + "ggnn_rel_reason.ggnn.gn.0.weight : [1024] ( 1024) (grad)\n", + "ggnn_rel_reason.ggnn.gn.1.weight : [1024] ( 1024) (grad)\n", + "union_boxes.conv.6.weight : [512] ( 512) (grad)\n", + "union_boxes.conv.2.weight : [256] ( 256) (grad)\n" + ] + } + ], + "source": [ + "print(print_para(detector), flush=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "def get_optim(lr):\n", + " # Lower the learning rate on the VGG fully connected layers by 1/10th. It's a hack, but it helps\n", + " # stabilize the models.\n", + " fc_params = [p for n,p in detector.named_parameters() if (n.startswith('roi_fmap') or 'clean' in n) and p.requires_grad]\n", + " non_fc_params = [p for n,p in detector.named_parameters() if not (n.startswith('roi_fmap') or 'clean' in n) and p.requires_grad]\n", + " params = [{'params': fc_params, 'lr': lr / 10.0}, {'params': non_fc_params}]\n", + " # params = [p for n,p in detector.named_parameters() if p.requires_grad]\n", + "\n", + " if conf.adam:\n", + " optimizer = optim.Adam(params, weight_decay=conf.adamwd, lr=lr, eps=1e-3)\n", + " else:\n", + " optimizer = optim.SGD(params, weight_decay=conf.l2, lr=lr, momentum=0.9)\n", + "\n", + " # scheduler = ReduceLROnPlateau(optimizer, 'max', patience=3, factor=0.1,\n", + " # verbose=True, threshold=0.0001, threshold_mode='abs', cooldown=1)\n", + " return optimizer #, scheduler\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 18, "metadata": {}, + "outputs": [], + "source": [ + "ckpt = torch.load(conf.ckpt)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "scrolled": false, + "tags": [] + }, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading EVERYTHING\n", + "Successfully copied detector.features.0.weight\n", + "Successfully copied detector.features.0.bias\n", + "Successfully copied detector.features.2.weight\n", + "Successfully copied detector.features.2.bias\n", + "Successfully copied detector.features.5.weight\n", + "Successfully copied detector.features.5.bias\n", + "Successfully copied detector.features.7.weight\n", + "Successfully copied detector.features.7.bias\n", + "Successfully copied detector.features.10.weight\n", + "Successfully copied detector.features.10.bias\n", + "Successfully copied detector.features.12.weight\n", + "Successfully copied detector.features.12.bias\n", + "Successfully copied detector.features.14.weight\n", + "Successfully copied detector.features.14.bias\n", + "Successfully copied detector.features.17.weight\n", + "Successfully copied detector.features.17.bias\n", + "Successfully copied detector.features.19.weight\n", + "Successfully copied detector.features.19.bias\n", + "Successfully copied detector.features.21.weight\n", + "Successfully copied detector.features.21.bias\n", + "Successfully copied detector.features.24.weight\n", + "Successfully copied detector.features.24.bias\n", + "Successfully copied detector.features.26.weight\n", + "Successfully copied detector.features.26.bias\n", + "Successfully copied detector.features.28.weight\n", + "Successfully copied detector.features.28.bias\n", + "Successfully copied detector.roi_fmap.0.weight\n", + "Successfully copied detector.roi_fmap.0.bias\n", + "Successfully copied detector.roi_fmap.3.weight\n", + "Successfully copied detector.roi_fmap.3.bias\n", + "Successfully copied detector.score_fc.weight\n", + "Successfully copied detector.score_fc.bias\n", + "Successfully copied detector.bbox_fc.weight\n", + "Successfully copied detector.bbox_fc.bias\n", + "Successfully copied detector.rpn_head.anchors\n", + "Successfully copied detector.rpn_head.conv.0.weight\n", + "Successfully copied detector.rpn_head.conv.0.bias\n", + "Successfully copied detector.rpn_head.conv.2.weight\n", + "Successfully copied detector.rpn_head.conv.2.bias\n", + "Successfully copied union_boxes.conv.0.weight\n", + "Successfully copied union_boxes.conv.0.bias\n", + "Successfully copied union_boxes.conv.2.weight\n", + "Successfully copied union_boxes.conv.2.bias\n", + "Successfully copied union_boxes.conv.2.running_mean\n", + "Successfully copied union_boxes.conv.2.running_var\n", + "Successfully copied union_boxes.conv.2.num_batches_tracked\n", + "Successfully copied union_boxes.conv.4.weight\n", + "Successfully copied union_boxes.conv.4.bias\n", + "Successfully copied union_boxes.conv.6.weight\n", + "Successfully copied union_boxes.conv.6.bias\n", + "Successfully copied union_boxes.conv.6.running_mean\n", + "Successfully copied union_boxes.conv.6.running_var\n", + "Successfully copied union_boxes.conv.6.num_batches_tracked\n", + "Successfully copied roi_fmap.1.0.weight\n", + "Successfully copied roi_fmap.1.0.bias\n", + "Successfully copied roi_fmap.1.3.weight\n", + "Successfully copied roi_fmap.1.3.bias\n", + "Successfully copied roi_fmap_obj.0.weight\n", + "Successfully copied roi_fmap_obj.0.bias\n", + "Successfully copied roi_fmap_obj.3.weight\n", + "Successfully copied roi_fmap_obj.3.bias\n", + "Successfully copied ggnn_rel_reason.obj_proj.weight\n", + "Successfully copied ggnn_rel_reason.obj_proj.bias\n", + "Successfully copied ggnn_rel_reason.rel_proj.weight\n", + "Successfully copied ggnn_rel_reason.rel_proj.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_init_ont_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_init_ont_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_init_ont_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_init_ont_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight\n", + "Successfully copied ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias\n", + "We couldn't find ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight,ggnn_rel_reason.ggnn.attention.1.w.0.bias,ggnn_rel_reason.ggnn.attention.3.w.0.weight,ggnn_rel_reason.ggnn.attention.2.w.0.bias,ggnn_rel_reason.ggnn.attention.2.w.0.weight,ggnn_rel_reason.ggnn.fc_output_proj_img_pred_clean.model.0.linear.bias,ggnn_rel_reason.ggnn.fc_output_proj_img_pred_clean.model.0.linear.weight,ggnn_rel_reason.ggnn.attention.0.w.0.bias,ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight,ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight,ggnn_rel_reason.ggnn.dimension_reduce.3.0.weight,ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias,ggnn_rel_reason.ggnn.attention.0.w.0.weight,ggnn_rel_reason.ggnn.attention.3.w.0.bias,ggnn_rel_reason.ggnn.attention.1.w.0.weight,ggnn_rel_reason.ggnn.gn.1.weight,ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias,ggnn_rel_reason.ggnn.gn.1.bias,ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias,ggnn_rel_reason.ggnn.gn.0.weight,ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias,ggnn_rel_reason.ggnn.fc_output_proj_img_pred_clean.model.2.linear.weight,ggnn_rel_reason.ggnn.fc_output_proj_img_pred_clean.model.2.linear.bias,ggnn_rel_reason.ggnn.dimension_reduce.3.0.bias,ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight,ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias,ggnn_rel_reason.ggnn.gn.0.bias,ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight\n", + "cannot restore optimistically\n" + ] } ], "source": [ - "ckpt = torch.load(conf.ckpt)\n", - "optimistic_restore(detector, ckpt['state_dict'])\n" + "using_pretrained_gbnet = conf.ckpt.split('-')[-2].split('/')[-1] == 'vgrel'\n", + "if using_pretrained_gbnet:\n", + " print(\"Loading EVERYTHING\")\n", + " start_epoch = ckpt['epoch']\n", + "\n", + " if not optimistic_restore(detector, ckpt['state_dict']):\n", + " print('cannot restore optimistically')\n", + " start_epoch = -1\n", + " # optimistic_restore(detector.detector, torch.load('checkpoints/vgdet/vg-28.tar')['state_dict'])\n", + "# raise\n", + "else:\n", + " start_epoch = -1\n", + " optimistic_restore(detector.detector, ckpt['state_dict'])\n", + "\n", + " detector.roi_fmap[1][0].weight.data.copy_(ckpt['state_dict']['roi_fmap.0.weight'])\n", + " detector.roi_fmap[1][3].weight.data.copy_(ckpt['state_dict']['roi_fmap.3.weight'])\n", + " detector.roi_fmap[1][0].bias.data.copy_(ckpt['state_dict']['roi_fmap.0.bias'])\n", + " detector.roi_fmap[1][3].bias.data.copy_(ckpt['state_dict']['roi_fmap.3.bias'])\n", + "\n", + " detector.roi_fmap_obj[0].weight.data.copy_(ckpt['state_dict']['roi_fmap.0.weight'])\n", + " detector.roi_fmap_obj[3].weight.data.copy_(ckpt['state_dict']['roi_fmap.3.weight'])\n", + " detector.roi_fmap_obj[0].bias.data.copy_(ckpt['state_dict']['roi_fmap.0.bias'])\n", + " detector.roi_fmap_obj[3].bias.data.copy_(ckpt['state_dict']['roi_fmap.3.bias'])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "detector.cuda();\n" ] }, { @@ -230,11 +666,118 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "def train_epoch(epoch_num):\n", + " detector.train()\n", + " tr = []\n", + " start = time.time()\n", + " for b, batch in enumerate(train_loader):\n", + " result, loss_pd = train_batch(batch, verbose=b % (conf.print_interval*10) == 0)\n", + " tr.append(loss_pd)\n", + " '''\n", + " if b % 100 == 0:\n", + " print(loss_pd)\n", + " gt = result.rel_labels[:,3].data.cpu().numpy()\n", + " out = result.rel_dists.data.cpu().numpy()\n", + " ind = np.where(gt)[0]\n", + " print(gt[ind])\n", + " print(np.argmax(out[ind], 1))\n", + " print(np.argmax(out[ind, 1:], 1) + 1)\n", + " '''\n", + "\n", + " if b % conf.print_interval == 0 and b >= conf.print_interval:\n", + " mn = pd.concat(tr[-conf.print_interval:], axis=1).mean(1)\n", + " time_per_batch = (time.time() - start) / conf.print_interval\n", + " print(\"\\ne{:2d}b{:5d}/{:5d} {:.3f}s/batch, {:.1f}m/epoch\".format(\n", + " epoch_num, b, len(train_loader), time_per_batch, len(train_loader) * time_per_batch / 60))\n", + " print(mn)\n", + " print('-----------', flush=True)\n", + " start = time.time()\n", + " return pd.concat(tr, axis=1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "def train_batch(b, verbose=False):\n", + " \"\"\"\n", + " :param b: contains:\n", + " :param imgs: the image, [batch_size, 3, IM_SIZE, IM_SIZE]\n", + " :param all_anchors: [num_anchors, 4] the boxes of all anchors that we'll be using\n", + " :param all_anchor_inds: [num_anchors, 2] array of the indices into the concatenated\n", + " RPN feature vector that give us all_anchors,\n", + " each one (img_ind, fpn_idx)\n", + " :param im_sizes: a [batch_size, 4] numpy array of (h, w, scale, num_good_anchors) for each image.\n", + "\n", + " :param num_anchors_per_img: int, number of anchors in total over the feature pyramid per img\n", + "\n", + " Training parameters:\n", + " :param train_anchor_inds: a [num_train, 5] array of indices for the anchors that will\n", + " be used to compute the training loss (img_ind, fpn_idx)\n", + " :param gt_boxes: [num_gt, 4] GT boxes over the batch.\n", + " :param gt_classes: [num_gt, 2] gt boxes where each one is (img_id, class)\n", + " :return:\n", + " \"\"\"\n", + " result = detector[b]\n", + " losses = {}\n", + " losses['class_loss'] = detector.obj_loss(result)\n", + " losses['rel_loss'] = detector.rel_loss(result)\n", + " loss = sum(losses.values())\n", + "\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " clip_grad_norm(\n", + " [(n, p) for n, p in detector.named_parameters() if p.grad is not None],\n", + " max_norm=conf.clip, verbose=verbose, clip=True)\n", + " losses['total'] = loss\n", + " optimizer.step()\n", + " loss_pd = pd.Series({x: y.data.item() for x, y in losses.items()})\n", + " return result, loss_pd\n" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "def val_epoch():\n", + " detector.eval()\n", + " evaluator_list = [] # for calculating recall of each relationship except no relationship\n", + " evaluator_multiple_preds_list = []\n", + " for index, name in enumerate(ind_to_predicates):\n", + " if index == 0:\n", + " continue\n", + " evaluator_list.append((index, name, BasicSceneGraphEvaluator.all_modes()))\n", + " evaluator_multiple_preds_list.append((index, name, BasicSceneGraphEvaluator.all_modes(multiple_preds=True)))\n", + " evaluator = BasicSceneGraphEvaluator.all_modes() # for calculating recall\n", + " evaluator_multiple_preds = BasicSceneGraphEvaluator.all_modes(multiple_preds=True)\n", + " for val_b, batch in enumerate(val_loader):\n", + " val_batch(conf.num_gpus * val_b, batch, evaluator, evaluator_multiple_preds, evaluator_list, evaluator_multiple_preds_list)\n", + "\n", + " recall = evaluator[conf.mode].print_stats()\n", + " recall_mp = evaluator_multiple_preds[conf.mode].print_stats()\n", + " \n", + " mean_recall = calculate_mR_from_evaluator_list(evaluator_list, conf.mode)\n", + " mean_recall_mp = calculate_mR_from_evaluator_list(evaluator_multiple_preds_list, conf.mode, multiple_preds=True)\n", + " \n", + " detector.train()\n", + " return recall, recall_mp, mean_recall, mean_recall_mp\n" + ] + }, + { + "cell_type": "code", + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ - "def val_batch(batch_num, b, evaluator, evaluator_multiple_preds, evaluator_list, evaluator_multiple_preds_list, thrs=(20, 50, 100)):\n", + "def val_batch(batch_num, b, evaluator, evaluator_multiple_preds, evaluator_list, evaluator_multiple_preds_list):\n", " det_res = detector[b]\n", " if conf.num_gpus == 1:\n", " det_res = [det_res]\n", @@ -245,49 +788,88 @@ " 'gt_relations': val.relationships[batch_num + i].copy(),\n", " 'gt_boxes': val.gt_boxes[batch_num + i].copy(),\n", " }\n", - " assert np.all(objs_i[rels_i[:,0]] > 0) and np.all(objs_i[rels_i[:,1]] > 0)\n", - " # assert np.all(rels_i[:,2] > 0)\n", + " assert np.all(objs_i[rels_i[:, 0]] > 0) and np.all(objs_i[rels_i[:, 1]] > 0)\n", "\n", " pred_entry = {\n", " 'pred_boxes': boxes_i * BOX_SCALE/IM_SCALE,\n", " 'pred_classes': objs_i,\n", " 'pred_rel_inds': rels_i,\n", " 'obj_scores': obj_scores_i,\n", - " 'rel_scores': pred_scores_i,\n", + " 'rel_scores': pred_scores_i, # hack for now.\n", " }\n", - " all_pred_entries.append(pred_entry)\n", "\n", " eval_entry(conf.mode, gt_entry, pred_entry, evaluator, evaluator_multiple_preds, \n", - " evaluator_list, evaluator_multiple_preds_list)\n" + " evaluator_list, evaluator_multiple_preds_list)\n", + "\n" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ - "evaluator = BasicSceneGraphEvaluator.all_modes()\n", - "evaluator_multiple_preds = BasicSceneGraphEvaluator.all_modes(multiple_preds=True)\n", - "evaluator_list = [] # for calculating recall of each relationship except no relationship\n", - "evaluator_multiple_preds_list = []\n", - "for index, name in enumerate(ind_to_predicates):\n", - " if index == 0:\n", - " continue\n", - " evaluator_list.append((index, name, BasicSceneGraphEvaluator.all_modes()))\n", - " evaluator_multiple_preds_list.append((index, name, BasicSceneGraphEvaluator.all_modes(multiple_preds=True)))\n" + "if conf.tb_log_dir is not None:\n", + " from tensorboardX import SummaryWriter\n", + " if not os.path.exists(conf.tb_log_dir):\n", + " os.makedirs(conf.tb_log_dir) \n", + " writer = SummaryWriter(log_dir=conf.tb_log_dir)\n", + " use_tb = True\n", + "else:\n", + " use_tb = False\n" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 26, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'/home/zhanwen/kangaroo_meta/kangaroo_gbnet_all_plus_wikidata_51_with_emb_txt_lrga_bpl_no_sa_1_20220302_mine/ipynb/train_predcls'" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "os.getcwd()" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "scrolled": false + }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch = -1\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 26446/26446 [48:30<00:00, 9.09it/s] \n" + ":9: UserWarning: Not loading optimizer state probably because we have BPL clean param groups\n", + " warn(f'Not loading optimizer state probably because we have BPL clean param groups')\n", + "/home/zhanwen/anaconda3/envs/gbnet/lib/python3.8/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1623448278899/work/c10/core/TensorImpl.h:1156.)\n", + " return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n" ] }, { @@ -295,74 +877,6104 @@ "output_type": "stream", "text": [ "======================predcls recall with constraint============================\n", - "R@20: 0.604273\n", - "R@50: 0.665774\n", - "R@100: 0.682296\n", + "R@20: 0.005394\n", + "R@50: 0.009919\n", + "R@100: 0.014265\n", "======================predcls recall without constraint============================\n", - "R@20: 0.694065\n", - "R@50: 0.836375\n", - "R@100: 0.904512\n", + "R@20: 0.008699\n", + "R@50: 0.027109\n", + "R@100: 0.053832\n", "\n", "\n", "======================predcls mean recall with constraint============================\n", - "mR@20: 0.15308003012923832\n", - "mR@50: 0.1929668659828239\n", - "mR@100: 0.209261631834308\n", + "mR@20: 0.004782483646935009\n", + "mR@50: 0.008622106633350411\n", + "mR@100: 0.011266621473873266\n", "\n", "\n", "======================predcls mean recall without constraint============================\n", - "mR@20: 0.23815979915630467\n", - "mR@50: 0.4105545606850103\n", - "mR@100: 0.5538253106795376\n" + "mR@20: 0.006997515545136262\n", + "mR@50: 0.014854117848638215\n", + "mR@100: 0.055458721230345266\n", + "epoch = 0\n" ] - } - ], - "source": [ - "all_pred_entries = []\n", - "\n", - "if conf.cache is not None and os.path.exists(conf.cache): ########## IMPORTANT ############\n", - " print(\"Found {}! Loading from it\".format(conf.cache))\n", - " with open(conf.cache,'rb') as f:\n", - " all_pred_entries = pickle.load(f)\n", - " for i, pred_entry in enumerate(tqdm(all_pred_entries)):\n", - " gt_entry = {\n", - " 'gt_classes': val.gt_classes[i].copy(),\n", - " 'gt_relations': val.relationships[i].copy(),\n", - " 'gt_boxes': val.gt_boxes[i].copy(),\n", - " }\n", - "\n", - " eval_entry(conf.mode, gt_entry, pred_entry, evaluator, evaluator_multiple_preds, \n", - " evaluator_list, evaluator_multiple_preds_list)\n", - "\n", - " recall = evaluator[conf.mode].print_stats()\n", - " recall_mp = evaluator_multiple_preds[conf.mode].print_stats()\n", - " \n", - " mean_recall = calculate_mR_from_evaluator_list(evaluator_list, conf.mode, save_file=conf.save_rel_recall)\n", - " mean_recall_mp = calculate_mR_from_evaluator_list(evaluator_multiple_preds_list, conf.mode, multiple_preds=True, save_file=conf.save_rel_recall)\n", - "\n", - "else:\n", - " detector.eval()\n", - " for val_b, batch in enumerate(tqdm(val_loader)):\n", - " val_batch(conf.num_gpus*val_b, batch, evaluator, evaluator_multiple_preds, evaluator_list, evaluator_multiple_preds_list)\n", - "\n", - " recall = evaluator[conf.mode].print_stats()\n", - " recall_mp = evaluator_multiple_preds[conf.mode].print_stats()\n", - " \n", - " mean_recall = calculate_mR_from_evaluator_list(evaluator_list, conf.mode, save_file=conf.save_rel_recall)\n", - " mean_recall_mp = calculate_mR_from_evaluator_list(evaluator_multiple_preds_list, conf.mode, multiple_preds=True, save_file=conf.save_rel_recall)\n", - "\n", - " if conf.cache is not None:\n", - " with open(conf.cache,'wb') as f:\n", - " pickle.dump(all_pred_entries, f)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zhanwen/anaconda3/envs/gbnet/lib/python3.8/site-packages/numpy/core/fromnumeric.py:3419: RuntimeWarning: Mean of empty slice.\n", + " return _methods._mean(a, axis=axis, dtype=dtype,\n", + "/home/zhanwen/anaconda3/envs/gbnet/lib/python3.8/site-packages/numpy/core/_methods.py:188: RuntimeWarning: invalid value encountered in double_scalars\n", + " ret = ret.dtype.type(ret / rcount)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---Total norm 64.785 clip coef 0.077-----------------\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 23.124, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 22.510, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 21.852, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 21.777, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 21.488, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 21.026, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.0.weight : 16.098, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 13.052, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 12.080, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.weight : 10.555, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 10.461, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 9.263, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 8.825, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.rel_proj.weight : 7.545, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 6.885, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 5.530, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 4.451, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 4.419, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 4.270, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 3.953, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 3.898, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 3.889, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.weight : 3.848, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 3.328, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 3.257, (torch.Size([200, 1024]))\n", + "roi_fmap_obj.3.weight : 3.218, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 2.325, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 2.191, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 2.156, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 1.990, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 1.972, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 1.960, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 1.909, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 1.647, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 1.647, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 1.622, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 1.521, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 1.521, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 1.461, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 1.060, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 1.057, (torch.Size([4096]))\n", + "ggnn_rel_reason.obj_proj.weight : 1.049, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 1.021, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.873, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.784, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.729, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.647, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.608, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.583, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.557, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.524, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.517, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.440, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.360, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.359, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.351, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.321, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.313, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.297, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.270, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.270, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.253, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.253, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.241, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.223, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.206, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.192, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.159, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.150, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.147, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.145, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.114, (torch.Size([256, 512]))\n", + "roi_fmap.1.0.bias : 0.107, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.103, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.103, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.100, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.095, (torch.Size([512, 1024]))\n", + "roi_fmap_obj.3.bias : 0.093, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.092, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.076, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.069, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.063, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.063, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.063, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.061, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.056, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.052, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.051, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.047, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.046, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.044, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.039, (torch.Size([1024, 3328]))\n", + "roi_fmap_obj.0.bias : 0.034, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.032, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.025, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.019, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.019, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.018, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.018, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.013, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.009, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.001, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e 0b 1000/ 6755 0.235s/batch, 26.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.107421\n", + "total 0.107421\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 0b 2000/ 6755 0.238s/batch, 26.8m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.094886\n", + "total 0.094886\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 0b 3000/ 6755 0.239s/batch, 26.9m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.08686\n", + "total 0.08686\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 0b 4000/ 6755 0.240s/batch, 27.1m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.089419\n", + "total 0.089419\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 0b 5000/ 6755 0.240s/batch, 27.0m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.086891\n", + "total 0.086891\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 0b 6000/ 6755 0.240s/batch, 27.0m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.082879\n", + "total 0.082879\n", + "dtype: float64\n", + "-----------\n", + "overall 0: (0.091)\n", + "class_loss 0.000000\n", + "rel_loss 0.091189\n", + "total 0.091189\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.183004\n", + "R@50: 0.204343\n", + "R@100: 0.209898\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.329669\n", + "R@50: 0.507364\n", + "R@100: 0.669344\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.23542286747133326\n", + "mR@50: 0.2655998967431735\n", + "mR@100: 0.274863421171903\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3227184188903188\n", + "mR@50: 0.446838240749411\n", + "mR@100: 0.5818913050796063\n", + "epoch = 1\n", + "---Total norm 1.634 clip coef 3.061-----------------\n", + "roi_fmap.1.0.weight : 1.036, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.645, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.weight : 0.483, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.392, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.weight : 0.390, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.302, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.299, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.224, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.219, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.203, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.184, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.179, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.168, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.146, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.123, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.122, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.118, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.118, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.116, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.112, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.106, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.105, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.101, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.095, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.094, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.093, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.082, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.070, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.069, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.068, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.063, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.057, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.055, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.055, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.053, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.050, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.045, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.042, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.042, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.042, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.040, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.040, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.038, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.032, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.032, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.031, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.030, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.030, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.029, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.026, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.021, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.021, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.020, (torch.Size([1024]))\n", + "roi_fmap.1.3.bias : 0.018, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.014, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.013, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.012, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.012, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.010, (torch.Size([1024]))\n", + "roi_fmap_obj.3.bias : 0.010, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.009, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.008, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.008, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.007, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.007, (torch.Size([256]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.006, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.006, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.005, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.005, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.005, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.004, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.001, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e 1b 1000/ 6755 0.226s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.080869\n", + "total 0.080869\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 1b 2000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.081311\n", + "total 0.081311\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 1b 3000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.078772\n", + "total 0.078772\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 1b 4000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.079946\n", + "total 0.079946\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 1b 5000/ 6755 0.226s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.077037\n", + "total 0.077037\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 1b 6000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.081267\n", + "total 0.081267\n", + "dtype: float64\n", + "-----------\n", + "overall 1: (0.080)\n", + "class_loss 0.000000\n", + "rel_loss 0.079731\n", + "total 0.079731\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.195003\n", + "R@50: 0.222230\n", + "R@100: 0.228714\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.316113\n", + "R@50: 0.476220\n", + "R@100: 0.637991\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.2616732033819975\n", + "mR@50: 0.3006165078724343\n", + "mR@100: 0.31439084300550624\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.33373209260029973\n", + "mR@50: 0.47586952918774517\n", + "mR@100: 0.6059931769654782\n", + "epoch = 2\n", + "---Total norm 1.667 clip coef 2.999-----------------\n", + "roi_fmap.1.0.weight : 1.052, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.544, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.weight : 0.525, (torch.Size([4096, 4096]))\n", + "roi_fmap_obj.0.weight : 0.514, (torch.Size([4096, 25088]))\n", + "roi_fmap.1.3.weight : 0.351, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.320, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.281, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.231, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.229, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.214, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.214, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.195, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.171, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.166, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.160, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.153, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.143, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.141, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.121, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.117, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.099, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.097, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.091, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.088, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.086, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.078, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.075, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.074, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.074, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.068, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.066, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.066, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.063, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.060, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.060, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.050, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.050, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.050, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.048, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.048, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.046, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.046, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.046, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.044, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.039, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.029, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.029, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.029, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.026, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.025, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.022, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.022, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.021, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.021, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.021, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.020, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.018, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.018, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.018, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.016, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.015, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.015, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.015, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.014, (torch.Size([1024]))\n", + "roi_fmap.1.3.bias : 0.013, (torch.Size([4096]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.013, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.012, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.010, (torch.Size([1024]))\n", + "roi_fmap_obj.3.bias : 0.010, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.009, (torch.Size([256]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.009, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.009, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.007, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.006, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.006, (torch.Size([256]))\n", + "roi_fmap.1.0.bias : 0.006, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.006, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.004, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.004, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.001, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e 2b 1000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.075938\n", + "total 0.075938\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 2b 2000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.07718\n", + "total 0.07718\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 2b 3000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.077804\n", + "total 0.077804\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 2b 4000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.07647\n", + "total 0.07647\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 2b 5000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.079794\n", + "total 0.079794\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 2b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.076439\n", + "total 0.076439\n", + "dtype: float64\n", + "-----------\n", + "overall 2: (0.077)\n", + "class_loss 0.000000\n", + "rel_loss 0.076817\n", + "total 0.076817\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.178999\n", + "R@50: 0.203003\n", + "R@100: 0.210712\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.303829\n", + "R@50: 0.484285\n", + "R@100: 0.651070\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.2676824096517776\n", + "mR@50: 0.30774600552324705\n", + "mR@100: 0.3205355886811942\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3683723725161218\n", + "mR@50: 0.500071981389079\n", + "mR@100: 0.6335343716089021\n", + "epoch = 3\n", + "---Total norm 0.912 clip coef 5.484-----------------\n", + "roi_fmap.1.0.weight : 0.601, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.289, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.271, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.weight : 0.267, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.184, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.weight : 0.170, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.134, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.133, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.121, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.114, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.111, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.109, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.106, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.084, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.082, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.075, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.068, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.067, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.067, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.067, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.059, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.057, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.057, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.054, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.052, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.050, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.047, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.047, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.047, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.043, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.037, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.032, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.032, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.031, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.030, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.030, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.028, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.027, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.026, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.026, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.025, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.023, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.020, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.019, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.017, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.016, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.016, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.016, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.015, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.013, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.011, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.011, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 0.009, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.008, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.bias : 0.007, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.007, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.007, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.005, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.0.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.004, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.bias : 0.003, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.001, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e 3b 1000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.073835\n", + "total 0.073835\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 3b 2000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.072701\n", + "total 0.072701\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 3b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.071623\n", + "total 0.071623\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 3b 4000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.076156\n", + "total 0.076156\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 3b 5000/ 6755 0.227s/batch, 25.6m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.072412\n", + "total 0.072412\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 3b 6000/ 6755 0.230s/batch, 25.8m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.072853\n", + "total 0.072853\n", + "dtype: float64\n", + "-----------\n", + "overall 3: (0.073)\n", + "class_loss 0.000000\n", + "rel_loss 0.073284\n", + "total 0.073284\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.198727\n", + "R@50: 0.226119\n", + "R@100: 0.232919\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.320106\n", + "R@50: 0.492705\n", + "R@100: 0.664905\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.2859606685124138\n", + "mR@50: 0.3214122764488056\n", + "mR@100: 0.3320034059611994\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.36840716684223584\n", + "mR@50: 0.5255559940687957\n", + "mR@100: 0.6482461525813251\n", + "epoch = 4\n", + "---Total norm 0.408 clip coef 12.252-----------------\n", + "roi_fmap.1.0.weight : 0.223, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.210, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.143, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.weight : 0.104, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.088, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.071, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.weight : 0.063, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.043, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.041, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.038, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.038, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.036, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.034, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.034, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.031, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.030, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.028, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.028, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.028, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.028, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.028, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.026, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.026, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.025, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.025, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.025, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.024, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.022, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.022, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.022, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.021, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.019, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.019, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.016, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.015, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.015, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.014, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.013, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.012, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.011, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.010, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.007, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.007, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.005, (torch.Size([512, 1024]))\n", + "roi_fmap.1.3.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.005, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.005, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.004, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "roi_fmap_obj.3.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.002, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.002, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.001, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e 4b 1000/ 6755 0.227s/batch, 25.6m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.070867\n", + "total 0.070867\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 4b 2000/ 6755 0.230s/batch, 25.8m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.073231\n", + "total 0.073231\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 4b 3000/ 6755 0.229s/batch, 25.8m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.071558\n", + "total 0.071558\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 4b 4000/ 6755 0.230s/batch, 25.9m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.072971\n", + "total 0.072971\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 4b 5000/ 6755 0.227s/batch, 25.6m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.069743\n", + "total 0.069743\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 4b 6000/ 6755 0.227s/batch, 25.6m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.069295\n", + "total 0.069295\n", + "dtype: float64\n", + "-----------\n", + "overall 4: (0.071)\n", + "class_loss 0.000000\n", + "rel_loss 0.071314\n", + "total 0.071314\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.195897\n", + "R@50: 0.226337\n", + "R@100: 0.235142\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.321341\n", + "R@50: 0.507726\n", + "R@100: 0.675808\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.2686705759139661\n", + "mR@50: 0.30877246761451416\n", + "mR@100: 0.3202959167233999\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3481037634110605\n", + "mR@50: 0.514636703770587\n", + "mR@100: 0.631616227003494\n", + "epoch = 5\n", + "---Total norm 0.329 clip coef 15.199-----------------\n", + "roi_fmap.1.0.weight : 0.213, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.110, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.103, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.weight : 0.096, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.089, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.weight : 0.061, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.048, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.042, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.037, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.035, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.034, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.031, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.030, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.027, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.027, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.026, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.022, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.022, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.021, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.020, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.020, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.020, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.019, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.018, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.018, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.018, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.016, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.016, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.015, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.014, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.014, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.013, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.012, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.012, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.006, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.005, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.005, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.004, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.004, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.004, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.003, (torch.Size([768]))\n", + "roi_fmap_obj.3.bias : 0.003, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.001, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.001, (torch.Size([256]))\n", + "roi_fmap.1.0.bias : 0.001, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.001, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.001, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.001, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.001, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e 5b 1000/ 6755 0.227s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.068045\n", + "total 0.068045\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 5b 2000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.067565\n", + "total 0.067565\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 5b 3000/ 6755 0.226s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.067465\n", + "total 0.067465\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 5b 4000/ 6755 0.226s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.067975\n", + "total 0.067975\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 5b 5000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.067746\n", + "total 0.067746\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 5b 6000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.07085\n", + "total 0.07085\n", + "dtype: float64\n", + "-----------\n", + "overall 5: (0.069)\n", + "class_loss 0.00000\n", + "rel_loss 0.06851\n", + "total 0.06851\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.202510\n", + "R@50: 0.231492\n", + "R@100: 0.240936\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.333975\n", + "R@50: 0.504549\n", + "R@100: 0.664912\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.2790205119832224\n", + "mR@50: 0.32227798729800883\n", + "mR@100: 0.3382716631303555\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.361516275297452\n", + "mR@50: 0.5084911954462548\n", + "mR@100: 0.6481194170651985\n", + "epoch = 6\n", + "---Total norm 0.543 clip coef 9.202-----------------\n", + "roi_fmap.1.0.weight : 0.424, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.202, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.144, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.126, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.100, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.weight : 0.099, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.042, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.039, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.037, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.037, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.037, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.036, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.033, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.029, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.026, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.026, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.026, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.025, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.023, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.021, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.018, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.015, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.015, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.015, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.014, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.012, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.012, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.010, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.007, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.007, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.007, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.007, (torch.Size([200, 1024]))\n", + "roi_fmap.1.3.bias : 0.006, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.006, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.006, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.006, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.005, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.005, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.004, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.004, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.003, (torch.Size([768]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.002, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.001, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.001, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.001, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e 6b 1000/ 6755 0.226s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.064087\n", + "total 0.064087\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 6b 2000/ 6755 0.226s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.064051\n", + "total 0.064051\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 6b 3000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.069614\n", + "total 0.069614\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 6b 4000/ 6755 0.226s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.064192\n", + "total 0.064192\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 6b 5000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.067555\n", + "total 0.067555\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 6b 6000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.067167\n", + "total 0.067167\n", + "dtype: float64\n", + "-----------\n", + "overall 6: (0.066)\n", + "class_loss 0.000000\n", + "rel_loss 0.066235\n", + "total 0.066235\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.193247\n", + "R@50: 0.219339\n", + "R@100: 0.227395\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.327913\n", + "R@50: 0.500837\n", + "R@100: 0.671181\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.30459781727900437\n", + "mR@50: 0.34843190323188317\n", + "mR@100: 0.3637728817344849\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3734796263445893\n", + "mR@50: 0.5246654354510202\n", + "mR@100: 0.6417092408043729\n", + "epoch = 7\n", + "---Total norm 0.584 clip coef 8.558-----------------\n", + "roi_fmap.1.0.weight : 0.372, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.280, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.202, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.weight : 0.141, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.108, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.104, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.weight : 0.099, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.053, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.052, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.046, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.046, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.046, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.043, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.040, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.040, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.037, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.036, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.031, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.031, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.030, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.030, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.029, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.029, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.025, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.024, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.023, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.021, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.021, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.021, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.020, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.020, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.018, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.017, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.016, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.013, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.011, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.011, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.011, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.010, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.009, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.007, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.006, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.005, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.005, (torch.Size([512, 1024]))\n", + "roi_fmap.1.3.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.004, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.004, (torch.Size([768]))\n", + "roi_fmap_obj.3.bias : 0.003, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.002, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.002, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.001, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.001, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.001, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e 7b 1000/ 6755 0.227s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.060627\n", + "total 0.060627\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 7b 2000/ 6755 0.226s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.064718\n", + "total 0.064718\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 7b 3000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.065931\n", + "total 0.065931\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 7b 4000/ 6755 0.226s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.064068\n", + "total 0.064068\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 7b 5000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.065313\n", + "total 0.065313\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 7b 6000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.066636\n", + "total 0.066636\n", + "dtype: float64\n", + "-----------\n", + "overall 7: (0.065)\n", + "class_loss 0.00000\n", + "rel_loss 0.06457\n", + "total 0.06457\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.184551\n", + "R@50: 0.215107\n", + "R@100: 0.224335\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.313406\n", + "R@50: 0.511105\n", + "R@100: 0.669204\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.30138160454889834\n", + "mR@50: 0.33778014204820955\n", + "mR@100: 0.35567750483816896\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.37697927719998114\n", + "mR@50: 0.5364824352343827\n", + "mR@100: 0.6354692450323519\n", + "epoch = 8\n", + "---Total norm 0.490 clip coef 10.201-----------------\n", + "roi_fmap.1.0.weight : 0.385, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.163, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.126, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.105, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.weight : 0.101, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.083, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.047, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.042, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.041, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.039, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.037, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.035, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.032, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.032, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.031, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.028, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.026, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.023, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.023, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.022, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.022, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.020, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.020, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.019, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.019, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.018, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.018, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.016, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.015, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.014, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.014, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.013, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.012, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.012, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.010, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.008, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.006, (torch.Size([512, 1024]))\n", + "roi_fmap.1.3.bias : 0.006, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.006, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.006, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.005, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.005, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.004, (torch.Size([768]))\n", + "roi_fmap_obj.3.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "roi_fmap.1.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.002, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "roi_fmap_obj.0.bias : 0.001, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e 8b 1000/ 6755 0.227s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.062128\n", + "total 0.062128\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 8b 2000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.061477\n", + "total 0.061477\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 8b 3000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.061259\n", + "total 0.061259\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 8b 4000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.062686\n", + "total 0.062686\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 8b 5000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.064893\n", + "total 0.064893\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 8b 6000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.065024\n", + "total 0.065024\n", + "dtype: float64\n", + "-----------\n", + "overall 8: (0.063)\n", + "class_loss 0.000000\n", + "rel_loss 0.063184\n", + "total 0.063184\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.200956\n", + "R@50: 0.229725\n", + "R@100: 0.238156\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.326165\n", + "R@50: 0.508180\n", + "R@100: 0.673306\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.30104250271011485\n", + "mR@50: 0.3346129344034858\n", + "mR@100: 0.3470616583129354\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3895288066188058\n", + "mR@50: 0.5271771671340155\n", + "mR@100: 0.6401589779079755\n", + "epoch = 9\n", + "---Total norm 0.492 clip coef 10.168-----------------\n", + "roi_fmap.1.0.weight : 0.364, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.191, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.156, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.101, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.095, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.075, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.051, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.044, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.042, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.041, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.037, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.037, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.036, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.034, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.033, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.030, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.027, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.026, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.026, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.024, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.023, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.023, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.022, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.020, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.019, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.018, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.018, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.016, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.013, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.013, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.012, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.011, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.008, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.007, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.007, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.006, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 0.006, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.006, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.006, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.005, (torch.Size([768]))\n", + "roi_fmap_obj.3.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.004, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.004, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.003, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.002, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.002, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e 9b 1000/ 6755 0.226s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.061043\n", + "total 0.061043\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 9b 2000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.060823\n", + "total 0.060823\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 9b 3000/ 6755 0.226s/batch, 25.5m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.059421\n", + "total 0.059421\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 9b 4000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.060136\n", + "total 0.060136\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 9b 5000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.062576\n", + "total 0.062576\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e 9b 6000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.06245\n", + "total 0.06245\n", + "dtype: float64\n", + "-----------\n", + "overall 9: (0.061)\n", + "class_loss 0.00000\n", + "rel_loss 0.06111\n", + "total 0.06111\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.191820\n", + "R@50: 0.217005\n", + "R@100: 0.224889\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.331813\n", + "R@50: 0.521925\n", + "R@100: 0.682943\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3072059653006553\n", + "mR@50: 0.3502115835107918\n", + "mR@100: 0.3607018715226253\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3703151053980723\n", + "mR@50: 0.5549431180048287\n", + "mR@100: 0.6456371177030433\n", + "epoch = 10\n", + "---Total norm 1.984 clip coef 2.520-----------------\n", + "roi_fmap.1.0.weight : 1.642, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.569, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.446, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.406, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.390, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.358, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.170, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.157, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.149, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.146, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.123, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.119, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.113, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.112, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.109, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.102, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.093, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.088, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.086, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.085, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.083, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.081, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.072, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.072, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.067, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.063, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.061, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.052, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.048, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.047, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.045, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.041, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.039, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.039, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.037, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.036, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.034, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.032, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.032, (torch.Size([256, 512]))\n", + "roi_fmap.1.3.bias : 0.030, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.030, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.029, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.029, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.027, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.023, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.021, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.019, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.019, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.019, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.019, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.019, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.017, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.016, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.015, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.015, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.014, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.014, (torch.Size([1024]))\n", + "roi_fmap_obj.3.bias : 0.013, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.012, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.012, (torch.Size([768]))\n", + "roi_fmap.1.0.bias : 0.012, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.010, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.010, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.010, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.010, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.009, (torch.Size([256]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.006, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.006, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.006, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.003, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.001, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e10b 1000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.05552\n", + "total 0.05552\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e10b 2000/ 6755 0.227s/batch, 25.6m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.053406\n", + "total 0.053406\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e10b 3000/ 6755 0.231s/batch, 26.0m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.052444\n", + "total 0.052444\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e10b 4000/ 6755 0.233s/batch, 26.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.054854\n", + "total 0.054854\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e10b 5000/ 6755 0.228s/batch, 25.7m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.051675\n", + "total 0.051675\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e10b 6000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.052944\n", + "total 0.052944\n", + "dtype: float64\n", + "-----------\n", + "overall10: (0.053)\n", + "class_loss 0.000000\n", + "rel_loss 0.053115\n", + "total 0.053115\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.197052\n", + "R@50: 0.225304\n", + "R@100: 0.234602\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.350210\n", + "R@50: 0.534164\n", + "R@100: 0.690270\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3085742297040878\n", + "mR@50: 0.35013978095496845\n", + "mR@100: 0.3641244907663764\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3829927993331084\n", + "mR@50: 0.554200668220999\n", + "mR@100: 0.6581172383606194\n", + "epoch = 11\n", + "---Total norm 0.415 clip coef 12.039-----------------\n", + "roi_fmap.1.0.weight : 0.315, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.124, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.weight : 0.113, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.107, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.weight : 0.095, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.084, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.044, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.039, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.039, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.036, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.033, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.030, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.028, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.026, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.025, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.024, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.024, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.023, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.023, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.021, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.021, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.018, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.018, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.015, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.015, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.015, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.014, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.009, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.009, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.009, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.008, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.008, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.007, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.007, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "roi_fmap.1.3.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.004, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.004, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.004, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.003, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.003, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.003, (torch.Size([256, 512]))\n", + "roi_fmap_obj.3.bias : 0.003, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.002, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.002, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.001, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.001, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.bias : 0.001, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e11b 1000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.049583\n", + "total 0.049583\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e11b 2000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.050477\n", + "total 0.050477\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e11b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.051592\n", + "total 0.051592\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e11b 4000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.049761\n", + "total 0.049761\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e11b 5000/ 6755 0.224s/batch, 25.2m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.051951\n", + "total 0.051951\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e11b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.050943\n", + "total 0.050943\n", + "dtype: float64\n", + "-----------\n", + "overall11: (0.051)\n", + "class_loss 0.000000\n", + "rel_loss 0.050583\n", + "total 0.050583\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.199702\n", + "R@50: 0.228238\n", + "R@100: 0.236961\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.353147\n", + "R@50: 0.542437\n", + "R@100: 0.696325\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3064065564745567\n", + "mR@50: 0.3508870154503547\n", + "mR@100: 0.36323454576710307\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.4058199431855597\n", + "mR@50: 0.5429651890583889\n", + "mR@100: 0.6554476388132646\n", + "epoch = 12\n", + "---Total norm 1.260 clip coef 3.967-----------------\n", + "roi_fmap.1.0.weight : 0.991, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.429, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.287, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.279, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.218, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.193, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.156, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.149, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.133, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.100, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.095, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.095, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.091, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.089, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.086, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.080, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.069, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.068, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.067, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.066, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.065, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.064, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.053, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.051, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.051, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.050, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.040, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.039, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.035, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.035, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.034, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.029, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.029, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.028, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.028, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.027, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.027, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.026, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.025, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.023, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.022, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.021, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.021, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.021, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.018, (torch.Size([1024]))\n", + "roi_fmap.1.3.bias : 0.017, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.015, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.015, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.014, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.011, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.011, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.011, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.010, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.009, (torch.Size([512, 1024]))\n", + "roi_fmap_obj.3.bias : 0.009, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.008, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.008, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.008, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.007, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.007, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.007, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.005, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.005, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.005, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "roi_fmap_obj.0.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.003, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.001, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e12b 1000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.050183\n", + "total 0.050183\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e12b 2000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.048989\n", + "total 0.048989\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e12b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.048642\n", + "total 0.048642\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e12b 4000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.049273\n", + "total 0.049273\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e12b 5000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.048283\n", + "total 0.048283\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e12b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.048367\n", + "total 0.048367\n", + "dtype: float64\n", + "-----------\n", + "overall12: (0.049)\n", + "class_loss 0.000000\n", + "rel_loss 0.048999\n", + "total 0.048999\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.195848\n", + "R@50: 0.223896\n", + "R@100: 0.232746\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.350285\n", + "R@50: 0.532306\n", + "R@100: 0.684590\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3093381696006558\n", + "mR@50: 0.3477204198884192\n", + "mR@100: 0.3612865425905209\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.38668309986691435\n", + "mR@50: 0.5408069968178547\n", + "mR@100: 0.652044903556654\n", + "epoch = 13\n", + "---Total norm 1.024 clip coef 4.881-----------------\n", + "roi_fmap.1.0.weight : 0.879, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.263, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.233, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.194, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.174, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.126, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.097, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.083, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.082, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.072, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.061, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.059, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.057, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.055, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.055, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.047, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.047, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.045, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.045, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.042, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.034, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.034, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.033, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.031, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.030, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.030, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.027, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.027, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.024, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.024, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.023, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.023, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.023, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.022, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.021, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.018, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.018, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.017, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.016, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.014, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.013, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.011, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.011, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.010, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 0.010, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.008, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.008, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.006, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.006, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.006, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.bias : 0.006, (torch.Size([4096]))\n", + "roi_fmap.1.0.bias : 0.006, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.005, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.005, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.003, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "roi_fmap_obj.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e13b 1000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.047338\n", + "total 0.047338\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e13b 2000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.04756\n", + "total 0.04756\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e13b 3000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.047114\n", + "total 0.047114\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e13b 4000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.048414\n", + "total 0.048414\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e13b 5000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.047433\n", + "total 0.047433\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e13b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.048645\n", + "total 0.048645\n", + "dtype: float64\n", + "-----------\n", + "overall13: (0.048)\n", + "class_loss 0.000000\n", + "rel_loss 0.047729\n", + "total 0.047729\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.195300\n", + "R@50: 0.222556\n", + "R@100: 0.230692\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.352607\n", + "R@50: 0.541020\n", + "R@100: 0.688312\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.30998140218262926\n", + "mR@50: 0.3511723500283277\n", + "mR@100: 0.3644061630074685\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.40194382851904203\n", + "mR@50: 0.535649196400075\n", + "mR@100: 0.6457788522916273\n", + "epoch = 14\n", + "---Total norm 1.870 clip coef 2.674-----------------\n", + "roi_fmap.1.0.weight : 1.570, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.480, (torch.Size([4096, 25088]))\n", + "roi_fmap.1.3.weight : 0.389, (torch.Size([4096, 4096]))\n", + "roi_fmap_obj.3.weight : 0.368, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.353, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.298, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.196, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.183, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.169, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.141, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.131, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.128, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.120, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.117, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.117, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.104, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.091, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.087, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.087, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.082, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.071, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.069, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.067, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.064, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.064, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.061, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.058, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.053, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.047, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.046, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.046, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.045, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.045, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.039, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.038, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.037, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.033, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.033, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.032, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.031, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.031, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.029, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.028, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.025, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.023, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.023, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 0.022, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.020, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.019, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.019, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.018, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.018, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.018, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.015, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.015, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.014, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.012, (torch.Size([256, 512]))\n", + "roi_fmap_obj.3.bias : 0.011, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.010, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.010, (torch.Size([256]))\n", + "roi_fmap.1.0.bias : 0.010, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.008, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.008, (torch.Size([512]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.008, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.007, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.006, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.005, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.005, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.005, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.001, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e14b 1000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.04755\n", + "total 0.04755\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e14b 2000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.046169\n", + "total 0.046169\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e14b 3000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.045728\n", + "total 0.045728\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e14b 4000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.045474\n", + "total 0.045474\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e14b 5000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.04859\n", + "total 0.04859\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e14b 6000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.046138\n", + "total 0.046138\n", + "dtype: float64\n", + "-----------\n", + "overall14: (0.047)\n", + "class_loss 0.000000\n", + "rel_loss 0.046551\n", + "total 0.046551\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.193513\n", + "R@50: 0.222970\n", + "R@100: 0.232254\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.350967\n", + "R@50: 0.541173\n", + "R@100: 0.690072\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3085122837133861\n", + "mR@50: 0.34700750451777085\n", + "mR@100: 0.36242392459843886\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.40511447911355014\n", + "mR@50: 0.5340601576766166\n", + "mR@100: 0.6465141357263455\n", + "epoch = 15\n", + "---Total norm 1.237 clip coef 4.042-----------------\n", + "roi_fmap.1.0.weight : 1.003, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.316, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.266, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.247, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.202, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.193, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.192, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.175, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.116, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.112, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.109, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.106, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.101, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.097, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.085, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.078, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.076, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.074, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.072, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.071, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.070, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.069, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.066, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.057, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.052, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.048, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.040, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.039, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.037, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.035, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.033, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.032, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.029, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.028, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.028, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.027, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.027, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.026, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.026, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.025, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.023, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.022, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.022, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.020, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.020, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.019, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.019, (torch.Size([1024]))\n", + "roi_fmap.1.3.bias : 0.018, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.015, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.015, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.014, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.013, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.013, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.008, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.008, (torch.Size([256]))\n", + "roi_fmap_obj.3.bias : 0.008, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.008, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.008, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.008, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.008, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.007, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.007, (torch.Size([4096]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.005, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.005, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.003, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.003, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e15b 1000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.046449\n", + "total 0.046449\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e15b 2000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.046308\n", + "total 0.046308\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e15b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.045308\n", + "total 0.045308\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e15b 4000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.046164\n", + "total 0.046164\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e15b 5000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.044933\n", + "total 0.044933\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e15b 6000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.045295\n", + "total 0.045295\n", + "dtype: float64\n", + "-----------\n", + "overall15: (0.046)\n", + "class_loss 0.000000\n", + "rel_loss 0.045785\n", + "total 0.045785\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.187999\n", + "R@50: 0.216099\n", + "R@100: 0.225472\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.351610\n", + "R@50: 0.540365\n", + "R@100: 0.688314\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.31491863690768385\n", + "mR@50: 0.3522749206280706\n", + "mR@100: 0.36777033483499666\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3996963039475968\n", + "mR@50: 0.5431906416546305\n", + "mR@100: 0.6458552181643058\n", + "epoch = 16\n", + "---Total norm 1.221 clip coef 4.094-----------------\n", + "roi_fmap.1.0.weight : 1.036, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.436, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.249, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.249, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.104, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.101, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.097, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.096, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.085, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.073, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.066, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.064, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.063, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.063, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.059, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.055, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.050, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.049, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.048, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.047, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.046, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.046, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.044, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.042, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.040, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.026, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.025, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.024, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.022, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.020, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 0.020, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.020, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.020, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.019, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.019, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.018, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.017, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.016, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.015, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.015, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.014, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.013, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.010, (torch.Size([1024]))\n", + "roi_fmap_obj.3.bias : 0.010, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.009, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.008, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.007, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.007, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.007, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.007, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.007, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.006, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.006, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.005, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.005, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.003, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e16b 1000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.045976\n", + "total 0.045976\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e16b 2000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.043604\n", + "total 0.043604\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e16b 3000/ 6755 0.224s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.044989\n", + "total 0.044989\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e16b 4000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.045236\n", + "total 0.045236\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e16b 5000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.04372\n", + "total 0.04372\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e16b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.044628\n", + "total 0.044628\n", + "dtype: float64\n", + "-----------\n", + "overall16: (0.045)\n", + "class_loss 0.000000\n", + "rel_loss 0.044762\n", + "total 0.044762\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.189845\n", + "R@50: 0.218969\n", + "R@100: 0.228518\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.347743\n", + "R@50: 0.532871\n", + "R@100: 0.676962\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3067744242796454\n", + "mR@50: 0.3459427946151542\n", + "mR@100: 0.3623585343445527\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.4019244968576783\n", + "mR@50: 0.5405919552781306\n", + "mR@100: 0.6499979844346115\n", + "epoch = 17\n", + "---Total norm 0.533 clip coef 9.372-----------------\n", + "roi_fmap.1.0.weight : 0.444, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.175, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.125, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.101, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.068, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.066, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.051, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.042, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.038, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.036, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.034, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.033, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.033, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.030, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.028, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.028, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.027, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.026, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.025, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.024, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.024, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.022, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.021, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.021, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.021, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.020, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.019, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.015, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.015, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.013, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.012, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.011, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.011, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 0.010, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.010, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.010, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.009, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.008, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.008, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.007, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.006, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.005, (torch.Size([1024]))\n", + "roi_fmap_obj.3.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.004, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "roi_fmap.1.0.bias : 0.003, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.003, (torch.Size([2048]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "roi_fmap_obj.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.001, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.001, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.001, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e17b 1000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.044517\n", + "total 0.044517\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e17b 2000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.043032\n", + "total 0.043032\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e17b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.044535\n", + "total 0.044535\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e17b 4000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.043426\n", + "total 0.043426\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e17b 5000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.044001\n", + "total 0.044001\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e17b 6000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.044957\n", + "total 0.044957\n", + "dtype: float64\n", + "-----------\n", + "overall17: (0.044)\n", + "class_loss 0.000000\n", + "rel_loss 0.044243\n", + "total 0.044243\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.187389\n", + "R@50: 0.216599\n", + "R@100: 0.226153\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.341430\n", + "R@50: 0.526185\n", + "R@100: 0.670433\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3105247383592557\n", + "mR@50: 0.3498494651570622\n", + "mR@100: 0.3673888791220209\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.39915858437218715\n", + "mR@50: 0.5381863645415207\n", + "mR@100: 0.647888123993504\n", + "epoch = 18\n", + "---Total norm 1.087 clip coef 4.601-----------------\n", + "roi_fmap.1.0.weight : 0.799, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.271, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.240, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.240, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.weight : 0.219, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.211, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.166, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.156, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.123, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.110, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.109, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.104, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.103, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.100, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.100, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.099, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.098, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.095, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.089, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.087, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.086, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.082, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.082, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.080, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.072, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.067, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.066, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.065, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.063, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.062, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.062, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.053, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.048, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.046, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.046, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.044, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.043, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.043, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.034, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.029, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.029, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.028, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.027, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.025, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.025, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.024, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.022, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.022, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.020, (torch.Size([200, 1024]))\n", + "roi_fmap.1.3.bias : 0.019, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.015, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.015, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.015, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.015, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.013, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.013, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.012, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.012, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.008, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.008, (torch.Size([512]))\n", + "roi_fmap_obj.3.bias : 0.008, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.007, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.007, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.006, (torch.Size([512]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.006, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.005, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.005, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.004, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.bias : 0.003, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.001, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e18b 1000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.042632\n", + "total 0.042632\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e18b 2000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.04366\n", + "total 0.04366\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e18b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.043438\n", + "total 0.043438\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e18b 4000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.042765\n", + "total 0.042765\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e18b 5000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.042672\n", + "total 0.042672\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e18b 6000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.041884\n", + "total 0.041884\n", + "dtype: float64\n", + "-----------\n", + "overall18: (0.043)\n", + "class_loss 0.000000\n", + "rel_loss 0.043019\n", + "total 0.043019\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.188973\n", + "R@50: 0.218433\n", + "R@100: 0.228342\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.344222\n", + "R@50: 0.525849\n", + "R@100: 0.669288\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3048694864534186\n", + "mR@50: 0.3487174486469945\n", + "mR@100: 0.3643151077003882\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.39126444896511586\n", + "mR@50: 0.5398354230914818\n", + "mR@100: 0.6319856638022658\n", + "epoch = 19\n", + "---Total norm 0.896 clip coef 5.583-----------------\n", + "roi_fmap.1.0.weight : 0.652, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.249, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.weight : 0.240, (torch.Size([4096, 25088]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.203, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.weight : 0.187, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.176, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.120, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.110, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.105, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.095, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.094, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.086, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.083, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.082, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.077, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.074, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.073, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.067, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.066, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.058, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.057, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.056, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.055, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.054, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.054, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.052, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.052, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.050, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.050, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.049, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.039, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.038, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.030, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.030, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.029, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.028, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.027, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.027, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.027, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.026, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.026, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.026, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.025, (torch.Size([200, 1024]))\n", + "roi_fmap.1.3.bias : 0.021, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.020, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.019, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.019, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.017, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.016, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.015, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.014, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.011, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.011, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.008, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.008, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.008, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.008, (torch.Size([1024]))\n", + "roi_fmap_obj.3.bias : 0.007, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.006, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.006, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.005, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.003, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "roi_fmap_obj.0.bias : 0.003, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.001, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e19b 1000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.042847\n", + "total 0.042847\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e19b 2000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.041947\n", + "total 0.041947\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e19b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040517\n", + "total 0.040517\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e19b 4000/ 6755 0.224s/batch, 25.2m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.041777\n", + "total 0.041777\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e19b 5000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.043803\n", + "total 0.043803\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e19b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.042018\n", + "total 0.042018\n", + "dtype: float64\n", + "-----------\n", + "overall19: (0.042)\n", + "class_loss 0.00000\n", + "rel_loss 0.04224\n", + "total 0.04224\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.188744\n", + "R@50: 0.218819\n", + "R@100: 0.228758\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.340204\n", + "R@50: 0.518125\n", + "R@100: 0.660757\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.30844391920343023\n", + "mR@50: 0.3506591740185187\n", + "mR@100: 0.36289269828944837\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.4005997125570492\n", + "mR@50: 0.5367777122907363\n", + "mR@100: 0.6376446063195761\n", + "epoch = 20\n", + "---Total norm 1.822 clip coef 2.745-----------------\n", + "roi_fmap.1.0.weight : 1.452, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.761, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.518, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.343, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.158, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.147, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.147, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.145, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.129, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.129, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.129, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.109, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.091, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.091, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.084, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.083, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.083, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.078, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.077, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.076, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.072, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.061, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.060, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.059, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.053, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.047, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.038, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.036, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.035, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.035, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.030, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.027, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 0.026, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.026, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.025, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.024, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.023, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.023, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.021, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.020, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.020, (torch.Size([1024]))\n", + "roi_fmap_obj.3.bias : 0.018, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.017, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.014, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.014, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.014, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.013, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.012, (torch.Size([512]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.012, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.009, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.009, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.009, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.008, (torch.Size([512, 1024]))\n", + "roi_fmap_obj.0.bias : 0.007, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.007, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.007, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.006, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.004, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e20b 1000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.041576\n", + "total 0.041576\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e20b 2000/ 6755 0.226s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040083\n", + "total 0.040083\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e20b 3000/ 6755 0.224s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.041955\n", + "total 0.041955\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e20b 4000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040215\n", + "total 0.040215\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e20b 5000/ 6755 0.224s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.041572\n", + "total 0.041572\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e20b 6000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039801\n", + "total 0.039801\n", + "dtype: float64\n", + "-----------\n", + "overall20: (0.041)\n", + "class_loss 0.000000\n", + "rel_loss 0.040795\n", + "total 0.040795\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.189414\n", + "R@50: 0.218770\n", + "R@100: 0.228259\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.345974\n", + "R@50: 0.528893\n", + "R@100: 0.671033\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3088529935952028\n", + "mR@50: 0.35194737261589687\n", + "mR@100: 0.36372562952380355\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3925584174341289\n", + "mR@50: 0.5356058037280429\n", + "mR@100: 0.6371567192685998\n", + "epoch = 21\n", + "---Total norm 2.705 clip coef 1.848-----------------\n", + "roi_fmap.1.0.weight : 2.469, (torch.Size([4096, 25088]))\n", + "roi_fmap.1.3.weight : 0.554, (torch.Size([4096, 4096]))\n", + "roi_fmap_obj.0.weight : 0.535, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.467, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.222, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.216, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.204, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.183, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.164, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.150, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.149, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.145, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.136, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.130, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.118, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.109, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.107, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.095, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.092, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.089, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.066, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.065, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.057, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.057, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.055, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.053, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.052, (torch.Size([256, 512]))\n", + "roi_fmap.1.3.bias : 0.047, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.042, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.042, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.037, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.036, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.036, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.034, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.034, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.034, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.033, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.030, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.029, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.029, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.029, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.028, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.027, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.026, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.025, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.022, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.021, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.020, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.019, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.019, (torch.Size([3328, 3328]))\n", + "roi_fmap.1.0.bias : 0.018, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.016, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.014, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.013, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.013, (torch.Size([1024]))\n", + "roi_fmap_obj.3.bias : 0.013, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.010, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.010, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.009, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.008, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.008, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.008, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.006, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.006, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.005, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.004, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e21b 1000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040918\n", + "total 0.040918\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e21b 2000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040236\n", + "total 0.040236\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e21b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040167\n", + "total 0.040167\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e21b 4000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040381\n", + "total 0.040381\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e21b 5000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040465\n", + "total 0.040465\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e21b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.042071\n", + "total 0.042071\n", + "dtype: float64\n", + "-----------\n", + "overall21: (0.041)\n", + "class_loss 0.000000\n", + "rel_loss 0.040563\n", + "total 0.040563\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.189425\n", + "R@50: 0.219193\n", + "R@100: 0.228860\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.345753\n", + "R@50: 0.527803\n", + "R@100: 0.670545\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.30849370224784084\n", + "mR@50: 0.35087980755791337\n", + "mR@100: 0.362405449187757\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3979935185878457\n", + "mR@50: 0.5350260864082671\n", + "mR@100: 0.6391599340325883\n", + "epoch = 22\n", + "---Total norm 1.563 clip coef 3.199-----------------\n", + "roi_fmap.1.0.weight : 1.310, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.384, (torch.Size([4096, 25088]))\n", + "roi_fmap.1.3.weight : 0.339, (torch.Size([4096, 4096]))\n", + "roi_fmap_obj.3.weight : 0.323, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.265, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.226, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.156, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.145, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.131, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.130, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.104, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.100, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.097, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.095, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.094, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.092, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.092, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.085, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.083, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.082, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.080, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.077, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.076, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.075, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.074, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.073, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.070, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.059, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.052, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.050, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.048, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.041, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.039, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.039, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.037, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.037, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.036, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.035, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.031, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.030, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.030, (torch.Size([1024]))\n", + "roi_fmap.1.3.bias : 0.030, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.029, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.027, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.024, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.023, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.022, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.021, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.021, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.021, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.021, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.020, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.020, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.019, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.018, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.017, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.015, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.015, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.014, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.012, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.010, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.010, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.009, (torch.Size([4096]))\n", + "roi_fmap_obj.3.bias : 0.009, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.008, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.008, (torch.Size([512]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.008, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.008, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.007, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.005, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.004, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.003, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e22b 1000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.04045\n", + "total 0.04045\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e22b 2000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040127\n", + "total 0.040127\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e22b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040134\n", + "total 0.040134\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e22b 4000/ 6755 0.224s/batch, 25.2m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040471\n", + "total 0.040471\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e22b 5000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.04045\n", + "total 0.04045\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e22b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.03983\n", + "total 0.03983\n", + "dtype: float64\n", + "-----------\n", + "overall22: (0.040)\n", + "class_loss 0.00000\n", + "rel_loss 0.04023\n", + "total 0.04023\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.189724\n", + "R@50: 0.219846\n", + "R@100: 0.229644\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.342556\n", + "R@50: 0.524205\n", + "R@100: 0.668122\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.308117875347586\n", + "mR@50: 0.35150952671755253\n", + "mR@100: 0.3628519523005492\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.391214130207529\n", + "mR@50: 0.53470301642616\n", + "mR@100: 0.6383348083540265\n", + "epoch = 23\n", + "---Total norm 1.388 clip coef 3.602-----------------\n", + "roi_fmap.1.0.weight : 1.196, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.466, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.299, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.270, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.125, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.109, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.102, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.093, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.086, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.081, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.077, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.077, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.072, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.066, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.061, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.056, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.054, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.053, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.050, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.049, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.041, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.040, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.037, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.035, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.034, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.032, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.027, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.026, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.026, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.024, (torch.Size([1024, 300]))\n", + "roi_fmap.1.3.bias : 0.022, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.022, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.020, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.020, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.016, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.015, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.015, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.013, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.013, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.012, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.012, (torch.Size([200, 1024]))\n", + "roi_fmap_obj.3.bias : 0.012, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.009, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.009, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.009, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.008, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.008, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.007, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.007, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.006, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.005, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.004, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.003, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.003, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.003, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.001, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e23b 1000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039772\n", + "total 0.039772\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e23b 2000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.041182\n", + "total 0.041182\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e23b 3000/ 6755 0.224s/batch, 25.2m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040521\n", + "total 0.040521\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e23b 4000/ 6755 0.224s/batch, 25.2m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040109\n", + "total 0.040109\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e23b 5000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.038886\n", + "total 0.038886\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e23b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040991\n", + "total 0.040991\n", + "dtype: float64\n", + "-----------\n", + "overall23: (0.040)\n", + "class_loss 0.000000\n", + "rel_loss 0.040132\n", + "total 0.040132\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.187757\n", + "R@50: 0.217104\n", + "R@100: 0.227148\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.345160\n", + "R@50: 0.527184\n", + "R@100: 0.670765\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.31067217813440634\n", + "mR@50: 0.3529682844522163\n", + "mR@100: 0.36525160783088817\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.39044807590370184\n", + "mR@50: 0.5338693734153688\n", + "mR@100: 0.6352568248851389\n", + "epoch = 24\n", + "---Total norm 1.030 clip coef 4.856-----------------\n", + "roi_fmap.1.0.weight : 0.870, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.371, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.212, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.185, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.107, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.088, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.086, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.083, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.074, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.068, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.059, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.056, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.053, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.053, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.052, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.051, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.050, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.045, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.042, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.042, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.039, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.038, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.038, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.036, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.035, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.035, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.031, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.026, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.025, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.024, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.024, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.020, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.020, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.020, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.019, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.017, (torch.Size([1024, 300]))\n", + "roi_fmap.1.3.bias : 0.016, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.015, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.015, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.013, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.013, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.009, (torch.Size([200, 1024]))\n", + "roi_fmap_obj.3.bias : 0.009, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.008, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.007, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.007, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.006, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.006, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.006, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.005, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.005, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.005, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.005, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.003, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.bias : 0.003, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.002, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e24b 1000/ 6755 0.224s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040073\n", + "total 0.040073\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e24b 2000/ 6755 0.224s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.038863\n", + "total 0.038863\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e24b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039204\n", + "total 0.039204\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e24b 4000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.038853\n", + "total 0.038853\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e24b 5000/ 6755 0.224s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040813\n", + "total 0.040813\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e24b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040691\n", + "total 0.040691\n", + "dtype: float64\n", + "-----------\n", + "overall24: (0.040)\n", + "class_loss 0.000000\n", + "rel_loss 0.039871\n", + "total 0.039871\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.188579\n", + "R@50: 0.218217\n", + "R@100: 0.228310\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.344169\n", + "R@50: 0.527543\n", + "R@100: 0.670763\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3092716028427643\n", + "mR@50: 0.352800910062875\n", + "mR@100: 0.36549435540095454\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.39216256345827183\n", + "mR@50: 0.5342247259601918\n", + "mR@100: 0.6361113138436463\n", + "epoch = 25\n", + "---Total norm 0.819 clip coef 6.102-----------------\n", + "roi_fmap.1.0.weight : 0.674, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.231, (torch.Size([4096, 25088]))\n", + "roi_fmap.1.3.weight : 0.171, (torch.Size([4096, 4096]))\n", + "roi_fmap_obj.3.weight : 0.170, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.131, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.115, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.078, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.078, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.074, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.072, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.068, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.067, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.065, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.057, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.056, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.054, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.053, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.052, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.051, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.049, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.044, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.041, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.039, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.039, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.039, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.038, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.036, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.036, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 0.025, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.023, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.023, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.023, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.021, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.019, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.018, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.016, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.016, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.015, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.014, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.014, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.012, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.010, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.009, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.007, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.007, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.006, (torch.Size([768]))\n", + "roi_fmap_obj.3.bias : 0.006, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.005, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.005, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.005, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.005, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.005, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.003, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.002, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e25b 1000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.03855\n", + "total 0.03855\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e25b 2000/ 6755 0.224s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.038282\n", + "total 0.038282\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e25b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.038994\n", + "total 0.038994\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e25b 4000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040398\n", + "total 0.040398\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e25b 5000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040135\n", + "total 0.040135\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e25b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.041342\n", + "total 0.041342\n", + "dtype: float64\n", + "-----------\n", + "overall25: (0.040)\n", + "class_loss 0.000000\n", + "rel_loss 0.039628\n", + "total 0.039628\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.190008\n", + "R@50: 0.220219\n", + "R@100: 0.229901\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.343565\n", + "R@50: 0.528922\n", + "R@100: 0.671455\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.30774330989467263\n", + "mR@50: 0.35213345507230026\n", + "mR@100: 0.3652595363860401\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.38815394279604143\n", + "mR@50: 0.5370097201816937\n", + "mR@100: 0.6373504406041784\n", + "epoch = 26\n", + "---Total norm 1.205 clip coef 4.150-----------------\n", + "roi_fmap.1.0.weight : 1.010, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.375, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.294, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.258, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.114, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.111, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.110, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.106, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.103, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.102, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.077, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.074, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.073, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.070, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.070, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.063, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.061, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.060, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.058, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.057, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.057, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.048, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.048, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.047, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.043, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.033, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.031, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.028, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.027, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.027, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.026, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.024, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.024, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.023, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.023, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.022, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.022, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.019, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.017, (torch.Size([1024, 2048]))\n", + "roi_fmap.1.3.bias : 0.016, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.016, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.013, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.011, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.009, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.bias : 0.009, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.008, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.007, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.007, (torch.Size([512]))\n", + "roi_fmap.1.0.bias : 0.007, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.007, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.007, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.005, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.005, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.005, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.004, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.003, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.003, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e26b 1000/ 6755 0.224s/batch, 25.3m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.03931\n", + "total 0.03931\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e26b 2000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040438\n", + "total 0.040438\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e26b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039698\n", + "total 0.039698\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e26b 4000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039406\n", + "total 0.039406\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e26b 5000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.03973\n", + "total 0.03973\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e26b 6000/ 6755 0.224s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.041256\n", + "total 0.041256\n", + "dtype: float64\n", + "-----------\n", + "overall26: (0.040)\n", + "class_loss 0.00000\n", + "rel_loss 0.03986\n", + "total 0.03986\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.188570\n", + "R@50: 0.218219\n", + "R@100: 0.228098\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.341252\n", + "R@50: 0.523290\n", + "R@100: 0.666787\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.30412055302859314\n", + "mR@50: 0.35086531543351773\n", + "mR@100: 0.363248506264286\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.39423302061890253\n", + "mR@50: 0.5357649775835921\n", + "mR@100: 0.6364392413751218\n", + "epoch = 27\n", + "---Total norm 1.159 clip coef 4.313-----------------\n", + "roi_fmap.1.0.weight : 0.938, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.325, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.271, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.260, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.164, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.162, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.138, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.130, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.115, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.101, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.096, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.087, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.081, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.080, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.080, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.075, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.073, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.072, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.067, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.066, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.066, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.065, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.065, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.065, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.062, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.055, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.055, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.039, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.034, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.034, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.034, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.033, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.029, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.028, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.027, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.025, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.025, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.024, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.024, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.022, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.022, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.022, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 0.019, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.018, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.018, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.018, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.016, (torch.Size([1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.014, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.014, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.013, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.011, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.011, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.010, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.010, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.008, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.008, (torch.Size([1024]))\n", + "roi_fmap_obj.3.bias : 0.008, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.008, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.008, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.007, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.007, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.007, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.007, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.006, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.0.bias : 0.006, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.005, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.004, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "roi_fmap_obj.0.bias : 0.003, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.001, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e27b 1000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040142\n", + "total 0.040142\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e27b 2000/ 6755 0.224s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.038954\n", + "total 0.038954\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e27b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.00000\n", + "rel_loss 0.03933\n", + "total 0.03933\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e27b 4000/ 6755 0.225s/batch, 25.4m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039953\n", + "total 0.039953\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e27b 5000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040201\n", + "total 0.040201\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e27b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039644\n", + "total 0.039644\n", + "dtype: float64\n", + "-----------\n", + "overall27: (0.040)\n", + "class_loss 0.000000\n", + "rel_loss 0.039576\n", + "total 0.039576\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.189062\n", + "R@50: 0.218494\n", + "R@100: 0.228144\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.345705\n", + "R@50: 0.529517\n", + "R@100: 0.671322\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3095317803248597\n", + "mR@50: 0.3530858069726655\n", + "mR@100: 0.36652486852862004\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3939107557452183\n", + "mR@50: 0.5347699863215479\n", + "mR@100: 0.637561072830523\n", + "epoch = 28\n", + "---Total norm 1.468 clip coef 3.405-----------------\n", + "roi_fmap.1.0.weight : 1.277, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.402, (torch.Size([4096, 25088]))\n", + "roi_fmap.1.3.weight : 0.316, (torch.Size([4096, 4096]))\n", + "roi_fmap_obj.3.weight : 0.303, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.146, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.140, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.127, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.124, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.115, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.095, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.088, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.083, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.079, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.078, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.077, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.076, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.069, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.059, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.057, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.050, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.048, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.047, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.044, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.043, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.041, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.039, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.037, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.028, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.027, (torch.Size([1024, 1024]))\n", + "roi_fmap.1.3.bias : 0.024, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.024, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.024, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.023, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.022, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.022, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.021, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.021, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.019, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.019, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.018, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.018, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.018, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.018, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.017, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.015, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.015, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.014, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.014, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.013, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.012, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.012, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.011, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.011, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.010, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.009, (torch.Size([768]))\n", + "roi_fmap.1.0.bias : 0.009, (torch.Size([4096]))\n", + "roi_fmap_obj.3.bias : 0.009, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.008, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.006, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.006, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.006, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.006, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.006, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.005, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.005, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.005, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.004, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.004, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.0.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.003, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.003, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.003, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e28b 1000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.038954\n", + "total 0.038954\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e28b 2000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039388\n", + "total 0.039388\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e28b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.038888\n", + "total 0.038888\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e28b 4000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039188\n", + "total 0.039188\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e28b 5000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.041293\n", + "total 0.041293\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e28b 6000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039867\n", + "total 0.039867\n", + "dtype: float64\n", + "-----------\n", + "overall28: (0.039)\n", + "class_loss 0.000000\n", + "rel_loss 0.039461\n", + "total 0.039461\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.187826\n", + "R@50: 0.217791\n", + "R@100: 0.227923\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.345836\n", + "R@50: 0.526836\n", + "R@100: 0.669470\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.3091854275018857\n", + "mR@50: 0.35196888722221414\n", + "mR@100: 0.3653506966900651\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3910016404847451\n", + "mR@50: 0.5368046893974812\n", + "mR@100: 0.6381624208782485\n", + "epoch = 29\n", + "---Total norm 0.683 clip coef 7.326-----------------\n", + "roi_fmap.1.0.weight : 0.572, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.0.weight : 0.202, (torch.Size([4096, 25088]))\n", + "roi_fmap_obj.3.weight : 0.141, (torch.Size([4096, 4096]))\n", + "roi_fmap.1.3.weight : 0.136, (torch.Size([4096, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.weight: 0.087, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.weight: 0.086, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.weight: 0.072, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.weight: 0.066, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.weight : 0.066, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.weight : 0.056, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.weight : 0.045, (torch.Size([1024, 1124]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.weight : 0.044, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.weight: 0.043, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.weight: 0.043, (torch.Size([768, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.weight: 0.042, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.obj_proj.weight : 0.041, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.rel_proj.weight : 0.038, (torch.Size([1024, 4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.weight : 0.036, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.weight: 0.036, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.weight : 0.036, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.weight: 0.035, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.weight: 0.032, (torch.Size([1024, 768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.weight: 0.031, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.weight: 0.031, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.weight : 0.028, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.weight : 0.021, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.weight : 0.020, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.weight : 0.019, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.weight : 0.019, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.weight : 0.018, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.0.0.bias : 0.017, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.weight: 0.016, (torch.Size([2048, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.weight : 0.016, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.weight: 0.014, (torch.Size([3328, 3328]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.weight: 0.014, (torch.Size([1024, 2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.weight: 0.014, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.weight: 0.013, (torch.Size([512, 1024]))\n", + "roi_fmap.1.3.bias : 0.013, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.weight : 0.013, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.weight : 0.012, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.0.linear.bias: 0.011, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.weight: 0.010, (torch.Size([1024, 3328]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.weight : 0.009, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.rel_proj.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_pred.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_pred.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.weight : 0.009, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_pred.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_pred.bias : 0.009, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.weight : 0.008, (torch.Size([1024, 300]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.2.linear.bias: 0.008, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.2.linear.bias: 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.2.0.bias : 0.007, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.weight : 0.007, (torch.Size([200, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.weight : 0.007, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_img_pred.model.0.linear.bias: 0.006, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_pred.model.0.linear.bias: 0.005, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.weight : 0.005, (torch.Size([1024, 1024]))\n", + "roi_fmap_obj.3.bias : 0.005, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.weight: 0.004, (torch.Size([512, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.dimension_reduce.1.0.bias : 0.004, (torch.Size([1024]))\n", + "roi_fmap.1.0.bias : 0.004, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.weight: 0.004, (torch.Size([256, 512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.2.linear.bias: 0.004, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_pred.model.0.linear.bias: 0.004, (torch.Size([2048]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.2.linear.bias: 0.004, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_ent.model.0.linear.bias: 0.004, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_img_pred.model.0.linear.bias: 0.003, (torch.Size([512]))\n", + "ggnn_rel_reason.obj_proj.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_pred.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_img_ent.bias : 0.003, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.weight : 0.003, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.gn.1.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.2.linear.bias: 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.gn.0.weight : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_img_ent.model.0.linear.bias: 0.002, (torch.Size([768]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_pred.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.gn.1.weight : 0.002, (torch.Size([1024]))\n", + "roi_fmap_obj.0.bias : 0.002, (torch.Size([4096]))\n", + "ggnn_rel_reason.ggnn.fc_init_ont_ent.bias : 0.002, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.weight : 0.002, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.0.linear.bias: 0.002, (torch.Size([512]))\n", + "ggnn_rel_reason.ggnn.fc_mp_send_ont_ent.model.2.linear.bias: 0.002, (torch.Size([256]))\n", + "ggnn_rel_reason.ggnn.attention.0.w.0.bias : 0.002, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.weight : 0.001, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.attention.2.w.0.bias : 0.001, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_pred.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_img_ent.bias : 0.001, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_pred.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.attention.1.w.0.bias : 0.000, (torch.Size([200]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq5_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.weight: 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq3_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_mp_receive_ont_ent.model.0.linear.bias: 0.000, (torch.Size([3328]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_img_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.weight : 0.000, (torch.Size([1024, 1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.0.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred_clean.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_w_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_eq4_u_ont_ent.bias : 0.000, (torch.Size([1024]))\n", + "ggnn_rel_reason.ggnn.fc_output_proj_ont_pred.model.2.linear.bias: 0.000, (torch.Size([1024]))\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "e29b 1000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.040307\n", + "total 0.040307\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e29b 2000/ 6755 0.224s/batch, 25.2m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039358\n", + "total 0.039358\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e29b 3000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039673\n", + "total 0.039673\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e29b 4000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039364\n", + "total 0.039364\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e29b 5000/ 6755 0.225s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.038909\n", + "total 0.038909\n", + "dtype: float64\n", + "-----------\n", + "\n", + "e29b 6000/ 6755 0.224s/batch, 25.3m/epoch\n", + "class_loss 0.000000\n", + "rel_loss 0.039541\n", + "total 0.039541\n", + "dtype: float64\n", + "-----------\n", + "overall29: (0.039)\n", + "class_loss 0.000000\n", + "rel_loss 0.039427\n", + "total 0.039427\n", + "dtype: float64\n", + "======================predcls recall with constraint============================\n", + "R@20: 0.186417\n", + "R@50: 0.216851\n", + "R@100: 0.226893\n", + "======================predcls recall without constraint============================\n", + "R@20: 0.344449\n", + "R@50: 0.523861\n", + "R@100: 0.666258\n", + "\n", + "\n", + "======================predcls mean recall with constraint============================\n", + "mR@20: 0.30903309619162395\n", + "mR@50: 0.35425862466280966\n", + "mR@100: 0.3675508048358685\n", + "\n", + "\n", + "======================predcls mean recall without constraint============================\n", + "mR@20: 0.3955416566986557\n", + "mR@50: 0.5338977339173456\n", + "mR@100: 0.6379193373544166\n" + ] + } + ], + "source": [ + "from warnings import warn\n", + "from pprint import pformat\n", + "# print(\"Training starts now!\")\n", + "optimizer = get_optim(conf.lr * conf.num_gpus * conf.batch_size)\n", + "if using_pretrained_gbnet:\n", + " try:\n", + " optimizer.load_state_dict(ckpt['optimizer'])\n", + " except:\n", + " warn(f'Not loading optimizer state probably because we have BPL clean param groups') \n", + " print('epoch = -1')\n", + " recall, recall_mp, mean_recall, mean_recall_mp = val_epoch()\n", + "\n", + "if not start_epoch: start_epoch = 0\n", + "for epoch in range(start_epoch+1, 30):\n", + " print('epoch =', epoch)\n", + " if epoch == 10 or epoch == 20:\n", + " for param_group in optimizer.param_groups:\n", + " param_group['lr'] /= 10\n", + " \n", + " rez = train_epoch(epoch)\n", + " loss_epoch = rez.mean(1)['total']\n", + " print(\"overall{:2d}: ({:.3f})\\n{}\".format(epoch, loss_epoch, rez.mean(1)), flush=True)\n", + "\n", + " if use_tb:\n", + " writer.add_scalar('loss/rel_loss', rez.mean(1)['rel_loss'], epoch)\n", + " writer.add_scalar('loss/class_loss', rez.mean(1)['class_loss'], epoch)\n", + " writer.add_scalar('loss/total', rez.mean(1)['total'], epoch)\n", + "\n", + " if conf.save_dir is not None:\n", + " torch.save({\n", + " 'epoch': epoch,\n", + " 'state_dict': detector.state_dict(), #{k:v for k,v in detector.state_dict().items() if not k.startswith('detector.')},\n", + " 'optimizer': optimizer.state_dict(),\n", + " }, os.path.join(conf.save_dir, '{}-{}.tar'.format('vgrel', epoch)))\n", + "\n", + " recall, recall_mp, mean_recall, mean_recall_mp = val_epoch()\n", + " if use_tb:\n", + " for key, value in recall.items():\n", + " writer.add_scalar('eval_' + conf.mode + '_with_constraint/' + key, value, epoch)\n", + " for key, value in recall_mp.items():\n", + " writer.add_scalar('eval_' + conf.mode + '_without_constraint/' + key, value, epoch)\n", + " for key, value in mean_recall.items():\n", + " writer.add_scalar('eval_' + conf.mode + '_with_constraint/mean ' + key, value, epoch)\n", + " for key, value in mean_recall_mp.items():\n", + " writer.add_scalar('eval_' + conf.mode + '_without_constraint/mean ' + key, value, epoch)\n", + " try:\n", + " writer.add_scalar('eval_' + conf.mode + 'loss', loss_epoch, epoch)\n", + " except:\n", + " warn(f'Cannot add loss to writer') \n", + "\n", + " body = pformat(mean_recall, indent=4)\n", + " try:\n", + " send_simple_message(epoch, body)\n", + " except Exception as e:\n", + " try:\n", + " warn('UNABLE TO SEND MESSAGE', e)\n", + " except:\n", + " warn('UNABLE to send message or print stacktrace') " + ] }, { "cell_type": "code", @@ -374,9 +6986,9 @@ ], "metadata": { "kernelspec": { - "display_name": "KERN", + "display_name": "gbnet", "language": "python", - "name": "kern" + "name": "gbnet" }, "language_info": { "codemirror_mode": { @@ -388,7 +7000,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.9" + "version": "3.8.11" } }, "nbformat": 4,