code/01-model/08_Train_RotatE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 08_Train_RotatE\n",
    "#\n",
    "# created by LuYF-Lemon-love <luyanfeng_nlp@qq.com> on February 28, 2023\n",
    "# updated by LuYF-Lemon-love <luyanfeng_nlp@qq.com> on March 1, 2023\n",
    "#\n",
    "# 该脚本展示了如何在 DRKG 上训练模型 (RotatE), 并利用网格搜索寻找到最优参数.\n",
    "#\n",
    "# 需要的包:\n",
    "#          torch\n",
    "#          dgl, version: 0.4.3\n",
    "#          dglke\n",
    "#          numpy\n",
    "#\n",
    "# 需要的文件:\n",
    "#          ./dataset\n",
    "#\n",
    "# 源教程链接: https://github.com/gnn4dr/DRKG/blob/master/embedding_analysis/Train_embeddings.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training DRKG Using RotatE\n",
    "\n",
    "这个 notebook 展示了如何在 DRKG 上训练模型 (RotatE), 并利用网格搜索寻找到最优参数."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入需要的库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 网格搜索参数\n",
    "\n",
    "我们能使用 DGL-KE 命令训练 RotatE 模型, 关于如何使用 DGL-KE 的更多信息请参考 https://github.com/awslabs/dgl-ke.\n",
    "\n",
    "这里我们使用两个 GPU 训练模型."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1\n",
    "\n",
    "- batch_size: **4096**\n",
    "\n",
    "- neg_sample_size: **256**\n",
    "\n",
    "- hidden_dim: **200**\n",
    "\n",
    "- gamma: **6**, 12, 18\n",
    "\n",
    "- lr: **0.01**, 0.05, 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading train triples....\n",
      "Finished. Read 5286834 train triples.\n",
      "Reading valid triples....\n",
      "Finished. Read 293713 valid triples.\n",
      "Reading test triples....\n",
      "Finished. Read 293714 test triples.\n",
      "|Train|: 5286834\n",
      "random partition 5286834 edges into 2 parts\n",
      "part 0 has 2643417 edges\n",
      "part 1 has 2643417 edges\n",
      "/home/luyanfeng/miniconda3/envs/drkg/lib/python3.8/site-packages/dgl/base.py:25: UserWarning: multigraph will be deprecated.DGL will treat all graphs as multigraph in the future.\n",
      "  warnings.warn(msg, warn_type)\n",
      "|valid|: 293713\n",
      "|test|: 293714\n",
      "Total initialize time 16.312 seconds\n",
      "[proc 1][Train](20000/60000) average pos_loss: 0.3985846310392022\n",
      "[proc 0][Train](20000/60000) average pos_loss: 0.3983381181076169\n",
      "[proc 1][Train](20000/60000) average neg_loss: 0.43240395369678736\n",
      "[proc 0][Train](20000/60000) average neg_loss: 0.43231760443150996\n",
      "[proc 1][Train](20000/60000) average loss: 0.41549429238438607\n",
      "[proc 1][Train](20000/60000) average regularization: 2.4081153454199013e-05\n",
      "[proc 0][Train](20000/60000) average loss: 0.4153278613075614\n",
      "[proc 1][Train] 20000 steps take 3216.963 seconds\n",
      "[proc 1]sample: 61.204, forward: 1245.344, backward: 80.251, update: 1829.918\n",
      "[proc 0][Train](20000/60000) average regularization: 2.407710666022922e-05\n",
      "[proc 0][Train] 20000 steps take 3216.977 seconds\n",
      "[proc 0]sample: 59.891, forward: 1222.879, backward: 78.348, update: 1828.839\n",
      "[proc 1][Train](40000/60000) average pos_loss: 0.399007642647624\n",
      "[proc 1][Train](40000/60000) average neg_loss: 0.43172531069368125\n",
      "[proc 0][Train](40000/60000) average pos_loss: 0.3986720512598753\n",
      "[proc 1][Train](40000/60000) average loss: 0.4153664767175913\n",
      "[proc 0][Train](40000/60000) average neg_loss: 0.4318026569709182\n",
      "[proc 0][Train](40000/60000) average loss: 0.4152373540252447\n",
      "[proc 0][Train](40000/60000) average regularization: 2.8067152616677048e-05\n",
      "[proc 1][Train](40000/60000) average regularization: 2.806864764943384e-05\n",
      "[proc 0][Train] 20000 steps take 3216.160 seconds\n",
      "[proc 0]sample: 59.623, forward: 1226.494, backward: 76.746, update: 1827.998\n",
      "[proc 1][Train] 20000 steps take 3216.166 seconds\n",
      "[proc 1]sample: 61.387, forward: 1245.750, backward: 80.164, update: 1828.614\n",
      "[proc 1][Train](60000/60000) average pos_loss: 0.4022897235646844\n",
      "[proc 0][Train](60000/60000) average pos_loss: 0.40198458015471694\n",
      "[proc 1][Train](60000/60000) average neg_loss: 0.43339845929145815\n",
      "[proc 1][Train](60000/60000) average loss: 0.4178440913379192\n",
      "[proc 0][Train](60000/60000) average neg_loss: 0.43332610063552857\n",
      "[proc 1][Train](60000/60000) average regularization: 2.975673274213477e-05\n",
      "[proc 1][Train] 20000 steps take 3224.531 seconds\n",
      "[proc 1]sample: 61.395, forward: 1253.107, backward: 80.677, update: 1829.101\n",
      "[proc 0][Train](60000/60000) average loss: 0.41765534055531023\n",
      "[proc 0][Train](60000/60000) average regularization: 2.9754905732625046e-05\n",
      "[proc 0][Train] 20000 steps take 3224.538 seconds\n",
      "[proc 0]sample: 59.489, forward: 1227.917, backward: 76.526, update: 1828.174\n",
      "[1]Valid average MRR: 0.5476224588249963\n",
      "[1]Valid average MR: 70.8774692387833\n",
      "[1]Valid average HITS@1: 0.4508365280510973\n",
      "[1]Valid average HITS@3: 0.6048502965469811\n",
      "[1]Valid average HITS@10: 0.7183654847913277\n",
      "[proc 1]validation take 572.065 seconds:\n",
      "[0]Valid average MRR: 0.545990260653221\n",
      "[0]Valid average MR: 73.28551097673912\n",
      "[0]Valid average HITS@1: 0.44880358991120556\n",
      "[0]Valid average HITS@3: 0.6036355341286702\n",
      "[0]Valid average HITS@10: 0.7168961431606472\n",
      "[proc 0]validation take 581.085 seconds:\n",
      "proc 0 takes 10238.760 seconds\n",
      "proc 1 takes 10238.752 seconds\n",
      "training takes 10239.260118484497 seconds\n",
      "Save model to ckpts/RotatE_DRKG_2\n",
      "-------------- Test result --------------\n",
      "Test average MRR : 0.5459065909437415\n",
      "Test average MR : 72.29622353718243\n",
      "Test average HITS@1 : 0.44897246981757766\n",
      "Test average HITS@3 : 0.6030390107383373\n",
      "Test average HITS@10 : 0.7171295886474598\n",
      "-----------------------------------------\n",
      "testing takes 565.091 seconds\n"
     ]
    }
   ],
   "source": [
    "!DGLBACKEND=pytorch dglke_train --dataset DRKG --data_path ./dataset \\\n",
    "--data_files drkg_train.tsv drkg_valid.tsv drkg_test.tsv --format 'raw_udd_hrt' \\\n",
    "--model_name RotatE -de \\\n",
    "--batch_size 4096 --neg_sample_size 256 --hidden_dim 200 \\\n",
    "--gamma 6.0 --lr 0.01 --max_step 60000 -adv --regularization_coef 1.00E-07 \\\n",
    "--gpu 0 1 --num_proc 2 --mix_cpu_gpu --async_update --force_sync_interval 1000 \\\n",
    "--valid --test \\\n",
    "--batch_size_eval 128 --neg_sample_size_eval 10000 \\\n",
    "--log_interval 20000 --eval_interval 60000 --num_thread 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2\n",
    "\n",
    "- batch_size: **4096**\n",
    "\n",
    "- neg_sample_size: **256**\n",
    "\n",
    "- hidden_dim: **200**\n",
    "\n",
    "- gamma: **6**, 12, 18\n",
    "\n",
    "- lr: 0.01, **0.05**, 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading train triples....\n",
      "Finished. Read 5286834 train triples.\n",
      "Reading valid triples....\n",
      "Finished. Read 293713 valid triples.\n",
      "Reading test triples....\n",
      "Finished. Read 293714 test triples.\n",
      "|Train|: 5286834\n",
      "random partition 5286834 edges into 2 parts\n",
      "part 0 has 2643417 edges\n",
      "part 1 has 2643417 edges\n",
      "/home/luyanfeng/miniconda3/envs/drkg/lib/python3.8/site-packages/dgl/base.py:25: UserWarning: multigraph will be deprecated.DGL will treat all graphs as multigraph in the future.\n",
      "  warnings.warn(msg, warn_type)\n",
      "|valid|: 293713\n",
      "|test|: 293714\n",
      "Total initialize time 16.489 seconds\n",
      "[proc 1][Train](20000/60000) average pos_loss: 0.4644792135819793\n",
      "[proc 0][Train](20000/60000) average pos_loss: 0.4645577530056238\n",
      "[proc 1][Train](20000/60000) average neg_loss: 0.46329552944106983\n",
      "[proc 1][Train](20000/60000) average loss: 0.4638873715639114\n",
      "[proc 0][Train](20000/60000) average neg_loss: 0.4630792877822183\n",
      "[proc 1][Train](20000/60000) average regularization: 0.0014319782928241694\n",
      "[proc 0][Train](20000/60000) average loss: 0.4638185203820467\n",
      "[proc 1][Train] 20000 steps take 3227.493 seconds\n",
      "[proc 1]sample: 61.349, forward: 1251.494, backward: 81.508, update: 1832.084\n",
      "[proc 0][Train](20000/60000) average regularization: 0.001429992993449423\n",
      "[proc 0][Train] 20000 steps take 3227.521 seconds\n",
      "[proc 0]sample: 60.663, forward: 1248.145, backward: 82.756, update: 1828.388\n",
      "[proc 1][Train](40000/60000) average pos_loss: 0.4292596078798175\n",
      "[proc 1][Train](40000/60000) average neg_loss: 0.4524590860247612\n",
      "[proc 1][Train](40000/60000) average loss: 0.4408593469545245\n",
      "[proc 0][Train](40000/60000) average pos_loss: 0.4289803568318486\n",
      "[proc 1][Train](40000/60000) average regularization: 0.0014776498246763367\n",
      "[proc 1][Train] 20000 steps take 3231.340 seconds\n",
      "[proc 1]sample: 61.745, forward: 1258.099, backward: 80.623, update: 1830.618\n",
      "[proc 0][Train](40000/60000) average neg_loss: 0.45241762038320305\n",
      "[proc 0][Train](40000/60000) average loss: 0.4406989884912968\n",
      "[proc 0][Train](40000/60000) average regularization: 0.00147553461791249\n",
      "[proc 0][Train] 20000 steps take 3231.352 seconds\n",
      "[proc 0]sample: 60.345, forward: 1248.021, backward: 83.825, update: 1828.000\n",
      "[proc 1][Train](60000/60000) average pos_loss: 0.42702000312507155\n",
      "[proc 0][Train](60000/60000) average pos_loss: 0.42704079540967943\n",
      "[proc 1][Train](60000/60000) average neg_loss: 0.4508185974314809\n",
      "[proc 1][Train](60000/60000) average loss: 0.43891930041611193\n",
      "[proc 0][Train](60000/60000) average neg_loss: 0.4506124938607216\n",
      "[proc 1][Train](60000/60000) average regularization: 0.0014790742959768976\n",
      "[proc 0][Train](60000/60000) average loss: 0.4388266446888447\n",
      "[proc 1][Train] 20000 steps take 3224.135 seconds\n",
      "[proc 1]sample: 60.558, forward: 1247.349, backward: 81.852, update: 1829.498\n",
      "[proc 0][Train](60000/60000) average regularization: 0.0014770393478334882\n",
      "[proc 0][Train] 20000 steps take 3224.120 seconds\n",
      "[proc 0]sample: 60.710, forward: 1248.917, backward: 84.824, update: 1825.812\n",
      "[1]Valid average MRR: 0.5219364930603517\n",
      "[1]Valid average MR: 71.40283404944947\n",
      "[1]Valid average HITS@1: 0.4167489462538388\n",
      "[1]Valid average HITS@3: 0.5841601013230557\n",
      "[1]Valid average HITS@10: 0.7088596389685204\n",
      "[proc 1]validation take 638.331 seconds:\n",
      "[0]Valid average MRR: 0.5193797369810023\n",
      "[0]Valid average MR: 73.27038731818925\n",
      "[0]Valid average HITS@1: 0.41355477474532876\n",
      "[0]Valid average HITS@3: 0.5824753500027238\n",
      "[0]Valid average HITS@10: 0.7072097837337256\n",
      "[proc 0]validation take 640.270 seconds:\n",
      "proc 0 takes 10323.263 seconds\n",
      "proc 1 takes 10323.240 seconds\n",
      "training takes 10323.78138756752 seconds\n",
      "Save model to ckpts/RotatE_DRKG_3\n",
      "-------------- Test result --------------\n",
      "Test average MRR : 0.5193282413800492\n",
      "Test average MR : 72.68231681159223\n",
      "Test average HITS@1 : 0.41369495495618186\n",
      "Test average HITS@3 : 0.5825003234438944\n",
      "Test average HITS@10 : 0.7067163975840444\n",
      "-----------------------------------------\n",
      "testing takes 608.376 seconds\n"
     ]
    }
   ],
   "source": [
    "!DGLBACKEND=pytorch dglke_train --dataset DRKG --data_path ./dataset \\\n",
    "--data_files drkg_train.tsv drkg_valid.tsv drkg_test.tsv --format 'raw_udd_hrt' \\\n",
    "--model_name RotatE -de \\\n",
    "--batch_size 4096 --neg_sample_size 256 --hidden_dim 200 \\\n",
    "--gamma 6.0 --lr 0.05 --max_step 60000 -adv --regularization_coef 1.00E-07 \\\n",
    "--gpu 0 1 --num_proc 2 --mix_cpu_gpu --async_update --force_sync_interval 1000 \\\n",
    "--valid --test \\\n",
    "--batch_size_eval 128 --neg_sample_size_eval 10000 \\\n",
    "--log_interval 20000 --eval_interval 60000 --num_thread 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3\n",
    "\n",
    "- batch_size: **4096**\n",
    "\n",
    "- neg_sample_size: **256**\n",
    "\n",
    "- hidden_dim: **200**\n",
    "\n",
    "- gamma: **6**, 12, 18\n",
    "\n",
    "- lr: 0.01, 0.05, **0.1**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading train triples....\n",
      "Finished. Read 5286834 train triples.\n",
      "Reading valid triples....\n",
      "Finished. Read 293713 valid triples.\n",
      "Reading test triples....\n",
      "Finished. Read 293714 test triples.\n",
      "|Train|: 5286834\n",
      "random partition 5286834 edges into 2 parts\n",
      "part 0 has 2643417 edges\n",
      "part 1 has 2643417 edges\n",
      "/home/luyanfeng/miniconda3/envs/drkg/lib/python3.8/site-packages/dgl/base.py:25: UserWarning: multigraph will be deprecated.DGL will treat all graphs as multigraph in the future.\n",
      "  warnings.warn(msg, warn_type)\n",
      "|valid|: 293713\n",
      "|test|: 293714\n",
      "Total initialize time 16.378 seconds\n",
      "[proc 1][Train](20000/60000) average pos_loss: 0.5467780142441392\n",
      "[proc 1][Train](20000/60000) average neg_loss: 0.4802279629944358\n",
      "[proc 0][Train](20000/60000) average pos_loss: 0.5480604741573334\n",
      "[proc 1][Train](20000/60000) average loss: 0.5135029887244106\n",
      "[proc 1][Train](20000/60000) average regularization: 0.008916160068033424\n",
      "[proc 0][Train](20000/60000) average neg_loss: 0.47927810215242206\n",
      "[proc 1][Train] 20000 steps take 3212.923 seconds\n",
      "[proc 1]sample: 61.550, forward: 1237.260, backward: 80.420, update: 1830.206\n",
      "[proc 0][Train](20000/60000) average loss: 0.5136692880183459\n",
      "[proc 0][Train](20000/60000) average regularization: 0.008905241361620346\n",
      "[proc 0][Train] 20000 steps take 3212.957 seconds\n",
      "[proc 0]sample: 60.304, forward: 1242.174, backward: 76.653, update: 1830.777\n",
      "[proc 1][Train](40000/60000) average pos_loss: 0.4428542936295271\n",
      "[proc 1][Train](40000/60000) average neg_loss: 0.46064405212700366\n",
      "[proc 1][Train](40000/60000) average loss: 0.45174917292594907\n",
      "[proc 0][Train](40000/60000) average pos_loss: 0.44264850018918517\n",
      "[proc 1][Train](40000/60000) average regularization: 0.009699696696037426\n",
      "[proc 1][Train] 20000 steps take 3213.360 seconds\n",
      "[proc 1]sample: 61.041, forward: 1242.472, backward: 79.788, update: 1829.701\n",
      "[proc 0][Train](40000/60000) average neg_loss: 0.4604341848194599\n",
      "[proc 0][Train](40000/60000) average loss: 0.4515413426965475\n",
      "[proc 0][Train](40000/60000) average regularization: 0.00968779081646353\n",
      "[proc 0][Train] 20000 steps take 3213.363 seconds\n",
      "[proc 0]sample: 61.001, forward: 1238.860, backward: 77.045, update: 1830.399\n",
      "[proc 0][Train](60000/60000) average pos_loss: 0.436297942429781\n",
      "[proc 1][Train](60000/60000) average pos_loss: 0.43656045038700103\n",
      "[proc 1][Train](60000/60000) average neg_loss: 0.4567485026359558\n",
      "[proc 0][Train](60000/60000) average neg_loss: 0.45686172361969946\n",
      "[proc 1][Train](60000/60000) average loss: 0.4466544765740633\n",
      "[proc 1][Train](60000/60000) average regularization: 0.009666544355126098\n",
      "[proc 0][Train](60000/60000) average loss: 0.4465798330873251\n",
      "[proc 1][Train] 20000 steps take 3213.645 seconds\n",
      "[proc 1]sample: 59.771, forward: 1244.693, backward: 76.497, update: 1831.912\n",
      "[proc 0][Train](60000/60000) average regularization: 0.009654898232501\n",
      "[proc 0][Train] 20000 steps take 3213.631 seconds\n",
      "[proc 0]sample: 61.194, forward: 1235.214, backward: 80.606, update: 1830.396\n",
      "[1]Valid average MRR: 0.4918308723175946\n",
      "[1]Valid average MR: 75.99342898193481\n",
      "[1]Valid average HITS@1: 0.3841560157159686\n",
      "[1]Valid average HITS@3: 0.5536201883464867\n",
      "[1]Valid average HITS@10: 0.686804170042967\n",
      "[proc 1]validation take 581.768 seconds:\n",
      "[0]Valid average MRR: 0.4892873800041604\n",
      "[0]Valid average MR: 77.6950958762325\n",
      "[0]Valid average HITS@1: 0.3809990739227543\n",
      "[0]Valid average HITS@3: 0.5513700495723702\n",
      "[0]Valid average HITS@10: 0.6845957945198017\n",
      "[proc 0]validation take 585.921 seconds:\n",
      "proc 0 takes 10225.873 seconds\n",
      "proc 1 takes 10225.853 seconds\n",
      "training takes 10226.411473274231 seconds\n",
      "Save model to ckpts/RotatE_DRKG_4\n",
      "-------------- Test result --------------\n",
      "Test average MRR : 0.49031910800541517\n",
      "Test average MR : 77.34867422050021\n",
      "Test average HITS@1 : 0.38256603362454633\n",
      "Test average HITS@3 : 0.5520336109276371\n",
      "Test average HITS@10 : 0.6848822323757124\n",
      "-----------------------------------------\n",
      "testing takes 587.814 seconds\n"
     ]
    }
   ],
   "source": [
    "!DGLBACKEND=pytorch dglke_train --dataset DRKG --data_path ./dataset \\\n",
    "--data_files drkg_train.tsv drkg_valid.tsv drkg_test.tsv --format 'raw_udd_hrt' \\\n",
    "--model_name RotatE -de \\\n",
    "--batch_size 4096 --neg_sample_size 256 --hidden_dim 200 \\\n",
    "--gamma 6.0 --lr 0.1 --max_step 60000 -adv --regularization_coef 1.00E-07 \\\n",
    "--gpu 0 1 --num_proc 2 --mix_cpu_gpu --async_update --force_sync_interval 1000 \\\n",
    "--valid --test \\\n",
    "--batch_size_eval 128 --neg_sample_size_eval 10000 \\\n",
    "--log_interval 20000 --eval_interval 60000 --num_thread 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4\n",
    "\n",
    "- batch_size: **4096**\n",
    "\n",
    "- neg_sample_size: **256**\n",
    "\n",
    "- hidden_dim: **200**\n",
    "\n",
    "- gamma: 6, **12**, 18\n",
    "\n",
    "- lr: **0.01**, 0.05, 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading train triples....\n",
      "Finished. Read 5286834 train triples.\n",
      "Reading valid triples....\n",
      "Finished. Read 293713 valid triples.\n",
      "Reading test triples....\n",
      "Finished. Read 293714 test triples.\n",
      "|Train|: 5286834\n",
      "random partition 5286834 edges into 2 parts\n",
      "part 0 has 2643417 edges\n",
      "part 1 has 2643417 edges\n",
      "/home/luyanfeng/miniconda3/envs/drkg/lib/python3.8/site-packages/dgl/base.py:25: UserWarning: multigraph will be deprecated.DGL will treat all graphs as multigraph in the future.\n",
      "  warnings.warn(msg, warn_type)\n",
      "|valid|: 293713\n",
      "|test|: 293714\n",
      "Total initialize time 16.800 seconds\n",
      "[proc 0][Train](20000/60000) average pos_loss: 0.36959120687544345\n",
      "[proc 1][Train](20000/60000) average pos_loss: 0.36949521178901196\n",
      "[proc 0][Train](20000/60000) average neg_loss: 0.4409327068939805\n",
      "[proc 1][Train](20000/60000) average neg_loss: 0.44107629915215074\n",
      "[proc 0][Train](20000/60000) average loss: 0.4052619569271803\n",
      "[proc 1][Train](20000/60000) average loss: 0.40528575550317764\n",
      "[proc 0][Train](20000/60000) average regularization: 6.63915239598282e-05\n",
      "[proc 1][Train](20000/60000) average regularization: 6.63803454808658e-05\n",
      "[proc 0][Train] 20000 steps take 3218.333 seconds\n",
      "[proc 0]sample: 61.853, forward: 1227.103, backward: 77.147, update: 1831.680\n",
      "[proc 1][Train] 20000 steps take 3218.325 seconds\n",
      "[proc 1]sample: 61.595, forward: 1245.912, backward: 79.948, update: 1830.633\n",
      "[proc 1][Train](40000/60000) average pos_loss: 0.35573004729002716\n",
      "[proc 1][Train](40000/60000) average neg_loss: 0.4313501788511872\n",
      "[proc 0][Train](40000/60000) average pos_loss: 0.3558629102036357\n",
      "[proc 1][Train](40000/60000) average loss: 0.3935401132121682\n",
      "[proc 0][Train](40000/60000) average neg_loss: 0.43092887749820946\n",
      "[proc 1][Train](40000/60000) average regularization: 7.905635422466731e-05\n",
      "[proc 1][Train] 20000 steps take 3215.250 seconds\n",
      "[proc 1]sample: 61.626, forward: 1244.447, backward: 79.128, update: 1829.800\n",
      "[proc 0][Train](40000/60000) average loss: 0.3933958938747644\n",
      "[proc 0][Train](40000/60000) average regularization: 7.906723205196612e-05\n",
      "[proc 0][Train] 20000 steps take 3215.262 seconds\n",
      "[proc 0]sample: 61.728, forward: 1225.296, backward: 76.546, update: 1830.259\n",
      "[proc 1][Train](60000/60000) average pos_loss: 0.35880708608031275\n",
      "[proc 0][Train](60000/60000) average pos_loss: 0.35893875443041323\n",
      "[proc 1][Train](60000/60000) average neg_loss: 0.4323795559704304\n",
      "[proc 0][Train](60000/60000) average neg_loss: 0.4321945361211896\n",
      "[proc 1][Train](60000/60000) average loss: 0.39559332102835176\n",
      "[proc 0][Train](60000/60000) average loss: 0.3955666452854872\n",
      "[proc 1][Train](60000/60000) average regularization: 8.382938799259137e-05\n",
      "[proc 1][Train] 20000 steps take 3215.449 seconds\n",
      "[proc 1]sample: 61.793, forward: 1244.309, backward: 79.424, update: 1829.674\n",
      "[proc 0][Train](60000/60000) average regularization: 8.384394412714755e-05\n",
      "[proc 0][Train] 20000 steps take 3215.445 seconds\n",
      "[proc 0]sample: 61.311, forward: 1223.731, backward: 76.531, update: 1829.651\n",
      "[1]Valid average MRR: 0.5985003701989992\n",
      "[1]Valid average MR: 67.07690474407076\n",
      "[1]Valid average HITS@1: 0.5028088548724269\n",
      "[1]Valid average HITS@3: 0.6593659137800718\n",
      "[1]Valid average HITS@10: 0.7615401376849589\n",
      "[proc 1]validation take 578.274 seconds:\n",
      "[0]Valid average MRR: 0.5969835263712435\n",
      "[0]Valid average MR: 69.65118891975813\n",
      "[0]Valid average HITS@1: 0.5013380454322601\n",
      "[0]Valid average HITS@3: 0.6580153347496868\n",
      "[0]Valid average HITS@10: 0.7596761453396524\n",
      "[proc 0]validation take 578.538 seconds:\n",
      "proc 0 takes 10227.580 seconds\n",
      "proc 1 takes 10227.569 seconds\n",
      "training takes 10228.105356931686 seconds\n",
      "Save model to ckpts/RotatE_DRKG_5\n",
      "-------------- Test result --------------\n",
      "Test average MRR : 0.5964056774241966\n",
      "Test average MR : 68.17194107192711\n",
      "Test average HITS@1 : 0.5005311289213317\n",
      "Test average HITS@3 : 0.6578235971046664\n",
      "Test average HITS@10 : 0.7596675677700075\n",
      "-----------------------------------------\n",
      "testing takes 607.364 seconds\n"
     ]
    }
   ],
   "source": [
    "!DGLBACKEND=pytorch dglke_train --dataset DRKG --data_path ./dataset \\\n",
    "--data_files drkg_train.tsv drkg_valid.tsv drkg_test.tsv --format 'raw_udd_hrt' \\\n",
    "--model_name RotatE -de \\\n",
    "--batch_size 4096 --neg_sample_size 256 --hidden_dim 200 \\\n",
    "--gamma 12.0 --lr 0.01 --max_step 60000 -adv --regularization_coef 1.00E-07 \\\n",
    "--gpu 0 1 --num_proc 2 --mix_cpu_gpu --async_update --force_sync_interval 1000 \\\n",
    "--valid --test \\\n",
    "--batch_size_eval 128 --neg_sample_size_eval 10000 \\\n",
    "--log_interval 20000 --eval_interval 60000 --num_thread 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5\n",
    "\n",
    "- batch_size: **4096**\n",
    "\n",
    "- neg_sample_size: **256**\n",
    "\n",
    "- hidden_dim: **200**\n",
    "\n",
    "- gamma: 6, **12**, 18\n",
    "\n",
    "- lr: 0.01, **0.05**, 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading train triples....\n",
      "Finished. Read 5286834 train triples.\n",
      "Reading valid triples....\n",
      "Finished. Read 293713 valid triples.\n",
      "Reading test triples....\n",
      "Finished. Read 293714 test triples.\n",
      "|Train|: 5286834\n",
      "random partition 5286834 edges into 2 parts\n",
      "part 0 has 2643417 edges\n",
      "part 1 has 2643417 edges\n",
      "/home/luyanfeng/miniconda3/envs/drkg/lib/python3.8/site-packages/dgl/base.py:25: UserWarning: multigraph will be deprecated.DGL will treat all graphs as multigraph in the future.\n",
      "  warnings.warn(msg, warn_type)\n",
      "|valid|: 293713\n",
      "|test|: 293714\n",
      "Total initialize time 16.295 seconds\n",
      "[proc 0][Train](20000/60000) average pos_loss: 0.4153521956175566\n",
      "[proc 0][Train](20000/60000) average neg_loss: 0.47424251278974117\n",
      "[proc 0][Train](20000/60000) average loss: 0.4447973541840911\n",
      "[proc 1][Train](20000/60000) average pos_loss: 0.4143628444954753\n",
      "[proc 0][Train](20000/60000) average regularization: 0.0010063675945350041\n",
      "[proc 1][Train](20000/60000) average neg_loss: 0.4744068071845919\n",
      "[proc 0][Train] 20000 steps take 3223.772 seconds\n",
      "[proc 0]sample: 62.623, forward: 1244.697, backward: 81.142, update: 1833.451\n",
      "[proc 1][Train](20000/60000) average loss: 0.4443848257318139\n",
      "[proc 1][Train](20000/60000) average regularization: 0.0010069222753887517\n",
      "[proc 1][Train] 20000 steps take 3223.758 seconds\n",
      "[proc 1]sample: 61.047, forward: 1245.407, backward: 80.189, update: 1833.656\n",
      "[proc 0][Train](40000/60000) average pos_loss: 0.38347347803115844\n",
      "[proc 0][Train](40000/60000) average neg_loss: 0.45398936755508185\n",
      "[proc 1][Train](40000/60000) average pos_loss: 0.3834198453217745\n",
      "[proc 0][Train](40000/60000) average loss: 0.41873142286986115\n",
      "[proc 0][Train](40000/60000) average regularization: 0.0010150431836606004\n",
      "[proc 1][Train](40000/60000) average neg_loss: 0.454316990929842\n",
      "[proc 0][Train] 20000 steps take 3218.288 seconds\n",
      "[proc 0]sample: 61.846, forward: 1242.214, backward: 79.975, update: 1833.100\n",
      "[proc 1][Train](40000/60000) average loss: 0.4188684180185199\n",
      "[proc 1][Train](40000/60000) average regularization: 0.0010155350731016369\n",
      "[proc 1][Train] 20000 steps take 3218.288 seconds\n",
      "[proc 1]sample: 60.337, forward: 1243.767, backward: 78.704, update: 1832.675\n",
      "[proc 0][Train](60000/60000) average pos_loss: 0.3816402970016003\n",
      "[proc 1][Train](60000/60000) average pos_loss: 0.3816695139363408\n",
      "[proc 0][Train](60000/60000) average neg_loss: 0.45099424796700477\n",
      "[proc 1][Train](60000/60000) average neg_loss: 0.4512510766223073\n",
      "[proc 0][Train](60000/60000) average loss: 0.41631727253645656\n",
      "[proc 1][Train](60000/60000) average loss: 0.4164602952495217\n",
      "[proc 0][Train](60000/60000) average regularization: 0.0010164719864871586\n",
      "[proc 0][Train] 20000 steps take 3221.187 seconds\n",
      "[proc 0]sample: 61.903, forward: 1244.139, backward: 81.539, update: 1832.044\n",
      "[proc 1][Train](60000/60000) average regularization: 0.0010169506585923954\n",
      "[proc 1][Train] 20000 steps take 3221.178 seconds\n",
      "[proc 1]sample: 60.030, forward: 1247.093, backward: 78.402, update: 1833.470\n",
      "[1]Valid average MRR: 0.6017782716234142\n",
      "[1]Valid average MR: 60.91930245068332\n",
      "[1]Valid average HITS@1: 0.502798640854709\n",
      "[1]Valid average HITS@3: 0.6661786635979218\n",
      "[1]Valid average HITS@10: 0.7695547369209503\n",
      "[proc 1]validation take 548.488 seconds:\n",
      "[0]Valid average MRR: 0.6001885700442473\n",
      "[0]Valid average MR: 63.01837173830147\n",
      "[0]Valid average HITS@1: 0.5011814294274664\n",
      "[0]Valid average HITS@3: 0.6649847469630114\n",
      "[0]Valid average HITS@10: 0.7679495560276733\n",
      "[proc 0]validation take 555.221 seconds:\n",
      "proc 0 takes 10218.469 seconds\n",
      "proc 1 takes 10218.444 seconds\n",
      "training takes 10218.990820169449 seconds\n",
      "Save model to ckpts/RotatE_DRKG_6\n",
      "-------------- Test result --------------\n",
      "Test average MRR : 0.599893358264066\n",
      "Test average MR : 62.04127654793439\n",
      "Test average HITS@1 : 0.5011507793295519\n",
      "Test average HITS@3 : 0.6641001109923259\n",
      "Test average HITS@10 : 0.7676379062625547\n",
      "-----------------------------------------\n",
      "testing takes 587.043 seconds\n"
     ]
    }
   ],
   "source": [
    "!DGLBACKEND=pytorch dglke_train --dataset DRKG --data_path ./dataset \\\n",
    "--data_files drkg_train.tsv drkg_valid.tsv drkg_test.tsv --format 'raw_udd_hrt' \\\n",
    "--model_name RotatE -de \\\n",
    "--batch_size 4096 --neg_sample_size 256 --hidden_dim 200 \\\n",
    "--gamma 12.0 --lr 0.05 --max_step 60000 -adv --regularization_coef 1.00E-07 \\\n",
    "--gpu 0 1 --num_proc 2 --mix_cpu_gpu --async_update --force_sync_interval 1000 \\\n",
    "--valid --test \\\n",
    "--batch_size_eval 128 --neg_sample_size_eval 10000 \\\n",
    "--log_interval 20000 --eval_interval 60000 --num_thread 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6\n",
    "\n",
    "- batch_size: **4096**\n",
    "\n",
    "- neg_sample_size: **256**\n",
    "\n",
    "- hidden_dim: **200**\n",
    "\n",
    "- gamma: 6, **12**, 18\n",
    "\n",
    "- lr: 0.01, 0.05, **0.1**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading train triples....\n",
      "Finished. Read 5286834 train triples.\n",
      "Reading valid triples....\n",
      "Finished. Read 293713 valid triples.\n",
      "Reading test triples....\n",
      "Finished. Read 293714 test triples.\n",
      "|Train|: 5286834\n",
      "random partition 5286834 edges into 2 parts\n",
      "part 0 has 2643417 edges\n",
      "part 1 has 2643417 edges\n",
      "/home/luyanfeng/miniconda3/envs/drkg/lib/python3.8/site-packages/dgl/base.py:25: UserWarning: multigraph will be deprecated.DGL will treat all graphs as multigraph in the future.\n",
      "  warnings.warn(msg, warn_type)\n",
      "|valid|: 293713\n",
      "|test|: 293714\n",
      "Total initialize time 16.515 seconds\n",
      "[proc 0][Train](20000/60000) average pos_loss: 0.4915726067259908\n",
      "[proc 1][Train](20000/60000) average pos_loss: 0.49016874236166474\n",
      "[proc 0][Train](20000/60000) average neg_loss: 0.5112811038811226\n",
      "[proc 0][Train](20000/60000) average loss: 0.5014268553838134\n",
      "[proc 1][Train](20000/60000) average neg_loss: 0.5106066586047411\n",
      "[proc 0][Train](20000/60000) average regularization: 0.004226207485045052\n",
      "[proc 1][Train](20000/60000) average loss: 0.5003877004981041\n",
      "[proc 0][Train] 20000 steps take 3224.964 seconds\n",
      "[proc 0]sample: 61.464, forward: 1249.008, backward: 81.182, update: 1832.526\n",
      "[proc 1][Train](20000/60000) average regularization: 0.004226797353167239\n",
      "[proc 1][Train] 20000 steps take 3224.937 seconds\n",
      "[proc 1]sample: 61.404, forward: 1243.487, backward: 77.365, update: 1834.336\n",
      "[proc 0][Train](40000/60000) average pos_loss: 0.4031825755774975\n",
      "[proc 1][Train](40000/60000) average pos_loss: 0.40341699343323706\n",
      "[proc 0][Train](40000/60000) average neg_loss: 0.4738922756582499\n",
      "[proc 0][Train](40000/60000) average loss: 0.43853742567002774\n",
      "[proc 1][Train](40000/60000) average neg_loss: 0.4731916511461139\n",
      "[proc 0][Train](40000/60000) average regularization: 0.004203305809083395\n",
      "[proc 0][Train] 20000 steps take 3220.971 seconds\n",
      "[proc 0]sample: 60.566, forward: 1248.934, backward: 79.208, update: 1831.788\n",
      "[proc 1][Train](40000/60000) average loss: 0.43830432225167754\n",
      "[proc 1][Train](40000/60000) average regularization: 0.004203667245386168\n",
      "[proc 1][Train] 20000 steps take 3220.974 seconds\n",
      "[proc 1]sample: 61.528, forward: 1241.031, backward: 79.523, update: 1830.252\n",
      "[proc 0][Train](60000/60000) average pos_loss: 0.3967563616901636\n",
      "[proc 1][Train](60000/60000) average pos_loss: 0.39676768165528775\n",
      "[proc 0][Train](60000/60000) average neg_loss: 0.466479061229527\n",
      "[proc 1][Train](60000/60000) average neg_loss: 0.46667896397709846\n",
      "[proc 1][Train](60000/60000) average loss: 0.43172332277297976\n",
      "[proc 0][Train](60000/60000) average loss: 0.4316177114456892\n",
      "[proc 1][Train](60000/60000) average regularization: 0.0041911295733880255\n",
      "[proc 0][Train](60000/60000) average regularization: 0.004190793096111156\n",
      "[proc 1][Train] 20000 steps take 3214.588 seconds\n",
      "[proc 1]sample: 60.830, forward: 1233.642, backward: 80.874, update: 1827.214\n",
      "[proc 0][Train] 20000 steps take 3214.596 seconds\n",
      "[proc 0]sample: 60.840, forward: 1246.677, backward: 75.345, update: 1830.949\n",
      "[0]Valid average MRR: 0.5774183128251846\n",
      "[0]Valid average MR: 62.680663370921174\n",
      "[0]Valid average HITS@1: 0.47259900855259573\n",
      "[0]Valid average HITS@3: 0.6458503568121152\n",
      "[0]Valid average HITS@10: 0.757517568230103\n",
      "[proc 0]validation take 615.160 seconds:\n",
      "[1]Valid average MRR: 0.5790303370345516\n",
      "[1]Valid average MR: 61.33354215325112\n",
      "[1]Valid average HITS@1: 0.47446495570520986\n",
      "[1]Valid average HITS@3: 0.6468435280579067\n",
      "[1]Valid average HITS@10: 0.7583635781746869\n",
      "[proc 1]validation take 620.267 seconds:\n",
      "proc 1 takes 10280.766 seconds\n",
      "proc 0 takes 10280.797 seconds\n",
      "training takes 10281.337394714355 seconds\n",
      "Save model to ckpts/RotatE_DRKG_7\n",
      "-------------- Test result --------------\n",
      "Test average MRR : 0.5777721433520705\n",
      "Test average MR : 62.11719053228651\n",
      "Test average HITS@1 : 0.47314564508331236\n",
      "Test average HITS@3 : 0.6463106968002887\n",
      "Test average HITS@10 : 0.7570459698890758\n",
      "-----------------------------------------\n",
      "testing takes 634.064 seconds\n"
     ]
    }
   ],
   "source": [
    "!DGLBACKEND=pytorch dglke_train --dataset DRKG --data_path ./dataset \\\n",
    "--data_files drkg_train.tsv drkg_valid.tsv drkg_test.tsv --format 'raw_udd_hrt' \\\n",
    "--model_name RotatE -de \\\n",
    "--batch_size 4096 --neg_sample_size 256 --hidden_dim 200 \\\n",
    "--gamma 12.0 --lr 0.1 --max_step 60000 -adv --regularization_coef 1.00E-07 \\\n",
    "--gpu 0 1 --num_proc 2 --mix_cpu_gpu --async_update --force_sync_interval 1000 \\\n",
    "--valid --test \\\n",
    "--batch_size_eval 128 --neg_sample_size_eval 10000 \\\n",
    "--log_interval 20000 --eval_interval 60000 --num_thread 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7\n",
    "\n",
    "- batch_size: **4096**\n",
    "\n",
    "- neg_sample_size: **256**\n",
    "\n",
    "- hidden_dim: **200**\n",
    "\n",
    "- gamma: 6, 12, **18**\n",
    "\n",
    "- lr: **0.01**, 0.05, 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading train triples....\n",
      "Finished. Read 5286834 train triples.\n",
      "Reading valid triples....\n",
      "Finished. Read 293713 valid triples.\n",
      "Reading test triples....\n",
      "Finished. Read 293714 test triples.\n",
      "|Train|: 5286834\n",
      "random partition 5286834 edges into 2 parts\n",
      "part 0 has 2643417 edges\n",
      "part 1 has 2643417 edges\n",
      "/home/luyanfeng/miniconda3/envs/drkg/lib/python3.8/site-packages/dgl/base.py:25: UserWarning: multigraph will be deprecated.DGL will treat all graphs as multigraph in the future.\n",
      "  warnings.warn(msg, warn_type)\n",
      "|valid|: 293713\n",
      "|test|: 293714\n",
      "Total initialize time 16.641 seconds\n",
      "[proc 1][Train](20000/60000) average pos_loss: 0.38049039629995823\n",
      "[proc 1][Train](20000/60000) average neg_loss: 0.46546533613130453\n",
      "[proc 0][Train](20000/60000) average pos_loss: 0.38027947252243754\n",
      "[proc 1][Train](20000/60000) average loss: 0.42297786618322136\n",
      "[proc 0][Train](20000/60000) average neg_loss: 0.46538856863863765\n",
      "[proc 1][Train](20000/60000) average regularization: 0.00016217409892597061\n",
      "[proc 0][Train](20000/60000) average loss: 0.4228340206176043\n",
      "[proc 1][Train] 20000 steps take 3217.276 seconds\n",
      "[proc 1]sample: 60.030, forward: 1245.201, backward: 78.328, update: 1833.078\n",
      "[proc 0][Train](20000/60000) average regularization: 0.00016217898656796024\n",
      "[proc 0][Train] 20000 steps take 3217.290 seconds\n",
      "[proc 0]sample: 61.104, forward: 1231.885, backward: 78.901, update: 1832.381\n",
      "[proc 1][Train](40000/60000) average pos_loss: 0.34731653598248957\n",
      "[proc 1][Train](40000/60000) average neg_loss: 0.43946993682533503\n",
      "[proc 0][Train](40000/60000) average pos_loss: 0.3471054085418582\n",
      "[proc 1][Train](40000/60000) average loss: 0.39339323651194574\n",
      "[proc 1][Train](40000/60000) average regularization: 0.00018922629421940656\n",
      "[proc 0][Train](40000/60000) average neg_loss: 0.43935260481089355\n",
      "[proc 1][Train] 20000 steps take 3208.852 seconds\n",
      "[proc 1]sample: 59.437, forward: 1241.211, backward: 74.292, update: 1833.523\n",
      "[proc 0][Train](40000/60000) average loss: 0.39322900668233635\n",
      "[proc 0][Train](40000/60000) average regularization: 0.00018923880955335336\n",
      "[proc 0][Train] 20000 steps take 3208.866 seconds\n",
      "[proc 0]sample: 60.058, forward: 1223.142, backward: 79.367, update: 1829.800\n",
      "[proc 0][Train](60000/60000) average pos_loss: 0.3482796751573682\n",
      "[proc 1][Train](60000/60000) average pos_loss: 0.3483945864662528\n",
      "[proc 0][Train](60000/60000) average neg_loss: 0.438743827162683\n",
      "[proc 0][Train](60000/60000) average loss: 0.39351175125688315\n",
      "[proc 1][Train](60000/60000) average neg_loss: 0.4387718275874853\n",
      "[proc 0][Train](60000/60000) average regularization: 0.0001994598304234387\n",
      "[proc 0][Train] 20000 steps take 3201.425 seconds\n",
      "[proc 0]sample: 60.749, forward: 1231.141, backward: 80.865, update: 1828.149\n",
      "[proc 1][Train](60000/60000) average loss: 0.3935832070142031\n",
      "[proc 1][Train](60000/60000) average regularization: 0.00019944346207485068\n",
      "[proc 1][Train] 20000 steps take 3201.455 seconds\n",
      "[proc 1]sample: 58.419, forward: 1233.081, backward: 73.982, update: 1832.729\n",
      "[0]Valid average MRR: 0.5694204668777523\n",
      "[0]Valid average MR: 79.21241556354524\n",
      "[0]Valid average HITS@1: 0.4661743476602931\n",
      "[0]Valid average HITS@3: 0.635278776488533\n",
      "[0]Valid average HITS@10: 0.7486755733507654\n",
      "[proc 0]validation take 632.019 seconds:\n",
      "[1]Valid average MRR: 0.5713861354586558\n",
      "[1]Valid average MR: 76.52468728082421\n",
      "[1]Valid average HITS@1: 0.46843868525163934\n",
      "[1]Valid average HITS@3: 0.6370074289955535\n",
      "[1]Valid average HITS@10: 0.7502945041775333\n",
      "[proc 1]validation take 651.953 seconds:\n",
      "proc 1 takes 10279.536 seconds\n",
      "proc 0 takes 10279.544 seconds\n",
      "training takes 10280.108121871948 seconds\n",
      "Save model to ckpts/RotatE_DRKG_8\n",
      "-------------- Test result --------------\n",
      "Test average MRR : 0.5697319505066681\n",
      "Test average MR : 78.29155232641277\n",
      "Test average HITS@1 : 0.46721470546177574\n",
      "Test average HITS@3 : 0.6348488665845006\n",
      "Test average HITS@10 : 0.7481904165276425\n",
      "-----------------------------------------\n",
      "testing takes 647.985 seconds\n"
     ]
    }
   ],
   "source": [
    "!DGLBACKEND=pytorch dglke_train --dataset DRKG --data_path ./dataset \\\n",
    "--data_files drkg_train.tsv drkg_valid.tsv drkg_test.tsv --format 'raw_udd_hrt' \\\n",
    "--model_name RotatE -de \\\n",
    "--batch_size 4096 --neg_sample_size 256 --hidden_dim 200 \\\n",
    "--gamma 18.0 --lr 0.01 --max_step 60000 -adv --regularization_coef 1.00E-07 \\\n",
    "--gpu 0 1 --num_proc 2 --mix_cpu_gpu --async_update --force_sync_interval 1000 \\\n",
    "--valid --test \\\n",
    "--batch_size_eval 128 --neg_sample_size_eval 10000 \\\n",
    "--log_interval 20000 --eval_interval 60000 --num_thread 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8\n",
    "\n",
    "- batch_size: **4096**\n",
    "\n",
    "- neg_sample_size: **256**\n",
    "\n",
    "- hidden_dim: **200**\n",
    "\n",
    "- gamma: 6, 12, **18**\n",
    "\n",
    "- lr: 0.01, **0.05**, 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading train triples....\n",
      "Finished. Read 5286834 train triples.\n",
      "Reading valid triples....\n",
      "Finished. Read 293713 valid triples.\n",
      "Reading test triples....\n",
      "Finished. Read 293714 test triples.\n",
      "|Train|: 5286834\n",
      "random partition 5286834 edges into 2 parts\n",
      "part 0 has 2643417 edges\n",
      "part 1 has 2643417 edges\n",
      "/home/luyanfeng/miniconda3/envs/drkg/lib/python3.8/site-packages/dgl/base.py:25: UserWarning: multigraph will be deprecated.DGL will treat all graphs as multigraph in the future.\n",
      "  warnings.warn(msg, warn_type)\n",
      "|valid|: 293713\n",
      "|test|: 293714\n",
      "Total initialize time 16.412 seconds\n",
      "[proc 0][Train](20000/60000) average pos_loss: 0.4032720058605075\n",
      "[proc 1][Train](20000/60000) average pos_loss: 0.40326393421292306\n",
      "[proc 0][Train](20000/60000) average neg_loss: 0.48477220088467005\n",
      "[proc 1][Train](20000/60000) average neg_loss: 0.4841169567797333\n",
      "[proc 0][Train](20000/60000) average loss: 0.4440221032679081\n",
      "[proc 1][Train](20000/60000) average loss: 0.44369044541567565\n",
      "[proc 0][Train](20000/60000) average regularization: 0.000508429336037807\n",
      "[proc 1][Train](20000/60000) average regularization: 0.0005083510163345636\n",
      "[proc 0][Train] 20000 steps take 3206.658 seconds\n",
      "[proc 0]sample: 61.325, forward: 1235.149, backward: 80.894, update: 1828.256\n",
      "[proc 1][Train] 20000 steps take 3206.641 seconds\n",
      "[proc 1]sample: 63.000, forward: 1228.975, backward: 79.282, update: 1830.121\n",
      "[proc 0][Train](40000/60000) average pos_loss: 0.36691446163356306\n",
      "[proc 1][Train](40000/60000) average pos_loss: 0.3668058087453246\n",
      "[proc 0][Train](40000/60000) average neg_loss: 0.4564678882449865\n",
      "[proc 1][Train](40000/60000) average neg_loss: 0.45644007593840363\n",
      "[proc 0][Train](40000/60000) average loss: 0.41169117480814454\n",
      "[proc 1][Train](40000/60000) average loss: 0.4116229423865676\n",
      "[proc 0][Train](40000/60000) average regularization: 0.0005233183638803894\n",
      "[proc 0][Train] 20000 steps take 3204.176 seconds\n",
      "[proc 1][Train](40000/60000) average regularization: 0.000523221502150409\n",
      "[proc 0]sample: 61.008, forward: 1233.386, backward: 82.155, update: 1825.744\n",
      "[proc 1][Train] 20000 steps take 3204.179 seconds\n",
      "[proc 1]sample: 62.717, forward: 1228.852, backward: 78.466, update: 1829.284\n",
      "[proc 0][Train](60000/60000) average pos_loss: 0.36455704486966134\n",
      "[proc 1][Train](60000/60000) average pos_loss: 0.36451500280052423\n",
      "[proc 0][Train](60000/60000) average neg_loss: 0.4524166697219014\n",
      "[proc 1][Train](60000/60000) average neg_loss: 0.45255929368585346\n",
      "[proc 0][Train](60000/60000) average loss: 0.4084868571475148\n",
      "[proc 1][Train](60000/60000) average loss: 0.408537148308754\n",
      "[proc 0][Train](60000/60000) average regularization: 0.0005288811601523776\n",
      "[proc 1][Train](60000/60000) average regularization: 0.0005287911817868007\n",
      "[proc 0][Train] 20000 steps take 3200.493 seconds\n",
      "[proc 0]sample: 60.788, forward: 1231.465, backward: 76.530, update: 1830.056\n",
      "[proc 1][Train] 20000 steps take 3200.492 seconds\n",
      "[proc 1]sample: 62.628, forward: 1227.641, backward: 80.578, update: 1827.264\n",
      "[1]Valid average MRR: 0.6159215820161785\n",
      "[1]Valid average MR: 62.89526546232049\n",
      "[1]Valid average HITS@1: 0.5171152890226547\n",
      "[1]Valid average HITS@3: 0.6825959947431856\n",
      "[1]Valid average HITS@10: 0.7816208965183818\n",
      "[proc 1]validation take 555.286 seconds:\n",
      "[0]Valid average MRR: 0.6135984576630072\n",
      "[0]Valid average MR: 64.93042504221823\n",
      "[0]Valid average HITS@1: 0.5142861033938008\n",
      "[0]Valid average HITS@3: 0.6807450836193277\n",
      "[0]Valid average HITS@10: 0.7798319442174647\n",
      "[proc 0]validation take 561.605 seconds:\n",
      "proc 0 takes 10172.932 seconds\n",
      "proc 1 takes 10172.915 seconds\n",
      "training takes 10173.468384027481 seconds\n",
      "Save model to ckpts/RotatE_DRKG_9\n",
      "-------------- Test result --------------\n",
      "Test average MRR : 0.6142171144502637\n",
      "Test average MR : 63.51198955446455\n",
      "Test average HITS@1 : 0.5154776415152155\n",
      "Test average HITS@3 : 0.6808017322974049\n",
      "Test average HITS@10 : 0.7799526069577889\n",
      "-----------------------------------------\n",
      "testing takes 590.722 seconds\n"
     ]
    }
   ],
   "source": [
    "!DGLBACKEND=pytorch dglke_train --dataset DRKG --data_path ./dataset \\\n",
    "--data_files drkg_train.tsv drkg_valid.tsv drkg_test.tsv --format 'raw_udd_hrt' \\\n",
    "--model_name RotatE -de \\\n",
    "--batch_size 4096 --neg_sample_size 256 --hidden_dim 200 \\\n",
    "--gamma 18.0 --lr 0.05 --max_step 60000 -adv --regularization_coef 1.00E-07 \\\n",
    "--gpu 0 1 --num_proc 2 --mix_cpu_gpu --async_update --force_sync_interval 1000 \\\n",
    "--valid --test \\\n",
    "--batch_size_eval 128 --neg_sample_size_eval 10000 \\\n",
    "--log_interval 20000 --eval_interval 60000 --num_thread 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9\n",
    "\n",
    "- batch_size: **4096**\n",
    "\n",
    "- neg_sample_size: **256**\n",
    "\n",
    "- hidden_dim: **200**\n",
    "\n",
    "- gamma: 6, 12, **18**\n",
    "\n",
    "- lr: 0.01, 0.05, **0.1**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading train triples....\n",
      "Finished. Read 5286834 train triples.\n",
      "Reading valid triples....\n",
      "Finished. Read 293713 valid triples.\n",
      "Reading test triples....\n",
      "Finished. Read 293714 test triples.\n",
      "|Train|: 5286834\n",
      "random partition 5286834 edges into 2 parts\n",
      "part 0 has 2643417 edges\n",
      "part 1 has 2643417 edges\n",
      "/home/luyanfeng/miniconda3/envs/drkg/lib/python3.8/site-packages/dgl/base.py:25: UserWarning: multigraph will be deprecated.DGL will treat all graphs as multigraph in the future.\n",
      "  warnings.warn(msg, warn_type)\n",
      "|valid|: 293713\n",
      "|test|: 293714\n",
      "Total initialize time 16.453 seconds\n",
      "[proc 0][Train](20000/60000) average pos_loss: 0.4747671257123351\n",
      "[proc 1][Train](20000/60000) average pos_loss: 0.47316176998317244\n",
      "[proc 0][Train](20000/60000) average neg_loss: 0.5237953473912552\n",
      "[proc 1][Train](20000/60000) average neg_loss: 0.5244519184571691\n",
      "[proc 0][Train](20000/60000) average loss: 0.4992812364414334\n",
      "[proc 1][Train](20000/60000) average loss: 0.49880684430748223\n",
      "[proc 0][Train](20000/60000) average regularization: 0.0031896132175268576\n",
      "[proc 1][Train](20000/60000) average regularization: 0.0031893700716358582\n",
      "[proc 0][Train] 20000 steps take 3215.651 seconds\n",
      "[proc 0]sample: 60.547, forward: 1237.957, backward: 86.600, update: 1823.178\n",
      "[proc 1][Train] 20000 steps take 3215.632 seconds\n",
      "[proc 1]sample: 60.291, forward: 1242.300, backward: 84.662, update: 1828.051\n",
      "[proc 1][Train](40000/60000) average pos_loss: 0.38418100795447824\n",
      "[proc 1][Train](40000/60000) average neg_loss: 0.47552528167515995\n",
      "[proc 1][Train](40000/60000) average loss: 0.4298531448364258\n",
      "[proc 0][Train](40000/60000) average pos_loss: 0.3836745307132602\n",
      "[proc 1][Train](40000/60000) average regularization: 0.003153605466533918\n",
      "[proc 1][Train] 20000 steps take 3218.698 seconds\n",
      "[proc 1]sample: 61.132, forward: 1245.653, backward: 83.172, update: 1828.514\n",
      "[proc 0][Train](40000/60000) average neg_loss: 0.47645306847393515\n",
      "[proc 0][Train](40000/60000) average loss: 0.43006379944980144\n",
      "[proc 0][Train](40000/60000) average regularization: 0.0031538884034380318\n",
      "[proc 0][Train] 20000 steps take 3218.715 seconds\n",
      "[proc 0]sample: 61.214, forward: 1224.829, backward: 81.914, update: 1829.148\n",
      "[proc 1][Train](60000/60000) average pos_loss: 0.37729849289804696\n",
      "[proc 0][Train](60000/60000) average pos_loss: 0.3771422233849764\n",
      "[proc 1][Train](60000/60000) average neg_loss: 0.46769127538055183\n",
      "[proc 0][Train](60000/60000) average neg_loss: 0.46776676778793336\n",
      "[proc 1][Train](60000/60000) average loss: 0.4224948842033744\n",
      "[proc 0][Train](60000/60000) average loss: 0.4224544956043363\n",
      "[proc 1][Train](60000/60000) average regularization: 0.003137280592985917\n",
      "[proc 0][Train](60000/60000) average regularization: 0.0031376515630749053\n",
      "[proc 1][Train] 20000 steps take 3215.421 seconds\n",
      "[proc 1]sample: 60.566, forward: 1244.725, backward: 79.765, update: 1830.133\n",
      "[proc 0][Train] 20000 steps take 3215.410 seconds\n",
      "[proc 0]sample: 60.978, forward: 1226.149, backward: 81.920, update: 1828.518\n",
      "[0]Valid average MRR: 0.5976592798000654\n",
      "[0]Valid average MR: 61.72555087977339\n",
      "[0]Valid average HITS@1: 0.4928671623903688\n",
      "[0]Valid average HITS@3: 0.668832053167729\n",
      "[0]Valid average HITS@10: 0.7746874489295636\n",
      "[proc 0]validation take 566.016 seconds:\n",
      "[1]Valid average MRR: 0.5992451523765985\n",
      "[1]Valid average MR: 59.85334032426102\n",
      "[1]Valid average HITS@1: 0.49443336034373575\n",
      "[1]Valid average HITS@3: 0.6704719557120192\n",
      "[1]Valid average HITS@10: 0.775645696153401\n",
      "[proc 1]validation take 574.765 seconds:\n",
      "proc 1 takes 10224.517 seconds\n",
      "proc 0 takes 10224.539 seconds\n",
      "training takes 10225.052880048752 seconds\n",
      "Save model to ckpts/RotatE_DRKG_10\n",
      "-------------- Test result --------------\n",
      "Test average MRR : 0.5969448094374135\n",
      "Test average MR : 61.249039882334515\n",
      "Test average HITS@1 : 0.4922986306406913\n",
      "Test average HITS@3 : 0.6680733638845953\n",
      "Test average HITS@10 : 0.7733815888925962\n",
      "-----------------------------------------\n",
      "testing takes 576.168 seconds\n"
     ]
    }
   ],
   "source": [
    "!DGLBACKEND=pytorch dglke_train --dataset DRKG --data_path ./dataset \\\n",
    "--data_files drkg_train.tsv drkg_valid.tsv drkg_test.tsv --format 'raw_udd_hrt' \\\n",
    "--model_name RotatE -de \\\n",
    "--batch_size 4096 --neg_sample_size 256 --hidden_dim 200 \\\n",
    "--gamma 18.0 --lr 0.1 --max_step 60000 -adv --regularization_coef 1.00E-07 \\\n",
    "--gpu 0 1 --num_proc 2 --mix_cpu_gpu --async_update --force_sync_interval 1000 \\\n",
    "--valid --test \\\n",
    "--batch_size_eval 128 --neg_sample_size_eval 10000 \\\n",
    "--log_interval 20000 --eval_interval 60000 --num_thread 32"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}