experiments/main/data_preprocess/extract_embedding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "472b25d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\r\n",
    "import numpy as np\r\n",
    "import gensim\r\n",
    "import wandb\r\n",
    "from pathlib import Path\r\n",
    "from gensim.models.word2vec import Word2Vec\r\n",
    "from gensim.models.callbacks import CallbackAny2Vec\r\n",
    "from tokenizers import Tokenizer, decoders, pre_tokenizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "95ad26fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "                Tracking run with wandb version 0.10.30<br/>\n",
       "                Syncing run <strong style=\"color:#cdcd00\">language_specific</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://wandb.ai/windsuzu/phonetic-translation\" target=\"_blank\">https://wandb.ai/windsuzu/phonetic-translation</a><br/>\n",
       "                Run page: <a href=\"https://wandb.ai/windsuzu/phonetic-translation/runs/2q4igpuu\" target=\"_blank\">https://wandb.ai/windsuzu/phonetic-translation/runs/2q4igpuu</a><br/>\n",
       "                Run data is saved locally in <code>d:\\Project\\phonetics-in-chinese-japanese-machine-translation\\experiments\\main\\wandb\\run-20210510_162721-2q4igpuu</code><br/><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "run = wandb.init(project='phonetic-translation', \r\n",
    "                 entity='windsuzu',\r\n",
    "                 group=\"embedding\",\r\n",
    "                 name=\"language_specific\",  # or sentencepiece\r\n",
    "                 job_type=\"word2vec\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f06040c",
   "metadata": {},
   "source": [
    "# Path Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68a43bee",
   "metadata": {},
   "source": [
    "## Path of Train Data + Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "668d69e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "wandb: Downloading large artifact train:latest, 205.64MB. 2 files... Done. 0:0:0\n"
     ]
    }
   ],
   "source": [
    "# Download Raw Data\r\n",
    "train_data_art = run.use_artifact(\"train:latest\")\r\n",
    "train_data_dir = train_data_art.download()\r\n",
    "\r\n",
    "train_ch = Path(train_data_dir) / \"ch.txt\"\r\n",
    "train_jp = Path(train_data_dir) / \"jp.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "f7eae88f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download Tokenizers\r\n",
    "tokenizer_art = run.use_artifact(\"language_specific:latest\")  # or sentencepiece:latest\r\n",
    "tokenizer_dir = tokenizer_art.download()\r\n",
    "\r\n",
    "tokenizer_ch_dir = Path(tokenizer_dir) / \"jieba_tokenizer.json\"  # or ch_tokenizer.json\r\n",
    "tokenizer_jp_dir = Path(tokenizer_dir) / \"janome_tokenizer.json\"  # or jp_tokenizer.json\r\n",
    "\r\n",
    "tokenizer_ch = Tokenizer.from_file(str(tokenizer_ch_dir))\r\n",
    "tokenizer_jp = Tokenizer.from_file(str(tokenizer_jp_dir))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18f8200e",
   "metadata": {},
   "source": [
    "## Path of Tokenized Train Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "bf1c54e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_path = Path(\"../tokenized_sentences/language_specific\")  # or sentencepiece\r\n",
    "tokenized_path.mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "060df003",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_ch_path = tokenized_path / \"ch.txt\"\r\n",
    "tokenized_jp_path = tokenized_path / \"jp.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "efb9e23d",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_ch_phonetic_path = tokenized_path / \"chp.txt\"\r\n",
    "tokenized_jp_phonetic_path = tokenized_path / \"jpp.txt\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85f44cf9",
   "metadata": {},
   "source": [
    "## Path of Word2Vec Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "65142cba",
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec_path = Path(\"../word2vec/language_specific/\")  # or sentencepiece\r\n",
    "word2vec_path.mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "34af3eaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "semantic_path = word2vec_path / \"semantic/\"\r\n",
    "semantic_path.mkdir(parents=True, exist_ok=True)\r\n",
    "\r\n",
    "ch_semantic_path = semantic_path / \"ch_word2vec\"\r\n",
    "jp_semantic_path = semantic_path / \"jp_word2vec\"\r\n",
    "\r\n",
    "ch_semantic_embedding_path = semantic_path / \"ch_embedding.npy\"\r\n",
    "jp_semantic_embedding_path = semantic_path / \"jp_embedding.npy\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "7ec54a5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "phonetic_path = word2vec_path / \"phonetic/\"\r\n",
    "phonetic_path.mkdir(parents=True, exist_ok=True)\r\n",
    "\r\n",
    "ch_phonetic_path = phonetic_path / \"chp_word2vec\"\r\n",
    "jp_phonetic_path = phonetic_path / \"jpp_word2vec\"\r\n",
    "\r\n",
    "ch_phonetic_embedding_path = phonetic_path / \"chp_embedding.npy\"\r\n",
    "jp_phonetic_embedding_path = phonetic_path / \"jpp_embedding.npy\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ec78a2b",
   "metadata": {},
   "source": [
    "# Tokenize raw sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "e9b1aa1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "if tokenized_ch_path.exists():\r\n",
    "    with open(tokenized_ch_path, \"rb\") as f:\r\n",
    "        ch_texts = pickle.load(f)\r\n",
    "else:\r\n",
    "    with open(train_ch, encoding=\"utf8\") as f:\r\n",
    "        ch_texts = [tokenizer_ch.encode(line).tokens[1:-1] for line in f.readlines()]\r\n",
    "    \r\n",
    "    with open(tokenized_ch_path, \"wb\") as f:\r\n",
    "         pickle.dump(ch_texts, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "405f61b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "if tokenized_jp_path.exists():\r\n",
    "    with open(tokenized_jp_path, \"rb\") as f:\r\n",
    "        jp_texts = pickle.load(f)\r\n",
    "else:\r\n",
    "    with open(train_jp, encoding=\"utf8\") as f:\r\n",
    "        jp_texts = [tokenizer_jp.encode(line).tokens[1:-1] for line in f.readlines()]\r\n",
    "    \r\n",
    "    with open(tokenized_jp_path, \"wb\") as f:\r\n",
    "         pickle.dump(jp_texts, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "759f5e6b",
   "metadata": {},
   "source": [
    "# Tokenize Phonetic Sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab784935",
   "metadata": {},
   "source": [
    "## Chinese"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "5385b702",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dragonmapper import hanzi\r\n",
    "\r\n",
    "def to_zhuyin(word):\r\n",
    "    try:\r\n",
    "        word = hanzi.to_zhuyin(word)\r\n",
    "    except:\r\n",
    "        pass\r\n",
    "    finally:\r\n",
    "        return word\r\n",
    "\r\n",
    "def generate_ch_phonetic_sentences(ch_texts):\r\n",
    "    return [[to_zhuyin(s) for s in line] for line in ch_texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "19cad75d",
   "metadata": {},
   "outputs": [],
   "source": [
    "if tokenized_ch_phonetic_path.exists():\r\n",
    "    with open(tokenized_ch_phonetic_path, 'rb') as f:\r\n",
    "        chp_texts = pickle.load(f)\r\n",
    "else:\r\n",
    "    chp_texts = generate_ch_phonetic_sentences(ch_texts)\r\n",
    "    with open(tokenized_ch_phonetic_path, 'wb') as f:\r\n",
    "        pickle.dump(chp_texts, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c889da83",
   "metadata": {},
   "source": [
    "## Japanese"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "f454595d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pykakasi\r\n",
    "kks = pykakasi.kakasi()\r\n",
    "\r\n",
    "def to_hira(kanji):\r\n",
    "    return \"\".join([item[\"hira\"] for item in kks.convert(kanji)])\r\n",
    "\r\n",
    "def generate_jp_phonetic_sentences(jp_texts):\r\n",
    "    return [[to_hira(s) for s in line] for line in jp_texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "aa48a050",
   "metadata": {},
   "outputs": [],
   "source": [
    "if tokenized_jp_phonetic_path.exists():\r\n",
    "    with open(tokenized_jp_phonetic_path, 'rb') as f:\r\n",
    "        jpp_texts = pickle.load(f)\r\n",
    "else:\r\n",
    "    jpp_texts = generate_jp_phonetic_sentences(jp_texts)\r\n",
    "    with open(tokenized_jp_phonetic_path, 'wb') as f:\r\n",
    "        pickle.dump(jpp_texts, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9035ded4",
   "metadata": {},
   "source": [
    "# Embedding Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "1271d441",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_coverage(vocab, embedding, map_func=None):\n",
    "    \"\"\"\n",
    "    vocab = vocabulary from tokenizer, input=[(voc, index)]\n",
    "    embedding = word2vec embedding\n",
    "    \"\"\"\n",
    "    count = 0\n",
    "    for word in vocab.keys():\n",
    "        if map_func:\n",
    "            word = map_func(word)\n",
    "        try:\n",
    "            if embedding[word] is not None:\n",
    "                count += 1\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    print(f\"{count / len(vocab):.0%} ({count}/{len(vocab)}) is covered.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "01dcc134",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_embedding_matrix(vocab, embedding, map_func=None):\r\n",
    "    \"\"\"\r\n",
    "    vocab = vocabulary from tokenizer, input=[(voc, index)]\r\n",
    "    embedding = word2vec embedding\r\n",
    "    \"\"\"\r\n",
    "    mean = embedding.vectors.mean()\r\n",
    "    std = embedding.vectors.std()\r\n",
    "    print(f\"Mean of embedding: {mean}\")\r\n",
    "    print(f\"std of embedding: {std}\")\r\n",
    "    \r\n",
    "    embed_matrix = np.random.default_rng(42).normal(mean, std, size=(len(vocab), 300))\r\n",
    "    \r\n",
    "    for word, i in vocab.items():\r\n",
    "        if map_func:\r\n",
    "            word = map_func(word)\r\n",
    "            \r\n",
    "        if word in embedding:\r\n",
    "            embed_matrix[i] = embedding[word]\r\n",
    "            \r\n",
    "    return embed_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "c81416b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "class callback(CallbackAny2Vec):\r\n",
    "    '''Callback to print loss after each epoch.'''\r\n",
    "    \r\n",
    "    def __init__(self):\r\n",
    "        self.epoch = 0\r\n",
    "        self.loss_to_be_subed = 0\r\n",
    "        \r\n",
    "    def on_epoch_end(self, model):\r\n",
    "        loss = model.get_latest_training_loss()\r\n",
    "        loss_now = loss - self.loss_to_be_subed\r\n",
    "        self.loss_to_be_subed = loss\r\n",
    "        print(f'Loss after epoch {self.epoch}: {loss_now}')\r\n",
    "        self.epoch += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "e0ac6672",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_word2vec(tokenized_corpus, vector_size=300, max_vocab_size=32000, rule=None):\r\n",
    "    return Word2Vec(\r\n",
    "        tokenized_corpus,\r\n",
    "        vector_size=vector_size,\r\n",
    "        max_vocab_size=max_vocab_size,\r\n",
    "        sg=1,\r\n",
    "        hs=0,\r\n",
    "        negative=5,\r\n",
    "        workers=4,\r\n",
    "        min_count=5,\r\n",
    "        trim_rule=rule,\r\n",
    "        epochs=13,\r\n",
    "        seed=42,\r\n",
    "        compute_loss=True,\r\n",
    "        callbacks=[callback()]\r\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d837b510",
   "metadata": {},
   "source": [
    "# Train Semantic Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "731b7fbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# if any chinese + japanese\n",
    "reg = re.compile(r'[\\u3040-\\u30ff\\u3400-\\u4dbf\\u4e00-\\u9fff\\uf900-\\ufaff\\uff66-\\uff9f]')\n",
    "\n",
    "def semantic_rule(word, count, min_count):\n",
    "    if reg.search(word):\n",
    "        return gensim.utils.RULE_DEFAULT\n",
    "    else:\n",
    "        return gensim.utils.RULE_DISCARD"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "243f9178",
   "metadata": {},
   "source": [
    "## Chinese"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "2dd982f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "if ch_semantic_path.exists():\r\n",
    "    ch_model = Word2Vec.load(str(ch_semantic_path))\r\n",
    "else:\r\n",
    "    ch_model = build_word2vec(ch_texts)\r\n",
    "    ch_model.save(str(ch_semantic_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "93ad0f36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93% (29613/32000) is covered.\n"
     ]
    }
   ],
   "source": [
    "check_coverage(tokenizer_ch.get_vocab(), ch_model.wv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "59397e7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of embedding: -0.0014938187086954713\n",
      "std of embedding: 0.2587750554084778\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(32000, 300)"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if ch_semantic_embedding_path.exists():\r\n",
    "    ch_semantic_embedding = np.load(ch_semantic_embedding_path)\r\n",
    "else:\r\n",
    "    ch_semantic_embedding = build_embedding_matrix(tokenizer_ch.get_vocab(), ch_model.wv)\r\n",
    "    np.save(ch_semantic_embedding_path, ch_semantic_embedding)\r\n",
    "\r\n",
    "ch_semantic_embedding.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b40916a",
   "metadata": {},
   "source": [
    "## Japanese"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "770cb729",
   "metadata": {},
   "outputs": [],
   "source": [
    "if jp_semantic_path.exists():\r\n",
    "    jp_model = Word2Vec.load(str(jp_semantic_path))\r\n",
    "else:\r\n",
    "    jp_model = build_word2vec(jp_texts)\r\n",
    "    jp_model.save(str(jp_semantic_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "414d63aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(29801, 300)\n"
     ]
    }
   ],
   "source": [
    "print(jp_model.wv.vectors.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "e379970a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93% (29801/32000) is covered.\n"
     ]
    }
   ],
   "source": [
    "check_coverage(tokenizer_jp.get_vocab(), jp_model.wv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "fd09dde0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of embedding: -0.002687199506908655\n",
      "std of embedding: 0.2520033121109009\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(32000, 300)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if jp_semantic_embedding_path.exists():\n",
    "    jp_semantic_embedding = np.load(jp_semantic_embedding_path)\n",
    "else:\n",
    "    jp_semantic_embedding = build_embedding_matrix(tokenizer_jp.get_vocab(), jp_model.wv)\n",
    "    np.save(jp_semantic_embedding_path, jp_semantic_embedding)\n",
    "\n",
    "jp_semantic_embedding.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47dc3180",
   "metadata": {},
   "source": [
    "# Train Phonetic Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "86618f5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/16027450/is-there-a-way-to-know-whether-a-unicode-string-contains-any-chinese-japanese-ch\n",
    "import unicodedata\n",
    "\n",
    "def has_zhuyin(s):\n",
    "    for c in s:\n",
    "        try:\n",
    "            if \"BOPOMOFO\" in unicodedata.name(c):\n",
    "                return True\n",
    "        except:\n",
    "            return False\n",
    "\n",
    "def has_hira(s):\n",
    "    for c in s:\n",
    "        try:\n",
    "            if \"HIRAGANA\" in unicodedata.name(c):\n",
    "                return True\n",
    "        except:\n",
    "            return False\n",
    "\n",
    "def phonetic_rule(word, count, min_count):\n",
    "    if has_zhuyin(word) or has_hira(word):\n",
    "        return gensim.utils.RULE_DEFAULT\n",
    "    else:\n",
    "        return gensim.utils.RULE_DISCARD"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ca9c059",
   "metadata": {},
   "source": [
    "## Chinese"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "3df24730",
   "metadata": {},
   "outputs": [],
   "source": [
    "if ch_phonetic_path.exists():\r\n",
    "    chp_model = Word2Vec.load(str(ch_phonetic_path))\r\n",
    "else:\r\n",
    "    chp_model = build_word2vec(chp_texts)\r\n",
    "    chp_model.save(str(ch_phonetic_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "ed60d694",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25412, 300)\n"
     ]
    }
   ],
   "source": [
    "print(chp_model.wv.vectors.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "eef230c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "97% (31068/32000) is covered.\n"
     ]
    }
   ],
   "source": [
    "check_coverage(tokenizer_ch.get_vocab(), chp_model.wv, map_func=to_zhuyin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "d10dafcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of embedding: 0.0009710414451546967\n",
      "std of embedding: 0.2592071294784546\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(32000, 300)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if ch_phonetic_embedding_path.exists():\n",
    "    ch_phonetic_embedding = np.load(ch_phonetic_embedding_path)\n",
    "else:\n",
    "    ch_phonetic_embedding = build_embedding_matrix(tokenizer_ch.get_vocab(), chp_model.wv, map_func=to_zhuyin)\n",
    "    np.save(ch_phonetic_embedding_path, ch_phonetic_embedding)\n",
    "\n",
    "ch_phonetic_embedding.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be4aa3dd",
   "metadata": {},
   "source": [
    "## Japanese"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "07315cc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "if jp_phonetic_path.exists():\r\n",
    "    jpp_model = Word2Vec.load(str(jp_phonetic_path))\r\n",
    "else:\r\n",
    "    jpp_model = build_word2vec(jpp_texts)\r\n",
    "    jpp_model.save(str(jp_phonetic_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "8626a7e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(24742, 300)\n"
     ]
    }
   ],
   "source": [
    "print(jpp_model.wv.vectors.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "25ed0358",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "96% (30809/32000) is covered.\n"
     ]
    }
   ],
   "source": [
    "check_coverage(tokenizer_jp.get_vocab(), jpp_model.wv, map_func=to_hira)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "03638cf9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of embedding: -0.003116233041509986\n",
      "std of embedding: 0.25073009729385376\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(32000, 300)"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if jp_phonetic_embedding_path.exists():\n",
    "    jp_phonetic_embedding = np.load(jp_phonetic_embedding_path)\n",
    "else:\n",
    "    jp_phonetic_embedding = build_embedding_matrix(tokenizer_jp.get_vocab(), jpp_model.wv, map_func=to_hira)\n",
    "    np.save(jp_phonetic_embedding_path, jp_phonetic_embedding)\n",
    "\n",
    "jp_phonetic_embedding.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88cfc350",
   "metadata": {},
   "source": [
    "# Concatenate Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "a1d41ea1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32000, 300)\n",
      "(32000, 300)\n",
      "(32000, 300)\n",
      "(32000, 300)\n"
     ]
    }
   ],
   "source": [
    "print(ch_semantic_embedding.shape)\n",
    "print(jp_semantic_embedding.shape)\n",
    "\n",
    "print(ch_phonetic_embedding.shape)\n",
    "print(jp_phonetic_embedding.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "0a497d15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32000, 600)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ch_embedding = np.concatenate([ch_phonetic_embedding, ch_semantic_embedding], axis=1)\r\n",
    "ch_embedding.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "49f835d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32000, 600)"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jp_embedding = np.concatenate([jp_phonetic_embedding, jp_semantic_embedding], axis=1)\r\n",
    "jp_embedding.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "419bf965",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(word2vec_path / \"ch-embedding-concat.npy\", ch_embedding)\n",
    "np.save(word2vec_path / \"jp-embedding-concat.npy\", jp_embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14cbd00d",
   "metadata": {},
   "source": [
    "# Meta Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "d495ad30",
   "metadata": {},
   "outputs": [],
   "source": [
    "beta = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "73ef2a32",
   "metadata": {},
   "outputs": [],
   "source": [
    "ch_meta_embedding = np.mean(\n",
    "    [(2 - (beta * 2)) * ch_semantic_embedding, (beta * 2) * ch_phonetic_embedding],\n",
    "    axis=0,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "f8311382",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32000, 300)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ch_meta_embedding.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "2e990832",
   "metadata": {},
   "outputs": [],
   "source": [
    "jp_meta_embedding = np.mean(\n",
    "    [(2 - (beta * 2)) * jp_semantic_embedding, (beta * 2) * jp_phonetic_embedding],\n",
    "    axis=0,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "bdcfce35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32000, 300)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jp_meta_embedding.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "e2e6652f",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(word2vec_path / f\"ch-embedding-meta-{beta=}.npy\", ch_meta_embedding)\n",
    "np.save(word2vec_path / f\"jp-embedding-meta-{beta=}.npy\", jp_meta_embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eed884c3",
   "metadata": {},
   "source": [
    "# Save Embedding Artifact"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "f25db0f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<wandb.sdk.wandb_artifacts.Artifact at 0x2411dcaa040>"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word2vec_art = wandb.Artifact('language_specific_word2vec', \"word2vec\")  # or sentencepiece_word2vec\r\n",
    "word2vec_art.add_file(ch_semantic_path, \"ch.word2vec\")\r\n",
    "word2vec_art.add_file(jp_semantic_path, \"jp.word2vec\")\r\n",
    "word2vec_art.add_file(ch_phonetic_path, \"chp.word2vec\")\r\n",
    "word2vec_art.add_file(jp_phonetic_path, \"jpp.word2vec\")\r\n",
    "\r\n",
    "run.log_artifact(word2vec_art)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "fda1814e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<wandb.sdk.wandb_artifacts.Artifact at 0x2411e6649a0>"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_art = wandb.Artifact(\"language_specific_embedding\", \"embedding\")  # or sentencepiece_embedding\r\n",
    "embedding_art.add_file(ch_semantic_embedding_path, \"ch_embedding.npy\")\r\n",
    "embedding_art.add_file(ch_phonetic_embedding_path, \"chp_embedding.npy\")\r\n",
    "\r\n",
    "embedding_art.add_file(jp_semantic_embedding_path, \"jp_embedding.npy\")\r\n",
    "embedding_art.add_file(jp_phonetic_embedding_path, \"jpp_embedding.npy\")\r\n",
    "\r\n",
    "embedding_art.add_file(word2vec_path / \"ch-embedding-concat.npy\", \"ch_concat_embedding.npy\")\r\n",
    "embedding_art.add_file(word2vec_path / \"jp-embedding-concat.npy\", \"jp_concat_embedding.npy\")\r\n",
    "\r\n",
    "embedding_art.add_file(word2vec_path / \"ch-embedding-meta-beta=0.5.npy\", \"ch_meta_embedding.npy\")\r\n",
    "embedding_art.add_file(word2vec_path / \"jp-embedding-meta-beta=0.5.npy\", \"jp_meta_embedding.npy\")\r\n",
    "\r\n",
    "run.log_artifact(embedding_art)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "b765c094",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<br/>Waiting for W&B process to finish, PID 10832<br/>Program ended successfully."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86955e7c77884c48b81f7569d9d1e4d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value=' 732.42MB of 732.42MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find user logs for this run at: <code>d:\\Project\\phonetics-in-chinese-japanese-machine-translation\\experiments\\main\\wandb\\run-20210510_162721-2q4igpuu\\logs\\debug.log</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find internal logs for this run at: <code>d:\\Project\\phonetics-in-chinese-japanese-machine-translation\\experiments\\main\\wandb\\run-20210510_162721-2q4igpuu\\logs\\debug-internal.log</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Synced 5 W&B file(s), 0 media file(s), 8 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                    <br/>Synced <strong style=\"color:#cdcd00\">language_specific</strong>: <a href=\"https://wandb.ai/windsuzu/phonetic-translation/runs/2q4igpuu\" target=\"_blank\">https://wandb.ai/windsuzu/phonetic-translation/runs/2q4igpuu</a><br/>\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "run.finish()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "metadata": {
   "interpreter": {
    "hash": "910af126f78e4f70975a50f5d0344a29878143e0b01cc32c99ca6cf65dbefcc1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}