discussion/nngen_improvement.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retrieval Index Experiments on Nngen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NNGen-Smooth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>NNGen-Smooth</th>\n",
       "      <td>16.47</td>\n",
       "      <td>23.12</td>\n",
       "      <td>16.80</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             B-Moses B-Norm  B-CC\n",
       "NNGen-Smooth   16.47  23.12 16.80"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = pd.DataFrame(columns=['B-Moses', 'B-Norm', 'B-CC'], index=[\"NNGen-Smooth\"])\n",
    "model_name = \"NNGenPlus\"\n",
    "dataset_name = \"NNGen\"\n",
    "ref_path = \"../experiment_results/model_{}/data_{}/Smooth/ref.msg\".format(model_name,dataset_name)\n",
    "gen_path = \"../experiment_results/model_{}/data_{}/Smooth/gen.msg\".format(model_name,dataset_name)\n",
    "BM_evaluate_cmd = \"../metrics/B-Moses.perl {} < {}\".format(ref_path, gen_path)\n",
    "results['B-Moses'][\"NNGen-Smooth\"] = float(os.popen(BM_evaluate_cmd).read().split(\"=\")[1].split(\",\")[0].strip())\n",
    "BN_evaluate_cmd = \"python ../metrics/B-Norm.py {} < {}\".format(ref_path, gen_path)\n",
    "results['B-Norm'][\"NNGen-Smooth\"] = float(os.popen(BN_evaluate_cmd).read())\n",
    "BC_evaluate_cmd = \"python ../metrics/B-CC.py --ref_path {} --gen_path {}\".format(ref_path, gen_path)\n",
    "results['B-CC'][\"NNGen-Smooth\"] = float(os.popen(BC_evaluate_cmd).read().strip())\n",
    "pd.options.display.float_format = '{:.2f}'.format\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NNGen-Gram4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>NNGen-Gram4</th>\n",
       "      <td>17.26</td>\n",
       "      <td>23.92</td>\n",
       "      <td>17.60</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            B-Moses B-Norm  B-CC\n",
       "NNGen-Gram4   17.26  23.92 17.60"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = pd.DataFrame(columns=['B-Moses', 'B-Norm', 'B-CC'], index=[\"NNGen-Gram4\"])\n",
    "model_name = \"NNGenPlus\"\n",
    "dataset_name = \"NNGen\"\n",
    "ref_path = \"../experiment_results/model_{}/data_{}/Gram/ref.msg\".format(model_name,dataset_name)\n",
    "gen_path = \"../experiment_results/model_{}/data_{}/Gram/gen.msg\".format(model_name,dataset_name)\n",
    "BM_evaluate_cmd = \"../metrics/B-Moses.perl {} < {}\".format(ref_path, gen_path)\n",
    "results['B-Moses'][\"NNGen-Gram4\"] = float(os.popen(BM_evaluate_cmd).read().split(\"=\")[1].split(\",\")[0].strip())\n",
    "BN_evaluate_cmd = \"python ../metrics/B-Norm.py {} < {}\".format(ref_path, gen_path)\n",
    "results['B-Norm'][\"NNGen-Gram4\"] = float(os.popen(BN_evaluate_cmd).read())\n",
    "BC_evaluate_cmd = \"python ../metrics/B-CC.py --ref_path {} --gen_path {}\".format(ref_path, gen_path)\n",
    "results['B-CC'][\"NNGen-Gram4\"] = float(os.popen(BC_evaluate_cmd).read().strip())\n",
    "pd.options.display.float_format = '{:.2f}'.format\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NNGen-Smooth-Gram4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B-Moses</th>\n",
       "      <th>B-Norm</th>\n",
       "      <th>B-CC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>NNGen-Smooth-Gram4</th>\n",
       "      <td>17.48</td>\n",
       "      <td>24.04</td>\n",
       "      <td>17.63</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   B-Moses B-Norm  B-CC\n",
       "NNGen-Smooth-Gram4   17.48  24.04 17.63"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = pd.DataFrame(columns=['B-Moses', 'B-Norm', 'B-CC'], index=[\"NNGen-Smooth-Gram4\"])\n",
    "model_name = \"NNGenPlus\"\n",
    "dataset_name = \"NNGen\"\n",
    "ref_path = \"../experiment_results/model_{}/data_{}/SmoothGram/ref.msg\".format(model_name,dataset_name)\n",
    "gen_path = \"../experiment_results/model_{}/data_{}/SmoothGram/gen.msg\".format(model_name,dataset_name)\n",
    "BM_evaluate_cmd = \"../metrics/B-Moses.perl {} < {}\".format(ref_path, gen_path)\n",
    "results['B-Moses'][\"NNGen-Smooth-Gram4\"] = float(os.popen(BM_evaluate_cmd).read().split(\"=\")[1].split(\",\")[0].strip())\n",
    "BN_evaluate_cmd = \"python ../metrics/B-Norm.py {} < {}\".format(ref_path, gen_path)\n",
    "results['B-Norm'][\"NNGen-Smooth-Gram4\"] = float(os.popen(BN_evaluate_cmd).read())\n",
    "BC_evaluate_cmd = \"python ../metrics/B-CC.py --ref_path {} --gen_path {}\".format(ref_path, gen_path)\n",
    "results['B-CC'][\"NNGen-Smooth-Gram4\"] = float(os.popen(BC_evaluate_cmd).read().strip())\n",
    "pd.options.display.float_format = '{:.2f}'.format\n",
    "results"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}