[MRG] Lda training visualization in visdom

docs/notebooks/Training_visualizations.ipynb

@@ -44,9 +44,11 @@
     "from gensim.models import ldamodel\n",
     "from gensim.corpora.dictionary import Dictionary\n",
     "\n",
-    "# Set file names for train data\n",
+    "\n",
+    "# Set file names for train and test data\n",
     "test_data_dir = '{}'.format(os.sep).join([gensim.__path__[0], 'test', 'test_data'])\n",
-    "lee_corpus = test_data_dir + os.sep + 'lee.cor'\n",
+    "lee_train_file = test_data_dir + os.sep + 'lee_background.cor'\n",
+    "lee_test_file = test_data_dir + os.sep + 'lee.cor'\n",
     "\n",
     "def read_corpus(fname):\n",
     "    texts = []\n",
@@ -59,12 +61,12 @@
     "            texts.append(words)\n",
     "    return texts\n",
     "\n",
-    "texts = read_corpus(lee_corpus)\n",
+    "training_texts = read_corpus(lee_train_file)\n",
+    "eval_texts = read_corpus(lee_test_file)\n",
     "\n",
     "# Split test data into hold_out and test corpus\n",
-    "training_texts = texts[:25]\n",
-    "holdout_texts = texts[25:40]\n",
-    "test_texts = texts[40:50]\n",
+    "holdout_texts = eval_texts[:25]\n",
+    "test_texts = eval_texts[25:]\n",
     "\n",
     "training_dictionary = Dictionary(training_texts)\n",
     "holdout_dictionary = Dictionary(holdout_texts)\n",
@@ -78,26 +80,25 @@
   {
    "cell_type": "code",
    "execution_count": 2,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from gensim.models.callbacks import CoherenceMetric, DiffMetric, PerplexityMetric, ConvergenceMetric\n",
     "\n",
     "# define perplexity callback for hold_out and test corpus\n",
-    "pl_holdout = PerplexityMetric(corpus=holdout_corpus, logger=\"visdom\", viz_env=\"LdaModel\", title=\"Perplexity (hold_out)\")\n",
-    "pl_test = PerplexityMetric(corpus=test_corpus, logger=\"visdom\", viz_env=\"LdaModel\", title=\"Perplexity (test)\")\n",
+    "pl_holdout = PerplexityMetric(corpus=holdout_corpus, logger=\"visdom\", title=\"Perplexity (hold_out)\")\n",
+    "pl_test = PerplexityMetric(corpus=test_corpus, logger=\"visdom\", title=\"Perplexity (test)\")\n",
     "\n",
     "# define other remaining metrics available\n",
-    "ch_umass = CoherenceMetric(corpus=training_corpus, coherence=\"u_mass\", logger=\"visdom\", viz_env=\"LdaModel\", title=\"Coherence (u_mass)\")\n",
-    "diff_kl = DiffMetric(distance=\"kullback_leibler\", logger=\"visdom\", viz_env=\"LdaModel\", title=\"Diff (kullback_leibler)\")\n",
-    "convergence_jc = ConvergenceMetric(distance=\"hellinger\", logger=\"visdom\", viz_env=\"LdaModel\", title=\"Convergence (jaccard)\")\n",
+    "ch_umass = CoherenceMetric(corpus=training_corpus, coherence=\"u_mass\", logger=\"visdom\", title=\"Coherence (u_mass)\")\n",
+    "ch_cv = CoherenceMetric(corpus=training_corpus, texts=training_texts, coherence=\"c_v\", logger=\"visdom\", title=\"Coherence (c_v)\")\n",
+    "diff_kl = DiffMetric(distance=\"kullback_leibler\", logger=\"visdom\", title=\"Diff (kullback_leibler)\")\n",
+    "convergence_hl = ConvergenceMetric(distance=\"hellinger\", logger=\"visdom\", title=\"Convergence (hellinger)\")\n",
     "\n",
-    "callbacks = [pl_holdout, pl_test, ch_umass, diff_kl, convergence_jc]\n",
+    "callbacks = [pl_holdout, pl_test, ch_umass, ch_cv, diff_kl, convergence_hl]\n",
     "\n",
     "# training LDA model\n",
-    "model = ldamodel.LdaModel(corpus=training_corpus, id2word=training_dictionary, passes=5, num_topics=5, callbacks=callbacks)"
+    "model = ldamodel.LdaModel(corpus=training_corpus, id2word=training_dictionary, passes=3, num_topics=5, callbacks=callbacks)"
    ]
   },
   {
@@ -116,7 +117,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "-22.4298221364\n"
+      "-0.259766196856\n"
      ]
     }
    ],
@@ -255,6 +256,42 @@
     "# training LDA model\n",
     "model = ldamodel.LdaModel(corpus=training_corpus, id2word=training_dictionary, passes=3, num_topics=5, callbacks=callbacks)"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The metric values can also be accessed from the model instance for custom uses."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'CoherenceMetric': [0.33266605793626819,\n",
+       "  0.3316839843742313,\n",
+       "  0.33237246830927009],\n",
+       " 'ConvergenceMetric': [0.0, 0.0, 0.0],\n",
+       " 'DiffMetric': [array([ 0.92795546,  0.83166895,  0.8926528 ,  0.96382424,  0.98886188]),\n",
+       "  array([ 0.1486518 ,  0.16031907,  0.18798994,  0.13619778,  0.11326997]),\n",
+       "  array([ 0.02155673,  0.03477041,  0.03180156,  0.02133546,  0.01840971])],\n",
+       " 'PerplexityMetric': [2374469.2517599338,\n",
+       "  1708181.2721127137,\n",
+       "  1485456.3900059697]}"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.metrics"
+   ]
   }
  ],
  "metadata": {

gensim/models/callbacks.py

@@ -24,19 +24,22 @@ class Metric(object):
     def __init__(self):
         pass
 
-    def get_value(self, **parameters):
+    def set_parameters(self, **parameters):
         """
         Set the parameters
         """
         for parameter, value in parameters.items():
             setattr(self, parameter, value)
 
+    def get_value(self):
+        pass
+
 
 class CoherenceMetric(Metric):
     """
     Metric class for coherence evaluation
     """
-    def __init__(self, corpus=None, texts=None, dictionary=None, coherence=None, window_size=None, topn=None, logger="shell", viz_env=None, title=None):
+    def __init__(self, corpus=None, texts=None, dictionary=None, coherence=None, window_size=None, topn=10, logger=None, viz_env=None, title=None):
         """
         Args:
             corpus : Gensim document corpus.
@@ -98,7 +101,7 @@ def get_value(self, **kwargs):
         # only one of the model or topic would be defined
         self.model = None
         self.topics = None
-        super(CoherenceMetric, self).get_value(**kwargs)
+        super(CoherenceMetric, self).set_parameters(**kwargs)
         cm = gensim.models.CoherenceModel(self.model, self.topics, self.texts, self.corpus, self.dictionary, self.window_size, self.coherence, self.topn)
         return cm.get_coherence()
 
@@ -107,7 +110,7 @@ class PerplexityMetric(Metric):
     """
     Metric class for perplexity evaluation
     """
-    def __init__(self, corpus=None, logger="shell", viz_env=None, title=None):
+    def __init__(self, corpus=None, logger=None, viz_env=None, title=None):
         """
         Args:
             corpus : Gensim document corpus
@@ -127,7 +130,7 @@ def get_value(self, **kwargs):
         Args:
             model : Trained topic model
         """
-        super(PerplexityMetric, self).get_value(**kwargs)
+        super(PerplexityMetric, self).set_parameters(**kwargs)
         corpus_words = sum(cnt for document in self.corpus for _, cnt in document)
         perwordbound = self.model.bound(self.corpus) / corpus_words
         return np.exp2(-perwordbound)
@@ -137,7 +140,7 @@ class DiffMetric(Metric):
     """
     Metric class for topic difference evaluation
     """
-    def __init__(self, distance="jaccard", num_words=100, n_ann_terms=10, normed=True, logger="shell", viz_env=None, title=None):
+    def __init__(self, distance="jaccard", num_words=100, n_ann_terms=10, normed=True, logger=None, viz_env=None, title=None):
         """
         Args:
             distance : measure used to calculate difference between any topic pair. Available values:
@@ -167,7 +170,7 @@ def get_value(self, **kwargs):
             model : Trained topic model
             other_model : second topic model instance to calculate the difference from
         """
-        super(DiffMetric, self).get_value(**kwargs)
+        super(DiffMetric, self).set_parameters(**kwargs)
         diff_matrix, _ = self.model.diff(self.other_model, self.distance, self.num_words, self.n_ann_terms, self.normed)
         return np.diagonal(diff_matrix)
 
@@ -176,7 +179,7 @@ class ConvergenceMetric(Metric):
     """
     Metric class for convergence evaluation
     """
-    def __init__(self, distance="jaccard", num_words=100, n_ann_terms=10, normed=True, logger="shell", viz_env=None, title=None):
+    def __init__(self, distance="jaccard", num_words=100, n_ann_terms=10, normed=True, logger=None, viz_env=None, title=None):
         """
         Args:
             distance : measure used to calculate difference between any topic pair. Available values:
@@ -206,7 +209,7 @@ def get_value(self, **kwargs):
             model : Trained topic model
             other_model : second topic model instance to calculate the difference from
         """
-        super(ConvergenceMetric, self).get_value(**kwargs)
+        super(ConvergenceMetric, self).set_parameters(**kwargs)
         diff_matrix, _ = self.model.diff(self.other_model, self.distance, self.num_words, self.n_ann_terms, self.normed)
         return np.sum(np.diagonal(diff_matrix))
 
@@ -257,10 +260,16 @@ def on_epoch_end(self, epoch, topics=None):
             epoch : current epoch no.
             topics : topic distribution from current epoch (required for coherence of unsupported topic models)
         """
+        # stores current epoch's metric values
+        current_metrics = {}
+
         # plot all metrics in current epoch
         for i, metric in enumerate(self.metrics):
             value = metric.get_value(topics=topics, model=self.model, other_model=self.previous)
-            metric_label = type(metric).__name__[:-6]
+            metric_label = type(metric).__name__
+
+            current_metrics[metric_label] = value
+
             # check for any metric which need model state from previous epoch
             if isinstance(metric, (DiffMetric, ConvergenceMetric)):
                 self.previous = copy.deepcopy(self.model)
@@ -269,24 +278,27 @@ def on_epoch_end(self, epoch, topics=None):
                 if epoch == 0:
                     if value.ndim > 0:
                         diff_mat = np.array([value])
-                        viz_metric = self.viz.heatmap(X=diff_mat.T, env=metric.viz_env, opts=dict(xlabel='Epochs', ylabel=metric_label, title=metric.title))
+                        viz_metric = self.viz.heatmap(X=diff_mat.T, env=metric.viz_env, opts=dict(xlabel='Epochs', ylabel=metric_label[:-6], title=metric.title))
                         # store current epoch's diff diagonal
                         self.diff_mat.put(diff_mat)
                         # saving initial plot window
                         self.windows.append(copy.deepcopy(viz_metric))
                     else:
-                        viz_metric = self.viz.line(Y=np.array([value]), X=np.array([epoch]), env=metric.viz_env, opts=dict(xlabel='Epochs', ylabel=metric_label, title=metric.title))
+                        viz_metric = self.viz.line(Y=np.array([value]), X=np.array([epoch]), env=metric.viz_env, opts=dict(xlabel='Epochs', ylabel=metric_label[:-6], title=metric.title))
                         # saving initial plot window
                         self.windows.append(copy.deepcopy(viz_metric))
                 else:
                     if value.ndim > 0:
                         # concatenate with previous epoch's diff diagonals
                         diff_mat = np.concatenate((self.diff_mat.get(), np.array([value])))
-                        self.viz.heatmap(X=diff_mat.T, env=metric.viz_env, win=self.windows[i], opts=dict(xlabel='Epochs', ylabel=metric_label, title=metric.title))
+                        self.viz.heatmap(X=diff_mat.T, env=metric.viz_env, win=self.windows[i], opts=dict(xlabel='Epochs', ylabel=metric_label[:-6], title=metric.title))
                         self.diff_mat.put(diff_mat)
                     else:
                         self.viz.updateTrace(Y=np.array([value]), X=np.array([epoch]), env=metric.viz_env, win=self.windows[i])
 
             if metric.logger == "shell":
-                statement = "".join(("Epoch ", str(epoch), ": ", metric_label, " estimate: ", str(value)))
+                statement = "".join(("Epoch ", str(epoch), ": ", metric_label[:-6], " estimate: ", str(value)))
                 self.log_type.info(statement)
+
+        return current_metrics
+

gensim/models/ldamodel.py

@@ -631,8 +631,13 @@ def rho():
             return pow(offset + pass_ + (self.num_updates / chunksize), -decay)
 
         if self.callbacks:
+            # pass the list of input callbacks to Callback class
             callback = Callback(self.callbacks)
             callback.set_model(self)
+            # initialize metrics dict to store metric values after every epoch
+            self.metrics = {}
+            for metric in self.callbacks:
+                self.metrics[type(metric).__name__] = []
 
         for pass_ in xrange(passes):
             if self.dispatcher:
@@ -686,8 +691,11 @@ def rho():
             if reallen != lencorpus:
                 raise RuntimeError("input corpus size changed during training (don't use generators as input)")
 
+            # append current epoch's metric values
             if self.callbacks:
-                callback.on_epoch_end(pass_)
+                current_metrics = callback.on_epoch_end(pass_)
+                for metric, value in current_metrics.items():
+                    self.metrics[metric].append(value)
 
             if dirty:
                 # finish any remaining updates