From e2c72faf2a35fabeaed5a1af5355298ab5332096 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Radim=20=C5=98eh=C5=AF=C5=99ek?= <radimrehurek@seznam.cz>
Date: Thu, 21 Jun 2018 17:33:34 +0200
Subject: [PATCH] add back smartirs tests

---
 gensim/test/test_data/tfidf_model.tst     | Bin 0 -> 1261 bytes
 gensim/test/test_data/tfidf_model.tst.bz2 | Bin 0 -> 822 bytes
 gensim/test/test_tfidfmodel.py            |  42 ++++++++++++++++++++++
 3 files changed, 42 insertions(+)
 create mode 100644 gensim/test/test_data/tfidf_model.tst
 create mode 100644 gensim/test/test_data/tfidf_model.tst.bz2

diff --git a/gensim/test/test_data/tfidf_model.tst b/gensim/test/test_data/tfidf_model.tst
new file mode 100644
index 0000000000000000000000000000000000000000..8d3c60c73ed35f9d341a1a9e65fd490301539739
GIT binary patch
literal 1261
zcma)++fvg&7{`00ZHfm}JRlxXDJoU)gbIR3l$EM0YE)`WleCGXN&ndt24<Mi3*sX<
zUU}hDc;OA+>RWi_OX%*>ah!27!)7+S+5PQz-n>m24a;Lrv)uGe%Vp)T?wEBsSLVcY
zRxB$>4822g6{(V%OC%CryQz8J3eu!sP4jQF&X8F}wnP#nMUuRWC(V0?_bi`v@iE88
zJRb{uRCq6vR9nsR<1J^=57AXo75<@VPQ&v9%Vfw^kyi^spC2@J*IBVpFzRj32z}pW
zN=pwyN9V>V@jkR$rXE_R_E&;};u^Xswv_DR&?>e{`#sIWHZ{pL8Q9J$A$n@kPsbcx
z@&gk)DiX6=3_GVM@ZsaT;a@#!ue3I>jCR1BMN4D)qUHKJH+)wWy=q==^H8;x8Fp8(
zN3Mv_nZCiWmlULqrH1R*bQk;VO}Np474)rPKSh7Z?&D4zP%|P(npwv|=|zY`!i&Q-
zxxur@aAXxnd3p?KPubuBE;rjQk6#ez%L+4eR}T~nZ0sv$xZJW798*(VaD5ENd190A
zae`v7MAA~nI7*dDW#%_;@<kFMhH8dp@PaXzVtHbs;G|mQBJB0A>z~#?C+Cw>IHh*+
z6;}@(Zv>|!3{#XNoS`^NlJ<Z-X#am?#OB;6#kmOQDK3yy40lm*mng;}T&B1}(lOjs
z!HrW)M7TyVNis3ql;EyY+=y_K;ugtvxbA;l`u1fKw*`2IVmiWIihHChM&OL#2*v#f
zvlJDQixc=jfDb7iMW7UqNj^s4oZy~NJdN;-LM4SbfzJi_g5qU_d5S7g;=oq|)F@s@
z&?#!97^B$`oJnCts8clBvbtGQ+{u8!-ndtx0Y|2L$@O`O;mxLsT(Sxl*5Fb!OZEb<
taj#S3VI97#KMyTY<EY7B2y^|Gg<yK}=XcS5bZ#Fr9C7<t6=Azp{sWbRoVWl0

literal 0
HcmV?d00001

diff --git a/gensim/test/test_data/tfidf_model.tst.bz2 b/gensim/test/test_data/tfidf_model.tst.bz2
new file mode 100644
index 0000000000000000000000000000000000000000..f25e0399f443a1d9261378115f290afa0ca9177b
GIT binary patch
literal 822
zcmV-61IheCT4*^jL0KkKS?3V!e*gkE|NZ~}|NsC0|NsC0|NsC0|FUC%0HBBf03sL$
z1P}lKzyg<a<C1c4Y7$J*4^Re~9-{&=iZlQX83F1IG{`YA4Kx5W0QCR>0iguRHi~%x
zA?h?~p^>#T$)GeeXaE2MLn8zLWMmp-4Ff_df0U{Enw!-F)Mx+z0004?0000001W^%
zGy$LiibXU44FDPd27mwn00000001-q02vN9)ipFWKm-7iISr==DeZtoswe^=J%_3~
z0RS^3l!ka3U<SSbDZ>4JX`ly^4FHH~1{4^;n1ulaz#yUm2to+vQ`CO>VQBO|C{`Rn
zq9aWt13)4lAQ*^k7SSvuAQ2(WHCu9SoxE++w~_6_5)ed%=@kG3Twol|0PI~Tj-*5$
zlMmPoWXJ&nODMo6Fv5Zi(NhK|2g$uk8Y|o;qx2&$j#+4Hol<t`&=g(BR^@6-1|}*P
z4BV1{3%Dx7i5EjkJegB9rtCo4+}_~gpOwwge%V^(3ImvnNSLe0*(piJ?>cJ6e|@L>
zEci%H0Fs%3jsPh@EGcs@R|P;?sw=X(QioL0LLqT)Zt(DlvtElFONUOqFB#Q<%9R{;
zc~@lSWmc(IF<(m(H7mx2T$n@_LGDDT+fw~sVY+CNk)Z_`4blQb&p_qoS&#=16rmCi
zF-pIG%^>Zq#F%MFkdunW<RIB01pZ!rp1#i1d_0k@XxIW7ur?t9UZ?`*1v9H~d;+Eu
zAxL8~C^_(W9<h4BH*UNE1)&t+UjVW;HbQeKfvnA78Bi5{6yz;1VDn6hdd;ZUmHpMR
zd0#ZP&?+6~`(W5g6T<k1-=h^2-(E1l1Zx<^NjgT9<sZOnIi=9jKym_!0FnVD0!SO=
zfTBQAASjR&NCc20;FO>tVw-f@;v07l7RP>5jon!nCQG=of}JuSFP+^e>m2|?WGZYN
zcQ9L36s5KTHcYg29Jm>r(Ba-|HnR|(nMqn>o>M$}tU5*-Bg5ix#);VdCLVr%lTXo7
zsio?>SF-nC3*~((H#1*<zykCH0eS&?0k;6{f%;!TS#`b#2!D&YBAh5lbBK07fVTEa
Ap#T5?

literal 0
HcmV?d00001

diff --git a/gensim/test/test_tfidfmodel.py b/gensim/test/test_tfidfmodel.py
index 3864076093..79e3742d48 100644
--- a/gensim/test/test_tfidfmodel.py
+++ b/gensim/test/test_tfidfmodel.py
@@ -89,6 +89,17 @@ def test_persistence(self):
         self.assertTrue(np.allclose(model[tstvec[1]], model2[tstvec[1]]))
         self.assertTrue(np.allclose(model[[]], model2[[]]))  # try projecting an empty vector
 
+        # Test persistence between Gensim v3.2.0 and current model.
+        model3 = tfidfmodel.TfidfModel(self.corpus, smartirs="ntc")
+        model4 = tfidfmodel.TfidfModel.load(datapath('tfidf_model.tst'))
+        idfs3 = [model3.idfs[key] for key in sorted(model3.idfs.keys())]
+        idfs4 = [model4.idfs[key] for key in sorted(model4.idfs.keys())]
+        self.assertTrue(np.allclose(idfs3, idfs4))
+        tstvec = [corpus[1], corpus[2]]
+        self.assertTrue(np.allclose(model3[tstvec[0]], model4[tstvec[0]]))
+        self.assertTrue(np.allclose(model3[tstvec[1]], model4[tstvec[1]]))
+        self.assertTrue(np.allclose(model3[[]], model4[[]]))  # try projecting an empty vector
+
         # Test persistence with using pivoted normalization
         fname = get_tmpfile('gensim_models_smartirs.tst')
         model = tfidfmodel.TfidfModel(self.corpus, pivot=0, slope=1)
@@ -99,6 +110,16 @@ def test_persistence(self):
         self.assertTrue(np.allclose(model[tstvec[0]], model2[tstvec[0]]))
         self.assertTrue(np.allclose(model[tstvec[1]], model2[tstvec[1]]))
 
+        # Test persistence between Gensim v3.2.0 and pivoted normalization compressed model.
+        model3 = tfidfmodel.TfidfModel(self.corpus, pivot=0, slope=1)
+        model4 = tfidfmodel.TfidfModel.load(datapath('tfidf_model.tst'))
+        idfs3 = [model3.idfs[key] for key in sorted(model3.idfs.keys())]
+        idfs4 = [model4.idfs[key] for key in sorted(model4.idfs.keys())]
+        self.assertTrue(np.allclose(idfs3, idfs4))
+        tstvec = [corpus[1], corpus[2]]
+        self.assertTrue(np.allclose(model3[tstvec[0]], model4[tstvec[0]]))
+        self.assertTrue(np.allclose(model3[tstvec[1]], model4[tstvec[1]]))
+
     def test_persistence_compressed(self):
         # Test persistence without using `smartirs`
         fname = get_tmpfile('gensim_models.tst.gz')
@@ -122,6 +143,17 @@ def test_persistence_compressed(self):
         self.assertTrue(np.allclose(model[tstvec[1]], model2[tstvec[1]]))
         self.assertTrue(np.allclose(model[[]], model2[[]]))  # try projecting an empty vector
 
+        # Test persistence between Gensim v3.2.0 and current compressed model.
+        model3 = tfidfmodel.TfidfModel(self.corpus, smartirs="ntc")
+        model4 = tfidfmodel.TfidfModel.load(datapath('tfidf_model.tst.bz2'))
+        idfs3 = [model3.idfs[key] for key in sorted(model3.idfs.keys())]
+        idfs4 = [model4.idfs[key] for key in sorted(model4.idfs.keys())]
+        self.assertTrue(np.allclose(idfs3, idfs4))
+        tstvec = [corpus[1], corpus[2]]
+        self.assertTrue(np.allclose(model3[tstvec[0]], model4[tstvec[0]]))
+        self.assertTrue(np.allclose(model3[tstvec[1]], model4[tstvec[1]]))
+        self.assertTrue(np.allclose(model3[[]], model4[[]]))  # try projecting an empty vector
+
         # Test persistence with using pivoted normalization
         fname = get_tmpfile('gensim_models_smartirs.tst.gz')
         model = tfidfmodel.TfidfModel(self.corpus, pivot=0, slope=1)
@@ -132,6 +164,16 @@ def test_persistence_compressed(self):
         self.assertTrue(np.allclose(model[tstvec[0]], model2[tstvec[0]]))
         self.assertTrue(np.allclose(model[tstvec[1]], model2[tstvec[1]]))
 
+        # Test persistence between Gensim v3.2.0 and pivoted normalization compressed model.
+        model3 = tfidfmodel.TfidfModel(self.corpus, pivot=0, slope=1)
+        model4 = tfidfmodel.TfidfModel.load(datapath('tfidf_model.tst.bz2'))
+        idfs3 = [model3.idfs[key] for key in sorted(model3.idfs.keys())]
+        idfs4 = [model4.idfs[key] for key in sorted(model4.idfs.keys())]
+        self.assertTrue(np.allclose(idfs3, idfs4))
+        tstvec = [corpus[1], corpus[2]]
+        self.assertTrue(np.allclose(model3[tstvec[0]], model4[tstvec[0]]))
+        self.assertTrue(np.allclose(model3[tstvec[1]], model4[tstvec[1]]))
+
     def test_consistency(self):
         docs = [corpus[1], corpus[2]]