From dc770d33d2ff5df44b1dbc45debb8bc3462905e3 Mon Sep 17 00:00:00 2001
From: Ben Jeffery <ben.jeffery@bdi.ox.ac.uk>
Date: Wed, 21 Apr 2021 00:45:22 +0100
Subject: [PATCH] Use dict representation for compress and decompress

---
 CHANGELOG.rst                   |   6 +
 requirements/CI-tests-conda.txt |   1 +
 requirements/development.txt    |   1 +
 tests/files/1.0.0.trees         | Bin 0 -> 10300 bytes
 tests/files/1.0.0.trees.tsz     | Bin 0 -> 28043 bytes
 tests/test_cli.py               |  14 +-
 tests/test_compression.py       |  41 +++++-
 tests/test_legacy.py            |  37 +++++
 tszip/compression.py            | 231 ++++++++++++--------------------
 9 files changed, 164 insertions(+), 167 deletions(-)
 create mode 100644 tests/files/1.0.0.trees
 create mode 100644 tests/files/1.0.0.trees.tsz
 create mode 100644 tests/test_legacy.py

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 824eccd..4e805dd 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,3 +1,9 @@
+--------------------
+[0.1.1] - 2021-04-XX
+--------------------
+
+- Add support for new columns in tskit. (benjeffery, #39, #42).
+
 --------------------
 [0.1.0] - 2019-05-10
 --------------------
diff --git a/requirements/CI-tests-conda.txt b/requirements/CI-tests-conda.txt
index 1c21b0b..f5ee0bc 100644
--- a/requirements/CI-tests-conda.txt
+++ b/requirements/CI-tests-conda.txt
@@ -1,4 +1,5 @@
 humanize==3.4.1
+h5py==3.2.1
 msprime==1.0.0
 pytest==6.2.3
 pytest-cov==2.11.1
diff --git a/requirements/development.txt b/requirements/development.txt
index a8609e2..f801ebe 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -1,6 +1,7 @@
 codecov
 coverage
 flake8
+h5py
 pre-commit
 pytest
 pytest-cov
diff --git a/tests/files/1.0.0.trees b/tests/files/1.0.0.trees
new file mode 100644
index 0000000000000000000000000000000000000000..c30951a9c7013613b138f325e0b22e2b858950f0
GIT binary patch
literal 10300
zcmds6eT-aH6`xg$ZIyOa&~~jbjHDXF&c2!b=<bTnnpT7TkVZ;UQ}Ol9%$=EM-^cjg
zn`O&dg~Y}xB$BAj{)r+9u_2g1%xcwah$b~gMerZ^3%>{w3=1(~)1<|7-aY4i+@04q
zOH_=LoW1X!dw%zP+;czo8wWmd+tph~whk2vg^vN<{9*XBf#2Im_U`TS;~i9Adyo8h
zH`TL>E!1!?;kQzK?i%@#hex~d_g%tolhwkLgr{m|v58t9B>c4sp4`a9v;JiTU*wK5
zk+Ol~_glihljIqn=Re2q8Nze?JET7k&*uGA!LJdX`(rt^<oLZzc#i+(_e#mPr~swT
z{$C>eE@~epe4hP`-#ILSBZNOXAfD|XC;Ynz{{Z1RABnxL;P~$&Jg40S!gGBckQl80
zDB<5v{LcI2M_&9n|9nON65%gqDOU7<UD3a^V`KWiui!5bK1Y3l$M5Ha=lM1$p5yl<
z;d%TSpBF#2??u9ICdS%;_A~xf!c!U-2F0`gLzH+b{wh|GQF8tbEBN7^5|D@I{JEa+
z9RE4OXQ++X9RDieIsS})dnDQagY_R&@W%(#&+%(1{xd$$e~w>|@HC8tg#qKo`cD&{
zio(MK;#vP$#r~B6@f^R02v5_#uuAy6{O9re8R4(vVWNTL{fOCW$@-rqJg?Up;V-KI
zrOx_aB0QO0&}bnXRR1PY%=qKHkShArDv#fFgy;Q>@p<uM{7r=C{JEd>4~oB8(Z4bv
zp7UqFf?pjF&+$J>c((Tz+K72QP~*mR_P?R{zce78^T${0e}V8G3$Q|^Wc%+SJRiG`
zQzGZ>zuf&<1^=UHA}9)#D*Jy<@t^T|@#FD3Px$D(alKTOhiChLMR@jiiSP#$d(|q(
z?-|1Lw0VK>?4OF`I_v+Zf?p#%%Vm{}-%gHFHwDJ;%UQtL{u>C-^M7u))R=evWc{;*
z=lr>a@VuX8+0XiKB|M*BI|J(H@jpR$-v7IVXZy1BbNgM2{R@QW`J9Dk`@f~we}?db
zuD|C9&+F$h;h)M<Onlb=6UF}fKO_YP)&DyMze4yN^#zXqUljWppBF#w{}saXeAz+w
z=o?ueqZG}y@W&b*(`w33OSA&{WsAUQ8iA3n>29m#3q?ft8y#VjWY_S7Ljs=J?gTB@
zvyEWPF>FDf^TP8@*D;-@SOE2?K{p*=cmZ&A*J}zd)#eG?oj0tg#hf>rJ;U<HT9(oF
zK{R4mZo|M%`tiv0WE1<?k9P%e@;8Z+UZ-p!7WKobxTHR8v+acz`(sVv2Z<_<M54;L
zJ}lM{cg4c%l{juwB)SL(C{g#_p4UhU1g0(QUJ#k+ne(El`@qOzvo#TjjfuNqPNpc^
z9rY2K9o1xH<NkazX$_i;@{}I(Gaf5hXA{yjGDu;{?3T=~+wEClG-Mr>#XNtE`|Zc0
z-upI59bM0z7mndHz<5tIU?!~Ra4!i_Gy;ai?k1%pCINi$rJjJEbxSzypo1~<$3j(s
z2XiYN(!NZ_hXD3ZN^-5Q2o0i??7C3G#O(FVCap!dci{erYZl(qa4mc*5UzQ+?p_Uq
z>ocB7t^&ff9q-3@#>IV#j_mOD|NBGVkuWA47mf|a2V;VS@j$}ZAYp8f@I1q3nnGc<
z237dI`DNgrFoSa73E_tpp-;Sbab939AP;i@#~*V9#~<^AW5TiE7$kk?Ji#`M70O}^
z5sQ6bU)U%1jXF>l>cpJna}9ik-^ja?Jt(*Sr^dN3M>14yR?1<jb9oKbx!gf@E_YF#
z%OcgeEKyyS=mYwQ{-W;~2aE~E2V;eK!sqwJQp<Hqvn2=i<!E=+wbH5n4VA2hYF5K|
zR>MSA!(>*&R93^DtcK~VhMCeS2m<}^EMEmW0MrD+rn5kg0$l+56VOGVH}b===tpm)
zuPb42j4}6dygAQt%;itb&%C;v3ph8JHrh`m{*euX%`fx==Wd*S(e9tl%9x$X$(WtW
z$e5kV$C#bU#+aST#h9JS#F(AR!<e1Q!k7&?*w4!fOg<Shd}AvEeG=&NK)8Z_6X<(D
z%Rs*edLHO95bgmu?ifGZA8`KTe8=?;3D-0vod5WHeFHy<7<~Lt{)WIj;RkD9{dbp~
zk9j5AoL6oj47sAHD<0AC*qF{)$x7v`WTkRdvQl{}S*aYAtW<tVRw_3oE0vd$mC8xU
zO68+uVJ@m3Rr=Up>El49j|Y`LE>!yXQ0e1DrH>aCa3daJyc+KYngKcpqyu?Cr-7D$
z&H?=b=qVtaH@K#6gG4F;O#$r(`T|haTOc$=>mh#wEBs*{fB%iO9`$HDPCvH%!1>zi
zJEz;z51y}m?IVxvJN@Hd*6#bq9WQ-n_tUj|kJm5#{mZMhN1lFs%eO!KO6>z;@}m<U
zf30@U%C;{){=}u)^Sgd{?6reKbMfdVZY$}mO*W*)KG^^XfM$ii<bJwXk{2GGo&rj<
z#ZslLm9^4M#ggx~f;-_A0ho&=d^W&V+wXdC&B69{m{Y)nJ0VC0@UjrX8m?^{P7{&7
zY2%eGg!*tvvV`uNcVVYa&$0me8T_umhrAiZ4R{5!>!uUl_jI_YCqU^o+vvtHUdtJQ
zUB^Mu8?Mtb+daDW1MC<?BW2el089t&OL#9!ncDW;JA)2)4DkX<zJ$dF=z7C4d|%gr
z#eplsz2AkmhHmF%`EYcJ-J7(f1abTyHU)isQ=)PQj-P14^NL6$K=f*&!|Rr3F2v!F
zB5gKw0r&X;QxXOlZ{+ZW=^)=(pn^`}!w-B`QH|$z-S8O<1;gRRR!Tt<${XH8F!qP|
zF-Q(-Kh<c*e^@m}PNp`7fTo)bZ!=b0mG2yS6i$>3yq*B42Q%2Ubzcaa$Y^ggwc5fF
zFoRq#?x^K~Y#3`&1Wy`Xpp)yw6HZg%kp32|1-jw2p}!V9Euwm-XX}<x7goY}l8zr*
z64r{YWy13#zGcG0CNyZcFbQYg^jrttq;QZIOD^^X)9j=Vk5=$mI%qn*1zgu~b43PT
zi|fMhn#lBoB@930PnIXj+UP`eG<4o<C~M&dC#FVkulJmwSG?`e;o_$}v$(%!6)Rd%
z(`Ks^vs$%y`;ooHs#XnGS=)eTUnlH%VR}lRf;|}2bx#Hz*9k2&aj8VJ!{rYR+k05H
zf?Q$^9xqRqVY3FNW!62zGllf<gyF;Zpn}Ducr{$+Q9OJEP@A>#czFsIt9jT{K&I_u
zswf@6F#vwNui*b^z^TfpHZfYQ9;tw*;O}^OW_o5~&p1mJ3M0|yYE6rO5f}Gr+^4T}
z&tHEJ37@ypJELT!_C(1_9fu?<bsCba)L}@nQfDE_N*#qHD|Hf*tkii<vQp<c$%^k5
z7>tdamq2`Ell@b8SGxP!%i0rI4t?&^zlL(<*l%9_1D3bmxc!e%7LIIbKMUn^XPOuP
zjQnNoD}RHsIHI3<5!;XLx)00ehE{&?a;?&wtZ4PRF)H?GVsuY+qA_Z;s#BxY@p@&t
LK5n!c&6$4z6|E9w

literal 0
HcmV?d00001

diff --git a/tests/files/1.0.0.trees.tsz b/tests/files/1.0.0.trees.tsz
new file mode 100644
index 0000000000000000000000000000000000000000..ca14d5bfdc94dd8f8d8d3e92139beaae588ec120
GIT binary patch
literal 28043
zcmeHP4RjPmww}%pBr!mM00BY91_BC%nIVLPA43F0AS_Y9w=6=OWID;fWG2o}gpi=J
zh&&ZA5M1!+17&y*pUR5-DWZTNi;q<l5RAwwsOTyv$~%I*CA_-bsji=%o}M{}oD+oR
z)R{0{_nTYy-nw<G>Q)tukBv(;!INI`%)Q;VK6E7w{SBohDj8L!UQeK6Zju>2VwK(N
zEiCbP%k6$~h}o7@Q@{}K?EJ=pTM!Wxh1l-*dwq?>3SFVwUM?YG(eJBrR){>rnfdRg
z+>6MeL<rGO;jIzAQu6wo9yj`hY4Z2Lw^gYEe_G@(r}IBMZs(X*ZuFB-n(${ES@lT|
z1|)Z}ROpi2rT#L+t2M*Iv*b0ux$ovUlL<=GvsB^rR7!3;VhVAiC=A3UO72Rh*W)gi
z+@NvtLVoFT7J2Pnr{vSTl3%+6<;5O{R2=%XT!m;$v}6p<7$~YbgV*rX5Zx(XAg27j
zSxzlG>T3pcV8)=xK*6>{wgYfQ3e$!dwT8fCSm)RlJ)wmlmfMTVoNl>8%+Ag#%*qr)
zwAYQKh*%T-UP&sF>|RHxN?yq&*?n@oK^d7Dmj0Qx{-u&z@;Zw{Rrk&D`K59|&3C#3
zvqM#(+T0^EtN)aufZHE1Pnu9*zRT-0j|;fWR*TtU8Dh&EVzHU0Ode^rS!@<oGQ1or
z=J?Cx-pQJhEMsSRg-ZfUv-_1EERHNts)vvX#oWM*Y+lJVsxl}B#0t9?IUhgr5E?~`
zmB^6oZojY4<95wKuS@JMpA-_sS6n8Q+Y2cMQ18vKP!be<o)Z5<b}!qP<TkS2k8Kxi
zWR?sBN<^u&r?D+!ME+y{oyS~51(OL%d*s%NJsz*a>9+eN-+&A-cKaMf2V!wqz&#5@
zGEHr-b)fv`bcI2&*i&BNm3%%A+e5@6m#27^&shZxwUEY$#V)B*a-lDC=r3{Gt9*Xi
zf<>nT)dVabZK<NKEKpM70%)tu3~s%{Kc_-A^I;{~jY1booGw>krQH>P7w&+|)yO#d
zzER!Omq-p0G(;SU$$e#Z8B$jO1_R&d?nlneMly!dij%b^Mp&LuhQ5RX6{x36eEIfV
zlj)HT4VHJ|Q<ysTee}6&+JRp_hR=yT+Z;ikrpbw=pP|n$7dbASg74q3%=-p?n!6M(
z`X1g->-rRYektq<{sjL=&$(Q7Y^DcxFbAjG;jDBz0?5pfIYcqYFR#7%7i?FZIiM1k
zy)+W*Y8-`|wRKeotk^uRzICwzFt9Gm7o~H$AW1{vRi{WCY)nW(uc1ISlMoXd7vG{~
zLLzGgCsw$7VC8j1)o3d~eOY$A#@@lkvVZ+~2TlM6`uqCnUVHM8u%UF+6FO31ZNeJ#
z77EmanEW<W`fM3+*}NuRFzqs(J7+Q_B&4LIrlzHJ?TVrj(PYZa9T8*7JC+syZ>*+E
z2QGbm9di9pdLWoknlAK|l=vjSvBIs22X5ZOK~x=<czR_Jguxje(eA^;3&B%fp_l5=
z6XKmRia8{g-_DQQSZ)ywJgle<OzW=CH$8hZ(k_&423nS##7(FY6d*9gs1Iv9qvt>I
z$Kl9spzvM-#ZPvJ-ETKG8rz_2!A(><o6gPi5-?Ea{iAbRO+{*f!uttT?BFMaOQ;4E
zs0n@|4;9%@TzXG1jf$BomUukk5Yb&o_4-1KSYx#f%p8<8I6FtIN$m5{`ILs$?W(=`
z+39;0Pwx`T+CVt}ftk7CKagj*ru|3jDRaKj^B=sHwf>{C`mnY`$C?lh!~8YMD6{jA
z<=^5!rJV}lca}@Wj>npFy-!|P0T{TQSIpb~P>n<kC7Bhq9F(j?-$8-*;A|YI$j)Zv
zi5OE|oXK)!!BbDZ+U?QPX`OFB`@msoSJ%g~H%%|vJgD`5Zz>&i=5pfDnbY3Leyr}F
z%QH7$ShIF(diA|`yqo{lTYC?m`}x9UA-C}vk+43+@^kgBuY{DN)yL9D*ReKLwWQ`2
zY!DCxD>fDBgo71G^9)!?Owk0>SyliB7V+lytEWNC2!-@tYN`PWfC3c+^+Eqp9#(TJ
zl3VP=noC0!6wOt5Dgv$u=Ns2Vhi=M1f)jv&`qp(x`x90l@tu^wWlv7)P*$6)N2E_B
zEQ~~lB)a&WIOXP}!-DQ#U)Ji761YZ(SF{hS!ykT*{S`;`5MC<%5RdZ48XP$IdXMLM
z2^jQ4&(v-`VX%afq7^s{u7Vd$v7kU57+mvFfx-0wuc<?f$xnsyvbvZc-ipI{v8LVn
zl#7=hdaZa?N<(L%KDPK@+ZW7R@OkCfB-R8LZu`3rFABhPku`yc!+CD`qj?Ue!veO*
z*Y6@&4j_2i<$w_HuF`S<Cu#$W%V>KdAsguxN+(_Yg6oLsgI+^{sxX*q|D_^>WdE_o
z#7Fs(qcqwmzozK*=W%ENBb5T-l1d_3lghpZqWmTuP*?#Nn8vV<ow}ptLK75-@+ncv
zLkWl<3cS}otmbL8K^UQ6d7Y(YH}n?4)>2e@`@ba%&SSknEKbW>qqt<v!z%8(s*TIB
z|A)~=0YlrOXw%g=0I;%*H&3@8Rga6n?Grq$+Yj;XD!F}5)CPUKi?HVmwvpDnS~W(J
zghG7wA8RC`dpAl`a5=6=8;N$~nC&D6n)~p`_BmK{q<1T_)+i2b>czE8jJ+wjReVy5
z)(I(X+OmqH2V`*N<<3&CJj0H1?Ja-BOAB&PObP{}2nr{%p*ib1ADP+q8<@av$3M=3
z=@^u@A*3yvK@0Q(3RF@aR`s;XV2y=RS%kwf%k%HRw@2A;+4dXgZ1$#pMObHCPgBIL
zQJOL^>nn}xXD~4{cD5|fH*^*@IknqI;%8WATtAbgtx;K0tArNu$@(gT<QJ-+`8)w{
z@eQRJW>&FAYtYfEk8L`Ml?F2nt-LjAQ&yz))yDNSv@;b!=*u#Fqo;-KY4uqyKUNyo
z(`0FDRHmdMuqt~nq2=Q`K}ck~fY*Y`nlgitZNEW3d+Lr4H;+L53ks()S=yko1Pgiz
z1u7}eIW^hP=>M#8=|~mAUY}FRN{u27vEY~Vzu>7X7@Go7;X$7$dDwXDyf#vrqDf9^
zzLVLgQks^`kDY>35@6MwQ>^Sf=qx66`%w^BBde=9@*rMZgvD+J2#z@T{9e1O(1-HO
zBN(rLe})oeWXuV`z>{=XFz5_c9}KmjrLNJUg*>blN9OysVJ-I5!;5(5!fLw~rlw7q
zdCUsHK$#Qb^X5YWAQU*%6;hd4__9u6X&DMokR;@!n^>i;ete#sTnO8d#Cb=)6=}0x
z?^}_)1Pl~uUwrY`DM%wwh*MPsj#O7G976A)Kpj{d9*&Bf=W_^+vvJf;q-+K3P9(}L
zOj@7v-Q~HERCwF&mI;S8)N$^Z?#RR)V-~Ox8J|pH4Y-+x;?1T9j~r^=hI=q^$Gp_F
zUAKLDPJW^Hv6U~sbBOJ_bm%5WiUH-gR+MXu^xzC0pj?Fk>$)Dpw<aBx7kt_6Bp#Ka
z|8wHj49Aq~X4S8+x5$>PxL~50cHd~i?k3e+pC4u?ht?Zev!a9Hwk?v{#I=r(kBv>y
z*PFaRtoFBvcmLQZ{)XnlpM_8S%c+mC=Hw+}Uf`NON9Awn>#jS~dC_LGvr=;0n988D
zIamL1O}qL>{{yQ~UhGv>$jV)#Nh^6+P0}$?y1piF1Vd=AJu-!|(V#&TOiyRb#^TpT
zMe3<aenK;Q?4G?iw-U}@lJOOXTE-!bZBXX5Fgb0)v5}X6fjT#Q_W4pc0fkRNg%CPY
zVXe@B>OjHu^3dP%#HQgr_Oz4=pq<>vr=|WH*Bwqvb$YF8U7CK!z7@f%NCU9`Z}jP=
z>C=~T-%rI89B|h9lGk?UP8p-`_dW5#8wXD}u<E2IIk4?Sa_6I*=D1rQd>OA^0hcro
zwhUv>FnG{)?ie@$82E)gx7DqihJ+0T=9VFaE?dKR^cD)#B!?!c`rrsczp(1Pu!-EQ
zNZzf3|Alw5Af=yzkKQ|66GKE3u70rXH!y@5CuSUkl`JT*<P=WwvOUD1N>G5n)Y&%P
zz$>VCCPnELZf(0`GxiEHUlb5*x1pVh@(N?OU}LxVuD=->CjbL$crIz)+qWaVLV=|)
za^;n80Fk<E2N1$RfvV_EBWr(4W`CP)vK3u)gswXpn30p6lR0=GYaPo<j~3u=4jUCA
z;UkU$*toWts<sLHIoo~%%|8Bo=_;6GLxF9Iu%ee09D`~>0SZ$WR*@ICow_d<ZA?G`
zBcRc|QZbrm8h1@=mWB0H%~XR;7|d%#V4%dqCwnAUAe}$~{~JQ=NOiTsD3k{Y1=sr{
zsxP0I8WkUCXZp|S_lcIp$6vYga?;cu@2}dt&oR;8&vQ{K?z8MS^W5%}$1Gj9#x#Dc
zX`E?{>Cny<73Vh|>+!PV#PV*#_tvMje0BK3g2MXX=!adNPdb0KVeXFezK!=h>)q1p
zi~Akv;?3RWoqBF+%*wk4joaO?^}^??dP^ru{x`dQ+o!CBSBg$7%PB1wk@V>L%4M^@
z9bSL<;BVjO_U?6TZHKd8xbB{6%UrssYEe(u{&L%vXNGPq?O*M<*!k^0&szV&JFnZs
zj+gR=*e`7E{pQRU(w5Yx9bLKX?+uP6FAclz&mUZ9`Ot;u4s2Lqo3d=<hU)6gV^_85
z(fhqEU-$3z>CroWUD$s8gA>xey|nK_;SW`Ze%!dcebK6Sj>aF(oRGgZC%C5Pq*lGw
zJbSFg!@ZBcT|9DT@%YWwBagIOG%QW(ba~H<8@t4=a4f#%^L@_ezNuaFTK3}NliAOf
zI8(>Zjejz0ce_W;{eJpwY-+p3@w?V0me!WVl&n8#*|X9+bAM75N&<Nx&9SdlOwQ=|
zywYy%t+g-CzjV*))(Z|;|Ep+t{n5S;59#>KTGPC<Cm;JLAtA7G`dj1H-}6%D@S5QX
z4F%akzp2hux4(LN%(&^jQ_l39^J?<B?SE_NAjQ4=sc%%*K3kJ#ob5EGW7X-?GgcHn
zy~A@LU0P&0;l5>D{fZq8k5^j;TX!G*Ti2)0ev;NQdHjdlkG$2YW7c;)&wTlD{f30C
z$4zldMpXP<aemE=zWa7>b(|6o<}F;^yKdC7rB{+F%wNra@ygp%i)vfFxqttYKhEq@
z_{fZ$skcqsJ$Gl{X=%4F{XX>*o8!m_X$xOF_1@k8<gUcnoOpjF_V6&t(^}~7%Dt0A
zZCC$1dQ|Rx_T`}sIE;fSp1$0{x5qF+9C8%}<As>iSUIE%dBrJ0l%F8-_v(eGJQ07j
zsX}5h1qIk9MZ1=8(J4>FUuH^(9?>V_uOcNx7xanv%R&j!6R1@>?D01Q5~BZ!4|!i8
z-#-4@I|7h}ZY-rQ9JKJH{0y6r)+17K&RLDDc>w+zIszajumH>*5j+fkw;F+|OAH00
zX$XI#8Ugr%1K{sh<5}Ua2_s08(H0La5*10q4S#D7!FXQ>gL7FtQ~U)}1n{pYaW528
z7*N9-f2$OMibsc*XefSVglCGsD2X6dx6wh;aHB7&M6^K~iW}Up#uNpfDgG)Q!qiV_
zpFA9(hBvx%Kzby8g2HGNu)@xtZnxm?%^{?VR8l&BmTv`r84RI6j~2p|pH?zLc}o6*
z7DBlS`Cf%GY**xO9U%bmXcdPB&<KpbVuX-RQc20=ABXRJ3;2682+CeGUsM>NP@>?X
z6WVfB=!qTZalzmGK)6_hHlZ{q8ZP*29tiz$mENd>@i!L`jC?czD?F$e8V>kd1_<>(
zG}H#2ia*d!px!|XCoCfvO2dzScj}8~i^>_;#w$>Hlc3K+%5%j=27-cHC8f{A@$~$u
zYr=m$+R0UZTBg@9P5&}1LE|SV^fF^g=4coH)BM3<0#J*BSLPToPs$$?CZzvRN!hdk
z9)UlRN)Wc7B~a$raTG0&Me8aGEpr-`|8f3MCHZkrFBU-a+q9{$`rd&^snaJq_ClY-
zgEc1n1cf-X-%HVlGaCLxAOZ3A)`Do-K=S~p9{33g+gS3j=EEOxBPi+UR#K`ZjR)R%
zCD2^3l5nx~(24bYA%A9zF!32$nxmMILBdJ``9Adt3f=oM<m}lkzA5}6B#aXjmi7w;
z6P6EF7DrhC`Y;mD1%IlBa4{-_;X*lM!;|ypX$bkhv*h8T1OAu@0l6@M;URR0K=&>-
zt|_tL<YJwU#UJ+|zx<L)${+sVMZljdAQ<y)_~%s&jS}h1H(0dCPf$3ZQtKQp;2Gg_
z<Ox(>CT4_(((ps)lE8X1eu6@+9ta*Gbs|_gJpz_d@e>qUXX&to5jEWL8Quh|M8#4g
z2pxX-q-BC}Y%u0W#n5m-%ZJt1pR##s+zcH?_#|4w$VWMPP#S({<A=mq`~-zthoHWr
z<`(I60NVUeOshUYp^PD?(`R|R<Fi09NKiO7lx9K3&@@1w1M@JNI-Eo2^BD;TOVK(C
z5khExPIJJHY@agJ;bRj%Gmr2v79D3JJT#(&l>?;X;U_56G4yPbn2u@jIc5YbIge(B
z1=H|E8#^SH;U_4}(1$UwdfFYBF(CR33McN=WesC$IOH>=Fkny^Jd#FJJG73#_@p0#
z^UA1@I2ul9S)p)NeGn(3Lm}WM=`h46pkTb9u<@>Na2md72f;@CEykclPxZb`-MK%X
zY(hAh$N?A$na`jgNPF*QSy7QR&EeA?2*#wbp)m{u&F|(Dzz@cS2h{MUFY2x+9fV$A
z*73&t4nE;U;s78LV)pZSr_66l6R6|)EH4B~(;$BHnSczLpaWtcZGKOcAa41C0iuRG
zen*vH#ZF|g*!@*rRQwhp!Ko?;iKF3!-@+jXT_=S?Fc1>IGeO|$CWnJVe8r<JOp|Fk
O`d5dBvXiK~$@KqtIOli(

literal 0
HcmV?d00001

diff --git a/tests/test_cli.py b/tests/test_cli.py
index 1d14e07..fd01d88 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -31,7 +31,6 @@
 
 import msprime
 import numpy as np
-import pytest
 import tskit
 
 import tszip
@@ -43,6 +42,9 @@ class TestException(Exception):
     Custom exception we can throw for testing.
     """
 
+    # We don't want pytest to use this as a class to test
+    __test__ = False
+
 
 def capture_output(func, *args, **kwargs):
     """
@@ -189,7 +191,6 @@ def setUp(self):
     def tearDown(self):
         del self.tmpdir
 
-    @pytest.mark.xfail
     def test_simple(self):
         self.assertTrue(self.trees_path.exists())
         self.run_tszip([str(self.trees_path)])
@@ -199,7 +200,6 @@ def test_simple(self):
         ts = tszip.decompress(outpath)
         self.assertEqual(ts.tables, self.ts.tables)
 
-    @pytest.mark.xfail
     def test_suffix(self):
         self.assertTrue(self.trees_path.exists())
         self.run_tszip([str(self.trees_path), "-S", ".XYZasdf"])
@@ -221,7 +221,6 @@ def test_variants_only(self):
         G2 = self.ts.genotype_matrix()
         self.assertTrue(np.array_equal(G1, G2))
 
-    @pytest.mark.xfail
     def test_keep(self):
         self.assertTrue(self.trees_path.exists())
         self.run_tszip([str(self.trees_path), "--keep"])
@@ -231,7 +230,6 @@ def test_keep(self):
         ts = tszip.decompress(outpath)
         self.assertEqual(ts.tables, self.ts.tables)
 
-    @pytest.mark.xfail
     def test_overwrite(self):
         self.assertTrue(self.trees_path.exists())
         outpath = pathlib.Path(str(self.trees_path) + ".tsz")
@@ -255,7 +253,6 @@ def test_no_overwrite(self):
                 f"'{outpath}' already exists; use --force to overwrite"
             )
 
-    @pytest.mark.xfail
     def test_bad_file_format(self):
         self.assertTrue(self.trees_path.exists())
         with open(str(self.trees_path), "w") as f:
@@ -283,7 +280,6 @@ def setUp(self):
     def tearDown(self):
         del self.tmpdir
 
-    @pytest.mark.xfail
     def test_simple(self):
         self.assertTrue(self.compressed_path.exists())
         self.run_decompress([str(self.compressed_path)])
@@ -293,7 +289,6 @@ def test_simple(self):
         ts = tskit.load(str(outpath))
         self.assertEqual(ts.tables, self.ts.tables)
 
-    @pytest.mark.xfail
     def test_suffix(self):
         suffix = ".XYGsdf"
         self.compressed_path = self.compressed_path.with_suffix(suffix)
@@ -306,7 +301,6 @@ def test_suffix(self):
         ts = tskit.load(str(outpath))
         self.assertEqual(ts.tables, self.ts.tables)
 
-    @pytest.mark.xfail
     def test_keep(self):
         self.assertTrue(self.compressed_path.exists())
         self.run_decompress([str(self.compressed_path), "--keep"])
@@ -316,7 +310,6 @@ def test_keep(self):
         ts = tskit.load(str(outpath))
         self.assertEqual(ts.tables, self.ts.tables)
 
-    @pytest.mark.xfail
     def test_overwrite(self):
         self.assertTrue(self.compressed_path.exists())
         outpath = self.trees_path
@@ -346,7 +339,6 @@ def test_decompress_bad_suffix(self):
                 "Compressed file must have 'asdf' suffix"
             )
 
-    @pytest.mark.xfail
     def test_bad_file_format(self):
         self.assertTrue(self.compressed_path.exists())
         with open(str(self.compressed_path), "w") as f:
diff --git a/tests/test_compression.py b/tests/test_compression.py
index 531bf8c..eb52f22 100644
--- a/tests/test_compression.py
+++ b/tests/test_compression.py
@@ -28,7 +28,6 @@
 
 import msprime
 import numpy as np
-import pytest
 import tskit
 import zarr
 
@@ -106,20 +105,17 @@ class RoundTripMixin:
     Set of example tree sequences that we should be able to round trip.
     """
 
-    @pytest.mark.xfail
     def test_small_msprime_no_recomb(self):
         ts = msprime.simulate(10, mutation_rate=2, random_seed=2)
         self.assertGreater(ts.num_sites, 2)
         self.verify(ts)
 
-    @pytest.mark.xfail
     def test_small_msprime_recomb(self):
         ts = msprime.simulate(10, recombination_rate=2, mutation_rate=2, random_seed=2)
         self.assertGreater(ts.num_sites, 2)
         self.assertGreater(ts.num_trees, 2)
         self.verify(ts)
 
-    @pytest.mark.xfail
     def test_small_msprime_migration(self):
         ts = msprime.simulate(
             population_configurations=[
@@ -137,7 +133,6 @@ def test_small_msprime_migration(self):
         self.assertGreater(ts.num_trees, 2)
         self.verify(ts)
 
-    @pytest.mark.xfail
     def test_small_msprime_top_level_metadata(self):
         ts = msprime.simulate(10, recombination_rate=2, mutation_rate=2, random_seed=2)
         self.assertGreater(ts.num_sites, 2)
@@ -151,7 +146,6 @@ def test_small_msprime_top_level_metadata(self):
         tables.metadata = {"my_int": 1234}
         self.verify(tables.tree_sequence())
 
-    @pytest.mark.xfail
     def test_small_msprime_individuals_metadata(self):
         ts = msprime.simulate(10, recombination_rate=1, mutation_rate=2, random_seed=2)
         self.assertGreater(ts.num_sites, 2)
@@ -159,7 +153,9 @@ def test_small_msprime_individuals_metadata(self):
         tables = ts.dump_tables()
         tables.nodes.clear()
         for j, node in enumerate(ts.nodes()):
-            tables.individuals.add_row(flags=j, location=[j] * j, metadata=b"x" * j)
+            tables.individuals.add_row(
+                flags=j, location=[j] * j, parents=[j - 1] * j, metadata=b"x" * j
+            )
             tables.nodes.add_row(
                 flags=node.flags,
                 population=node.population,
@@ -169,6 +165,7 @@ def test_small_msprime_individuals_metadata(self):
             )
         tables.populations.clear()
         tables.populations.add_row(metadata=b"X" * 1024)
+        tables.sort()
         self.verify(tables.tree_sequence())
 
     def test_small_msprime_complex_mutations(self):
@@ -200,6 +197,36 @@ def test_mutation_parent_example(self):
         tables.mutations.add_row(site=0, node=0, parent=0, derived_state="A")
         self.verify(tables.tree_sequence())
 
+    def test_all_fields(self):
+        demography = msprime.Demography()
+        demography.add_population(name="A", initial_size=10_000)
+        demography.add_population(name="B", initial_size=5_000)
+        demography.add_population(name="C", initial_size=1_000)
+        demography.add_population_split(time=1000, derived=["A", "B"], ancestral="C")
+        ts = msprime.sim_ancestry(
+            samples={"A": 1, "B": 1},
+            demography=demography,
+            random_seed=42,
+            record_migrations=True,
+        )
+        ts = msprime.sim_mutations(ts, rate=1, random_seed=42)
+        tables = ts.dump_tables()
+        for name, table in tables.name_map.items():
+            if name not in ["provenances", "edges"]:
+                table.metadata_schema = tskit.MetadataSchema({"codec": "json"})
+                metadatas = [f'{{"foo":"n_{name}_{u}"}}' for u in range(len(table))]
+                metadata, metadata_offset = tskit.pack_strings(metadatas)
+                table.set_columns(
+                    **{
+                        **table.asdict(),
+                        "metadata": metadata,
+                        "metadata_offset": metadata_offset,
+                    }
+                )
+        tables.metadata_schema = tskit.MetadataSchema({"codec": "json"})
+        tables.metadata = "Test metadata"
+        self.verify(tables.tree_sequence())
+
 
 class TestGenotypeRoundTrip(unittest.TestCase, RoundTripMixin):
     """
diff --git a/tests/test_legacy.py b/tests/test_legacy.py
new file mode 100644
index 0000000..6c334e2
--- /dev/null
+++ b/tests/test_legacy.py
@@ -0,0 +1,37 @@
+# MIT License
+#
+# Copyright (c) 2021 Tskit Developers
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""
+Tests for files from previous releases.
+"""
+import pathlib
+
+import tskit
+
+import tszip
+
+
+class Testv1:
+    def test_decompress(self):
+        files = pathlib.Path(__file__).parent / "files"
+        ts = tszip.decompress(files / "1.0.0.trees.tsz")
+        ts2 = tskit.load(files / "1.0.0.trees")
+        assert ts == ts2
diff --git a/tszip/compression.py b/tszip/compression.py
index 7a42a49..2eeda6e 100644
--- a/tszip/compression.py
+++ b/tszip/compression.py
@@ -29,6 +29,7 @@
 import tempfile
 import warnings
 import zipfile
+from typing import Mapping
 
 import humanize
 import numcodecs
@@ -158,16 +159,47 @@ def compress_zarr(ts, root, variants_only=False):
 
     if variants_only:
         logging.info("Using lossy variants-only compression")
-        # Reduce to site topology and quantise node times. Note that we will remove
+        # Reduce to site topology. Note that we will remove
         # any sites, individuals and populations here that have no references.
         ts = ts.simplify(reduce_to_site_topology=True)
-        tables = ts.tables
+
+    tables = ts.tables
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        # When using a zipfile in Zarr we get some harmless warnings. See
+        # https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.ZipStore
+        root.attrs["format_name"] = FORMAT_NAME
+        root.attrs["format_version"] = FORMAT_VERSION
+        root.attrs["sequence_length"] = tables.sequence_length
+        root.attrs["provenance"] = provenance_dict
+
+    columns = {}
+    for key, value in tables.asdict().items():
+        if isinstance(value, dict):
+            for sub_key, sub_value in value.items():
+                columns[f"{key}/{sub_key}"] = sub_value
+        else:
+            columns[key] = value
+
+    if variants_only:
         time = np.unique(tables.nodes.time)
-        node_time = np.searchsorted(time, tables.nodes.time)
-    else:
-        tables = ts.tables
-        node_time = tables.nodes.time
+        columns["node/time"] = np.searchsorted(time, tables.nodes.time)
 
+    # Encoding array is a tuple so must be converted
+    columns["encoding_version"] = np.asarray(columns["encoding_version"])
+
+    # Sequence length is stored as an attr for compatibility with older versions of tszip
+    del columns["sequence_length"]
+
+    # Schemas and metadata need to be converted to arrays
+    for name in columns:
+        if name.endswith("metadata_schema"):
+            columns[name] = np.frombuffer(columns[name].encode("utf-8"), np.int8)
+        if name.endswith("metadata"):
+            columns[name] = np.frombuffer(columns[name], np.int8)
+
+    # Some columns benefit from being quantised
     coordinates = np.unique(
         np.hstack(
             [
@@ -180,78 +212,18 @@ def compress_zarr(ts, root, variants_only=False):
             ]
         )
     )
-
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        # When using a zipfile in Zarr we get some harmless warnings. See
-        # https://zarr.readthedocs.io/en/stable/api/storage.html#zarr.storage.ZipStore
-        root.attrs["format_name"] = FORMAT_NAME
-        root.attrs["format_version"] = FORMAT_VERSION
-        root.attrs["sequence_length"] = tables.sequence_length
-        root.attrs["provenance"] = provenance_dict
-        if tables.metadata_schema.schema is not None:
-            root.attrs["metadata_schema"] = tables.metadata_schema.schema
-            root.attrs["metadata"] = tables.metadata
-
-    columns = [
-        Column("coordinates", coordinates),
-        Column("individuals/flags", tables.individuals.flags),
-        Column("individuals/location", tables.individuals.location),
-        Column(
-            "individuals/location_offset",
-            tables.individuals.location_offset,
-            delta_filter=True,
-        ),
-        Column("individuals/metadata", tables.individuals.metadata),
-        Column(
-            "individuals/metadata_offset",
-            tables.individuals.metadata_offset,
-            delta_filter=True,
-        ),
-        Column("nodes/time", node_time),
-        Column("nodes/flags", tables.nodes.flags),
-        Column("nodes/population", tables.nodes.population),
-        Column("nodes/individual", tables.nodes.individual),
-        Column("nodes/metadata", tables.nodes.metadata),
-        Column(
-            "nodes/metadata_offset", tables.nodes.metadata_offset, delta_filter=True
-        ),
-        # Delta filtering makes storage slightly worse for everything except parent.
-        Column("edges/left", np.searchsorted(coordinates, tables.edges.left)),
-        Column("edges/right", np.searchsorted(coordinates, tables.edges.right)),
-        Column("edges/parent", tables.edges.parent, delta_filter=True),
-        Column("edges/child", tables.edges.child),
-        Column("migrations/left", np.searchsorted(coordinates, tables.migrations.left)),
-        Column(
-            "migrations/right", np.searchsorted(coordinates, tables.migrations.right)
-        ),
-        Column("migrations/node", tables.migrations.node),
-        Column("migrations/source", tables.migrations.source),
-        Column("migrations/dest", tables.migrations.dest),
-        Column("migrations/time", tables.migrations.time),
-        Column(
-            "sites/position",
-            np.searchsorted(coordinates, tables.sites.position),
-            delta_filter=True,
-        ),
-        Column("sites/ancestral_state", tables.sites.ancestral_state),
-        Column("sites/ancestral_state_offset", tables.sites.ancestral_state_offset),
-        Column("sites/metadata", tables.sites.metadata),
-        Column("sites/metadata_offset", tables.sites.metadata_offset),
-        Column("mutations/site", tables.mutations.site),
-        Column("mutations/node", tables.mutations.node),
-        Column("mutations/parent", tables.mutations.parent),
-        Column("mutations/derived_state", tables.mutations.derived_state),
-        Column("mutations/derived_state_offset", tables.mutations.derived_state_offset),
-        Column("mutations/metadata", tables.mutations.metadata),
-        Column("mutations/metadata_offset", tables.mutations.metadata_offset),
-        Column("populations/metadata", tables.populations.metadata),
-        Column("populations/metadata_offset", tables.populations.metadata_offset),
-        Column("provenances/timestamp", tables.provenances.timestamp),
-        Column("provenances/timestamp_offset", tables.provenances.timestamp_offset),
-        Column("provenances/record", tables.provenances.record),
-        Column("provenances/record_offset", tables.provenances.record_offset),
-    ]
+    columns["coordinates"] = coordinates
+    for name in [
+        "edges/left",
+        "edges/right",
+        "migrations/left",
+        "migrations/right",
+        "sites/position",
+    ]:
+        columns[name] = np.searchsorted(coordinates, columns[name])
+
+    # Some columns benefit from additional options
+    delta_filter_cols = ["edges/parent", "sites/position"]
 
     # Note: we're not providing any options to set this here because Blosc+Zstd seems to
     # have a clear advantage in compression performance and speed. There is very little
@@ -261,8 +233,10 @@ def compress_zarr(ts, root, variants_only=False):
     compressor = numcodecs.Blosc(
         cname="zstd", clevel=9, shuffle=numcodecs.Blosc.SHUFFLE
     )
-    for column in columns:
-        column.compress(root, compressor)
+    for name, data in columns.items():
+        Column(
+            name, data, delta_filter="_offset" in name or name in delta_filter_cols
+        ).compress(root, compressor)
 
 
 def check_format(root):
@@ -307,76 +281,35 @@ def load_zarr(path):
 
 
 def decompress_zarr(root):
-    tables = tskit.TableCollection(root.attrs["sequence_length"])
     coordinates = root["coordinates"][:]
-    if "metadata_schema" in root.attrs and "metadata" in root.attrs:
-        tables.metadata_schema = tskit.MetadataSchema(root.attrs["metadata_schema"])
-        tables.metadata = root.attrs["metadata"]
-
-    tables.individuals.set_columns(
-        flags=root["individuals/flags"],
-        location=root["individuals/location"],
-        location_offset=root["individuals/location_offset"],
-        metadata=root["individuals/metadata"],
-        metadata_offset=root["individuals/metadata_offset"],
-    )
-
-    tables.nodes.set_columns(
-        flags=root["nodes/flags"],
-        time=root["nodes/time"],
-        population=root["nodes/population"],
-        individual=root["nodes/individual"],
-        metadata=root["nodes/metadata"],
-        metadata_offset=root["nodes/metadata_offset"],
-    )
-
-    tables.edges.set_columns(
-        left=coordinates[root["edges/left"]],
-        right=coordinates[root["edges/right"]],
-        parent=root["edges/parent"],
-        child=root["edges/child"],
-    )
-
-    tables.migrations.set_columns(
-        left=coordinates[root["migrations/left"]],
-        right=coordinates[root["migrations/right"]],
-        node=root["migrations/node"],
-        source=root["migrations/source"],
-        dest=root["migrations/dest"],
-        time=root["migrations/time"],
-    )
-
-    tables.sites.set_columns(
-        position=coordinates[root["sites/position"]],
-        ancestral_state=root["sites/ancestral_state"],
-        ancestral_state_offset=root["sites/ancestral_state_offset"],
-        metadata=root["sites/metadata"],
-        metadata_offset=root["sites/metadata_offset"],
-    )
-
-    tables.mutations.set_columns(
-        site=root["mutations/site"],
-        node=root["mutations/node"],
-        parent=root["mutations/parent"],
-        derived_state=root["mutations/derived_state"],
-        derived_state_offset=root["mutations/derived_state_offset"],
-        metadata=root["mutations/metadata"],
-        metadata_offset=root["mutations/metadata_offset"],
-    )
-
-    tables.populations.set_columns(
-        metadata=root["populations/metadata"],
-        metadata_offset=root["populations/metadata_offset"],
-    )
-
-    tables.provenances.set_columns(
-        timestamp=root["provenances/timestamp"],
-        timestamp_offset=root["provenances/timestamp_offset"],
-        record=root["provenances/record"],
-        record_offset=root["provenances/record_offset"],
-    )
-
-    return tables.tree_sequence()
+    dict_repr = {"sequence_length": root.attrs["sequence_length"]}
+
+    quantised_arrays = [
+        "edges/left",
+        "edges/right",
+        "migrations/left",
+        "migrations/right",
+        "sites/position",
+    ]
+    for key, value in root.items():
+        if isinstance(value, Mapping):
+            for sub_key, sub_value in value.items():
+                if f"{key}/{sub_key}" in quantised_arrays:
+                    dict_repr.setdefault(key, {})[sub_key] = coordinates[sub_value]
+                elif sub_key.endswith("metadata_schema"):
+                    dict_repr.setdefault(key, {})[sub_key] = bytes(sub_value).decode(
+                        "utf-8"
+                    )
+                else:
+                    dict_repr.setdefault(key, {})[sub_key] = sub_value
+        elif key.endswith("metadata_schema"):
+            dict_repr[key] = bytes(value).decode("utf-8")
+        elif key.endswith("metadata"):
+            dict_repr[key] = bytes(value)
+        else:
+            dict_repr[key] = value
+
+    return tskit.TableCollection.fromdict(dict_repr).tree_sequence()
 
 
 def print_summary(path, verbosity=0):