From 15b6421d0789a5402275358d43719f4b37979929 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 5 Jan 2018 23:26:22 +0100 Subject: [PATCH] :white_check_mark: added UTF-8 decoder capability and stress test As described in http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt; Markus Kuhn - 2015-08-28 - CC BY 4.0 --- test/data/markus_kuhn/UTF-8-test.txt | Bin 0 -> 22781 bytes test/src/unit-unicode.cpp | 392 +++++++++++++++++++++++++++ 2 files changed, 392 insertions(+) create mode 100644 test/data/markus_kuhn/UTF-8-test.txt diff --git a/test/data/markus_kuhn/UTF-8-test.txt b/test/data/markus_kuhn/UTF-8-test.txt new file mode 100644 index 0000000000000000000000000000000000000000..a5b5d50e6b61eb9a3b751b3954f83e61bb59db9b GIT binary patch literal 22781 zcmdU1X_Fh*b>CYjiIrHEbIyIeVd1bb#8j*y4MbNJE8ngr{T%> zH=?M0*NGF|Zns=*maBHFY*)*j-4j3B+Sy$_dEy6TNmiFvPA)BPEUj+fmviUj>}zDb zylhzyHeB;;sk==Fw0Y8Snr+$lJK|ijTdwCUO2hB+4}n}98At=QQVnO2tbk<@(9!SA*KTMb-()7zJ}9Yk)m)3ovMPE_?< z4&m`=AYfdi2gL9x(zW;T2&>e#!>~kZOLg0AmhhcCgBHcvb3FXf@9Z>qKS;P_9xq{M zX9r^v+X4}FC$KAEXd?8A)3Eozr9kXRLJ%VDrme$0ABAGOEs(JYzJ!rugIk~^6$2>n zcEzq>1dFOMR23y23`tPI^(hkaJx~+w1GHs>5#U|33BkDdJ8tNBov>p!@zi!F0^_$X zwVlAF6hNWW!L)ctp%pbicFV`;>TVE5OOcBa*d6d8P>GKyMu;y#v2jP!v2ly zk#^vNEf_7C=(s`3piW3-P;OK9WvZqTur#5BA;#K-8dm*1m=Jj3)$CxO%20DT9xyLe zwsDmL*Fxz!%_e`wy4l5ZQiI8()+YdS1n&OdNLR+dk#pITbEYjt^fc~L?miBLAnr&X@2<+Unk z1xxnas$;=~9Ct4@5j@+*5Y%*_V9+xuWkXKLj~OeW4FN%?jpW=YyARqX)rK9|RoI{z z6*L^m6}lKPRobrk?FftsJC)FhN~FyyrMJYKuwgkgxUNh6oN`}GFDwzt;8txQTCib0 zpaX3vNd^8-vO$_rm*s*-zPDPw2N~g;P^31r#;!I1WQ{2rZPN#c8k3rQ#uD?@lrel9 zc&guQIdCZHh-Bg7@$K#H&0~wwZMb+&)Ax7DrV9Ahba&}X@N9^jV}PH6M+Px?o*Hkbs)2c49sb4l;8pIYPlARu zoGPT(Z8+q>K>~n^Evy`)jv=&wc~r>Af(RF7VRiKHP%*^JY6M2Y=RqsH9fW|J=p5L9 znCI@cwPR{0b+~K4%VEB*<45q|vDS%Qr|sIF1&&Z+V?K-_EmasQ z8kByYLD(%L0K%By=V1!rzE^DozUTgkfsVl1(8MKiTZq~&WQ?&SPutJLna1mUxDM_8 zP)ZNpy(L@sZI)m$I*2Ai09`mdo-eM(DM*@X6gpHP8)DPgmBh+wV22I*AB0J!9+fy; z8{>>!u-$}rV^pxJHiRsQ=#+ES2`J?=&=eCklnU@e6AC)5L_(+U%sfB{lLwa<+D)c0 z@KC(hsyj~Ym{ISVi82sR3hY2obbeCmm0&TKs!kix5mfO0s0&XAKVcanR?92p6&3Aj z8SfK619~GH1bV)=)P}u8YO7}+ICuU$#}Wm8Gqf%qN3>YhoB8(iHYkdjXatJ~CgW3j zY>f4iQ(Nk~HBd)WUqlpu?vm2{m{U#~UP-4-nIT8$v^)WO76SEQ@N+P=V`Y;of&qRv z)cla%oCy!KxVqci&#Y-mCc=zBS!*Ss((AM;P-3Qwi3bb$e1=vyB2^fTxfonxrVgS2 zoKKuoB7(+ABNaY2vzm!32iYF<<33-#7gY~A0sMvCghhF zq-e|{8C-$Ugm&8k%c>);(`u^J2$kppjT%GKyf1X4j@nzOxnwR`JcchcJNi(NkUAXU zsBwD@$v}mutT zh$~}5e&&(OGeVveIhlv#Txx{K7vFe!R?Jf(r*fF<3Yq8RORop!41|tK>vn&mbRg_| zm`Ke*yX7d<8wxo^y-#oF)cePZYNK$x)71O)b{_BPK~X?uV0lu#G3m<#4hJ(Lj;r?y z0rQC89Lx$irrv9cn1}opAZH|VRK3AOy>gQIHZf-ayJ7t0~3rOy}M?jxyqUWGiivnIv=ZUUoW)yJ@%RT>A zZ|6_n7AGs61L#Y#T@(gG2L*alV=(oPCW|Yqyd_s}%50DMoQs02-}TDL_lo|U|9IgU zC=ru;a?(B`3|YRkv3!@5islZ`Ka<&Mu%8qz@G5oS74LhKL_mwUeB`g-0 z885ud+RgBaa?Su6Jh+(ko8rwxp}|7{J$ZAyStvAkr{x|eJ@}@Vx<*jdf`#awGu6MuZz3+SfqlZB<_ZYm#KJdX0efT3E z{n*Dp@ySnp`tc{O+%Dc{KKqNm^vj?7mHw}O{tLhM#V4=MWD%p!p1(ZSbgF~$IZ9|@ zg!S8ed2saM1%5jc`snXjU|w4_tD$k&MOa#0(%M{7dBDVc`BIg7T&ttBKjlj4(;A;C z%OM8p(pcEW0&zC_v`#PuCjC>2m;UAU-Y8z_ntVyVEMJkY%CF0B$ZyJT$#2W=$nVPU z$?wZk^0gTrVd(n<`9t|5`D6JL`BV8b`E&UT`AhjL`D^(b`CIurd0L7ms`DQ&^p6@y z`L3KAm+#sX`8t%Z^NN=5b@{sdz5Ijxqx_Tnv;2$vtNfe%yZndzr`(Wl$ngJN9>_N+ zkBxD0tWOa~o#LpUo^Ebg{P?c6w`b&8c}|{}7v#U>zvY`V@PFJCkH$WbuTdWB&64GgT zysDZJJZ*k%+6;6}KSAQkmiF_oRc4|oB7=iG(#2q13uZ9^c@>u(QBsJ|L8XgMXgXOq zm#rG3WWd$cxU&sS8R<2D{;i>eh5TVo7);F63rof4=zI7naP@a<-sQ;!47FNLX26 z$t>m86H2;}Fo~I(<6PcQ(on~9X!b$vkUblwOT$G!=a?r8i#%}tK7u}$E(?52e5$%z z*gDHJG1+X$Lc>MgN8*O8*Ce66mo|5>-H8DsEtVFV6DByx&-srR$fax7FT;7z+SNRt z9uA?!e{cTuJqZ2D`v2HZW++5+qqcUo+YTB| z5%$vF(4+}4($33K3g=HB_{TU{ceuFp^0>{9KK<6e(WXyw-d)#qrSHrK4`C4W@n1a# ze);9^9145>{DTL(fKE5hGUG_Lmbq~a<|`Q&!6_2p=}QBWMm2ea0_}P(FGrd?Dc08U2^rTakV#X z(Xa@!XT)ZkA}VQUq(i@Jt;5YC4X0I>Gg1m|sD$YQ%kbjn(^BG&1Lw8KC-o-N)2xl) z*{Cy)XwN6?Gk^8aUTcTgmT;Fg4`YW4w{vK-y7n%!r@hIhc*HIwuXW+JhA~|rCz7kD zn_I9`2oX=0$j~ z30Q8Z3fn^)s82;V(zUh!)zHLRu!j}ZoYp3d(oJZC#sC`E$OSyN?QmdVhS@=Y=}{-( zrZe5oLhIafo$g}7J)};MR=h6_ACB8crtjR2W%^xdvJl5O|J8_AObq4NWVKM1)X&S53})42gHvR~Io zCU5;QAF6wAcnkNvJ|GL31u)a|b4=={0}XCp4!onNK0W z_(((4S{kxYI#EJ5wb`r%y)(juZ6z1&m-~8h>Gr6A7gfNC(|{@KZm0;o+q!9y_RIk$ z7vs^vE!Ql8fE`MnZe=scf6dnoFWAP%kJAe|{{-W{H~n<>H}jV-^bZ=y^2RC~gR0n} zkc;=A5NYAHD~0p8o<+db4A9Y@pgbl3uV0=OaVhN4#N5@hu%i?v24T zJ(F?mj%30|pjAJ?#2fmEhk#j-o4F^<;Vp>-Cod#(nAA=*Rwo7g`!3#-%}x${>ipI5 z!O2`BF>Lq(lPBm7t^@2UBe@rEL(U*LH;rLfodHTRS5xE6G8Hmm;>xXhvJ0b73K;#4*}HwR>dIZnepaCUAE6hTQO%}vGzUetHYUmh$4KVTMi zNfR2UFij8q9n>fC1Idl)ptnatpK)KC*w6+8W0gf`1F*Flj}by43 zC48);eEiH|X~d!}__&>n8g^^E;Ek(?MZ~f#5tC7yM2Vv=mLH8tX5`n~{)LiWmHTZs zfIPlSt;OW-$JJWE$W++IXh3%+^~WyXxV7MQ%qHF^n5l3w%v3jld22x#%-CXJgClgR zDUQ(j_@FSIvCSCFjLsA@r4yJ%=`>bD%O0UqReglcCyMja*iDR?(V1eVbON&|oyKzL zFGlE8s1Z7^6z3;;9sm|&W^|^QDV@M9Lg#Ip;0OTqXie_JvV5|DfU-W8I1Zch%s+aK zW;-RE7Lt;R(lwWfN<5k-*n@yrRL~=$aGDH@-g?Z^*m(16PpHNsaT!^{^P4=S4s$u4 zn4_g-PG6ET$^-GbU5eSWO3BPePsM`!RsujH>i4x$Ilc89&MIe9-DdQnA3T!WrWLlMIw zp{lN#KWL=_YYXWDsV)UqntpW`df>;3Z59c&tui(!b9f~fie^igAJ#+iN8yc(*PM%o zgQ+lQ#JPjq$ur$Tjq@=p!HGQLh0Y9+wa(;}L%?JP$Ax_^nZjg87@pla_1*mCaesQs zBt{CNc)%f4raCGP@*pJw5#o&C&kib4X2*-!d*E^m64IZa^19wsZbc|NX=Mqr?7V z!~O?`{SOZN9~$;QJnVmD*#GFT|FL2J|ZGz&1f`h z_Eg0#9_P<(SztEeVN5IK^2&0qE&E{3&!4{dd8KC_9bCn0@XQtbyn>$t{6x|G*@p*L z@ftjf&+!^Oi_h^IJd4lq8a#*3@fsXVxA`=q{yCuGHFyqacnzKd8eW6vfri)Md3=u7 p;CXzG*Wh`4j@JOEDqJmkvO@N>BDCH5J3T8tzwg#xE%c3^^?x0SNE-kE literal 0 HcmV?d00001 diff --git a/test/src/unit-unicode.cpp b/test/src/unit-unicode.cpp index 3b2a239b73..97fc2751aa 100644 --- a/test/src/unit-unicode.cpp +++ b/test/src/unit-unicode.cpp @@ -1076,3 +1076,395 @@ TEST_CASE("Unicode", "[hide]") CHECK_THROWS_AS(json::parse("\xef\xbb\xbb"), json::parse_error&); } } + +void roundtrip(bool success_expected, const std::string& s); + +void roundtrip(bool success_expected, const std::string& s) +{ + CAPTURE(s); + + // create JSON string value + json j = s; + // create JSON text + std::string ps = std::string("\"") + s + "\""; + + if (success_expected) + { + // serialization succeeds + CHECK_NOTHROW(j.dump()); + + // exclude parse test for U+0000 + if (s[0] != '\0') + { + // parsing JSON text succeeds + CHECK_NOTHROW(json::parse(ps)); + } + + // roundtrip succeeds + CHECK_NOTHROW(json::parse(j.dump())); + + // after roundtrip, the same string is stored + json jr = json::parse(j.dump()); + CHECK(jr.get() == s); + } + else + { + // serialization fails + CHECK_THROWS_AS(j.dump(), json::type_error&); + + // parsing JSON text fails + CHECK_THROWS_AS(json::parse(ps), json::parse_error&); + } +} + +TEST_CASE("Markus Kuhn's UTF-8 decoder capability and stress test") +{ + // Markus Kuhn - 2015-08-28 - CC BY 4.0 + // http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + + SECTION("1 Some correct UTF-8 text") + { + roundtrip(true, "κόσμε"); + } + + SECTION("2 Boundary condition test cases") + { + SECTION("2.1 First possible sequence of a certain length") + { + // 2.1.1 1 byte (U-00000000) + roundtrip(true, std::string("\0", 1)); + // 2.1.2 2 bytes (U-00000080) + roundtrip(true, "\xc2\x80"); + // 2.1.3 3 bytes (U-00000800) + roundtrip(true, "\xe0\xa0\x80"); + // 2.1.4 4 bytes (U-00010000) + roundtrip(true, "\xf0\x90\x80\x80"); + + // 2.1.5 5 bytes (U-00200000) + roundtrip(false, "\xF8\x88\x80\x80\x80"); + // 2.1.6 6 bytes (U-04000000) + roundtrip(false, "\xFC\x84\x80\x80\x80\x80"); + } + + SECTION("2.2 Last possible sequence of a certain length") + { + // 2.2.1 1 byte (U-0000007F) + roundtrip(true, "\x7f"); + // 2.2.2 2 bytes (U-000007FF) + roundtrip(true, "\xdf\xbf"); + // 2.2.3 3 bytes (U-0000FFFF) + roundtrip(true, "\xef\xbf\xbf"); + + // 2.2.4 4 bytes (U-001FFFFF) + roundtrip(false, "\xF7\xBF\xBF\xBF"); + // 2.2.5 5 bytes (U-03FFFFFF) + roundtrip(false, "\xFB\xBF\xBF\xBF\xBF"); + // 2.2.6 6 bytes (U-7FFFFFFF) + roundtrip(false, "\xFD\xBF\xBF\xBF\xBF\xBF"); + } + + SECTION("2.3 Other boundary conditions") + { + // 2.3.1 U-0000D7FF = ed 9f bf + roundtrip(true, "\xed\x9f\xbf"); + // 2.3.2 U-0000E000 = ee 80 80 + roundtrip(true, "\xee\x80\x80"); + // 2.3.3 U-0000FFFD = ef bf bd + roundtrip(true, "\xef\xbf\xbd"); + // 2.3.4 U-0010FFFF = f4 8f bf bf + roundtrip(true, "\xf4\x8f\xbf\xbf"); + + // 2.3.5 U-00110000 = f4 90 80 80 + roundtrip(false, "\xf4\x90\x80\x80"); + } + } + + SECTION("3 Malformed sequences") + { + SECTION("3.1 Unexpected continuation bytes") + { + // Each unexpected continuation byte should be separately signalled as a + // malformed sequence of its own. + + // 3.1.1 First continuation byte 0x80 + roundtrip(false, "\x80"); + // 3.1.2 Last continuation byte 0xbf + roundtrip(false, "\xbf"); + + // 3.1.3 2 continuation bytes + roundtrip(false, "\x80\xbf"); + // 3.1.4 3 continuation bytes + roundtrip(false, "\x80\xbf\x80"); + // 3.1.5 4 continuation bytes + roundtrip(false, "\x80\xbf\x80\xbf"); + // 3.1.6 5 continuation bytes + roundtrip(false, "\x80\xbf\x80\xbf\x80"); + // 3.1.7 6 continuation bytes + roundtrip(false, "\x80\xbf\x80\xbf\x80\xbf"); + // 3.1.8 7 continuation bytes + roundtrip(false, "\x80\xbf\x80\xbf\x80\xbf\x80"); + + // 3.1.9 Sequence of all 64 possible continuation bytes (0x80-0xbf) + roundtrip(false, "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"); + } + + SECTION("3.2 Lonely start characters") + { + // 3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf) + roundtrip(false, "\xc0 \xc1 \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 \xca \xcb \xcc \xcd \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 \xd9 \xda \xdb \xdc \xdd \xde \xdf"); + // 3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef) + roundtrip(false, "\xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 \xe8 \xe9 \xea \xeb \xec \xed \xee \xef"); + // 3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7) + roundtrip(false, "\xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7"); + // 3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb) + roundtrip(false, "\xf8 \xf9 \xfa \xfb"); + // 3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd) + roundtrip(false, "\xfc \xfd"); + } + + SECTION("3.3 Sequences with last continuation byte missing") + { + // All bytes of an incomplete sequence should be signalled as a single + // malformed sequence, i.e., you should see only a single replacement + // character in each of the next 10 tests. (Characters as in section 2) + + // 3.3.1 2-byte sequence with last byte missing (U+0000) + roundtrip(false, "\xc0"); + // 3.3.2 3-byte sequence with last byte missing (U+0000) + roundtrip(false, "\xe0\x80"); + // 3.3.3 4-byte sequence with last byte missing (U+0000) + roundtrip(false, "\xf0\x80\x80"); + // 3.3.4 5-byte sequence with last byte missing (U+0000) + roundtrip(false, "\xf8\x80\x80\x80"); + // 3.3.5 6-byte sequence with last byte missing (U+0000) + roundtrip(false, "\xfc\x80\x80\x80\x80"); + // 3.3.6 2-byte sequence with last byte missing (U-000007FF) + roundtrip(false, "\xdf"); + // 3.3.7 3-byte sequence with last byte missing (U-0000FFFF) + roundtrip(false, "\xef\xbf"); + // 3.3.8 4-byte sequence with last byte missing (U-001FFFFF) + roundtrip(false, "\xf7\xbf\xbf"); + // 3.3.9 5-byte sequence with last byte missing (U-03FFFFFF) + roundtrip(false, "\xfb\xbf\xbf\xbf"); + // 3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF) + roundtrip(false, "\xfd\xbf\xbf\xbf\xbf"); + } + + SECTION("3.4 Concatenation of incomplete sequences") + { + // All the 10 sequences of 3.3 concatenated, you should see 10 malformed + // sequences being signalled: + roundtrip(false, "\xc0\xe0\x80\xf0\x80\x80\xf8\x80\x80\x80\xfc\x80\x80\x80\x80\xdf\xef\xbf\xf7\xbf\xbf\xfb\xbf\xbf\xbf\xfd\xbf\xbf\xbf\xbf"); + } + + SECTION("3.5 Impossible bytes") + { + // The following two bytes cannot appear in a correct UTF-8 string + + // 3.5.1 fe + roundtrip(false, "\xfe"); + // 3.5.2 ff + roundtrip(false, "\xff"); + // 3.5.3 fe fe ff ff + roundtrip(false, "\xfe\xfe\xff\xff"); + } + } + + SECTION("4 Overlong sequences") + { + // The following sequences are not malformed according to the letter of + // the Unicode 2.0 standard. However, they are longer then necessary and + // a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8 + // decoder" should reject them just like malformed sequences for two + // reasons: (1) It helps to debug applications if overlong sequences are + // not treated as valid representations of characters, because this helps + // to spot problems more quickly. (2) Overlong sequences provide + // alternative representations of characters, that could maliciously be + // used to bypass filters that check only for ASCII characters. For + // instance, a 2-byte encoded line feed (LF) would not be caught by a + // line counter that counts only 0x0a bytes, but it would still be + // processed as a line feed by an unsafe UTF-8 decoder later in the + // pipeline. From a security point of view, ASCII compatibility of UTF-8 + // sequences means also, that ASCII characters are *only* allowed to be + // represented by ASCII bytes in the range 0x00-0x7f. To ensure this + // aspect of ASCII compatibility, use only "safe UTF-8 decoders" that + // reject overlong UTF-8 sequences for which a shorter encoding exists. + + SECTION("4.1 Examples of an overlong ASCII character") + { + // With a safe UTF-8 decoder, all of the following five overlong + // representations of the ASCII character slash ("/") should be rejected + // like a malformed UTF-8 sequence, for instance by substituting it with + // a replacement character. If you see a slash below, you do not have a + // safe UTF-8 decoder! + + // 4.1.1 U+002F = c0 af + roundtrip(false, "\xc0\xaf"); + // 4.1.2 U+002F = e0 80 af + roundtrip(false, "\xe0\x80\xaf"); + // 4.1.3 U+002F = f0 80 80 af + roundtrip(false, "\xf0\x80\x80\xaf"); + // 4.1.4 U+002F = f8 80 80 80 af + roundtrip(false, "\xf8\x80\x80\x80\xaf"); + // 4.1.5 U+002F = fc 80 80 80 80 af + roundtrip(false, "\xfc\x80\x80\x80\x80\xaf"); + } + + SECTION("4.2 Maximum overlong sequences") + { + // Below you see the highest Unicode value that is still resulting in an + // overlong sequence if represented with the given number of bytes. This + // is a boundary test for safe UTF-8 decoders. All five characters should + // be rejected like malformed UTF-8 sequences. + + // 4.2.1 U-0000007F = c1 bf + roundtrip(false, "\xc1\xbf"); + // 4.2.2 U-000007FF = e0 9f bf + roundtrip(false, "\xe0\x9f\xbf"); + // 4.2.3 U-0000FFFF = f0 8f bf bf + roundtrip(false, "\xf0\x8f\xbf\xbf"); + // 4.2.4 U-001FFFFF = f8 87 bf bf bf + roundtrip(false, "\xf8\x87\xbf\xbf\xbf"); + // 4.2.5 U-03FFFFFF = fc 83 bf bf bf bf + roundtrip(false, "\xfc\x83\xbf\xbf\xbf\xbf"); + } + + SECTION("4.3 Overlong representation of the NUL character") + { + // The following five sequences should also be rejected like malformed + // UTF-8 sequences and should not be treated like the ASCII NUL + // character. + + // 4.3.1 U+0000 = c0 80 + roundtrip(false, "\xc0\x80"); + // 4.3.2 U+0000 = e0 80 80 + roundtrip(false, "\xe0\x80\x80"); + // 4.3.3 U+0000 = f0 80 80 80 + roundtrip(false, "\xf0\x80\x80\x80"); + // 4.3.4 U+0000 = f8 80 80 80 80 + roundtrip(false, "\xf8\x80\x80\x80\x80"); + // 4.3.5 U+0000 = fc 80 80 80 80 80 + roundtrip(false, "\xfc\x80\x80\x80\x80\x80"); + } + } + + SECTION("5 Illegal code positions") + { + // The following UTF-8 sequences should be rejected like malformed + // sequences, because they never represent valid ISO 10646 characters and + // a UTF-8 decoder that accepts them might introduce security problems + // comparable to overlong UTF-8 sequences. + + SECTION("5.1 Single UTF-16 surrogates") + { + // 5.1.1 U+D800 = ed a0 80 + roundtrip(false, "\xed\xa0\x80"); + // 5.1.2 U+DB7F = ed ad bf + roundtrip(false, "\xed\xad\xbf"); + // 5.1.3 U+DB80 = ed ae 80 + roundtrip(false, "\xed\xae\x80"); + // 5.1.4 U+DBFF = ed af bf + roundtrip(false, "\xed\xaf\xbf"); + // 5.1.5 U+DC00 = ed b0 80 + roundtrip(false, "\xed\xb0\x80"); + // 5.1.6 U+DF80 = ed be 80 + roundtrip(false, "\xed\xbe\x80"); + // 5.1.7 U+DFFF = ed bf bf + roundtrip(false, "\xed\xbf\xbf"); + } + + SECTION("5.2 Paired UTF-16 surrogates") + { + // 5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80 + roundtrip(false, "\xed\xa0\x80\xed\xb0\x80"); + // 5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf + roundtrip(false, "\xed\xa0\x80\xed\xbf\xbf"); + // 5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80 + roundtrip(false, "\xed\xad\xbf\xed\xb0\x80"); + // 5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf + roundtrip(false, "\xed\xad\xbf\xed\xbf\xbf"); + // 5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80 + roundtrip(false, "\xed\xae\x80\xed\xb0\x80"); + // 5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf + roundtrip(false, "\xed\xae\x80\xed\xbf\xbf"); + // 5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80 + roundtrip(false, "\xed\xaf\xbf\xed\xb0\x80"); + // 5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf + roundtrip(false, "\xed\xaf\xbf\xed\xbf\xbf"); + } + + SECTION("5.3 Noncharacter code positions") + { + // The following "noncharacters" are "reserved for internal use" by + // applications, and according to older versions of the Unicode Standard + // "should never be interchanged". Unicode Corrigendum #9 dropped the + // latter restriction. Nevertheless, their presence in incoming UTF-8 data + // can remain a potential security risk, depending on what use is made of + // these codes subsequently. Examples of such internal use: + // + // - Some file APIs with 16-bit characters may use the integer value -1 + // = U+FFFF to signal an end-of-file (EOF) or error condition. + // + // - In some UTF-16 receivers, code point U+FFFE might trigger a + // byte-swap operation (to convert between UTF-16LE and UTF-16BE). + // + // With such internal use of noncharacters, it may be desirable and safer + // to block those code points in UTF-8 decoders, as they should never + // occur legitimately in incoming UTF-8 data, and could trigger unsafe + // behaviour in subsequent processing. + + // Particularly problematic noncharacters in 16-bit applications: + + // 5.3.1 U+FFFE = ef bf be + roundtrip(true, "\xef\xbf\xbe"); + // 5.3.2 U+FFFF = ef bf bf + roundtrip(true, "\xef\xbf\xbf"); + + // 5.3.3 U+FDD0 .. U+FDEF + roundtrip(true, "\xEF\xB7\x90"); + roundtrip(true, "\xEF\xB7\x91"); + roundtrip(true, "\xEF\xB7\x92"); + roundtrip(true, "\xEF\xB7\x93"); + roundtrip(true, "\xEF\xB7\x94"); + roundtrip(true, "\xEF\xB7\x95"); + roundtrip(true, "\xEF\xB7\x96"); + roundtrip(true, "\xEF\xB7\x97"); + roundtrip(true, "\xEF\xB7\x98"); + roundtrip(true, "\xEF\xB7\x99"); + roundtrip(true, "\xEF\xB7\x9A"); + roundtrip(true, "\xEF\xB7\x9B"); + roundtrip(true, "\xEF\xB7\x9C"); + roundtrip(true, "\xEF\xB7\x9D"); + roundtrip(true, "\xEF\xB7\x9E"); + roundtrip(true, "\xEF\xB7\x9F"); + roundtrip(true, "\xEF\xB7\xA0"); + roundtrip(true, "\xEF\xB7\xA1"); + roundtrip(true, "\xEF\xB7\xA2"); + roundtrip(true, "\xEF\xB7\xA3"); + roundtrip(true, "\xEF\xB7\xA4"); + roundtrip(true, "\xEF\xB7\xA5"); + roundtrip(true, "\xEF\xB7\xA6"); + roundtrip(true, "\xEF\xB7\xA7"); + roundtrip(true, "\xEF\xB7\xA8"); + roundtrip(true, "\xEF\xB7\xA9"); + roundtrip(true, "\xEF\xB7\xAA"); + roundtrip(true, "\xEF\xB7\xAB"); + roundtrip(true, "\xEF\xB7\xAC"); + roundtrip(true, "\xEF\xB7\xAD"); + roundtrip(true, "\xEF\xB7\xAE"); + roundtrip(true, "\xEF\xB7\xAF"); + + // 5.3.4 U+nFFFE U+nFFFF (for n = 1..10) + roundtrip(true, "\xF0\x9F\xBF\xBF"); + roundtrip(true, "\xF0\xAF\xBF\xBF"); + roundtrip(true, "\xF0\xBF\xBF\xBF"); + roundtrip(true, "\xF1\x8F\xBF\xBF"); + roundtrip(true, "\xF1\x9F\xBF\xBF"); + roundtrip(true, "\xF1\xAF\xBF\xBF"); + roundtrip(true, "\xF1\xBF\xBF\xBF"); + roundtrip(true, "\xF2\x8F\xBF\xBF"); + roundtrip(true, "\xF2\x9F\xBF\xBF"); + roundtrip(true, "\xF2\xAF\xBF\xBF"); + } + } +}