1#!./perl 2 3BEGIN { 4 chdir 't' if -d 't'; 5 require './test.pl'; 6 set_up_inc('../lib'); 7} 8 9$|=1; 10 11{ 12 my $wide = v256; 13 use bytes; 14 my $ordwide = ord($wide); 15 printf "# under use bytes ord(v256) = 0x%02x\n", $ordwide; 16 skip_all('UTF-8-centric tests (not valid for UTF-EBCDIC)') if $ordwide == 140; 17 # This could be ported to EBCDIC, but a lot of trouble. 18 # ext/XS-APItest/t/utf8.t contains comprehensive tests for both platforms 19 20 if ($ordwide != 196) { 21 printf "# v256 starts with 0x%02x\n", $ordwide; 22 } 23} 24 25no utf8; 26 27my $is64bit = length sprintf("%x", ~0) > 8; 28 29foreach (<DATA>) { 30 if (/^(?:\d+(?:\.\d+)?)\s/ || /^#/) { 31 # print "# $_\n"; 32 } elsif (my ($id, $okay, $Unicode, $byteslen, $hex, $charslen, $experr) 33 = /^(\d+\.\d+\.\d+[bu]?) # ID 34 \s+(y|n|N-?\d+(?:,\d+)?) # expect to pass or fail 35 # 'n' means expect one diagnostic 36 # 'N\d+' means expect this 37 # number of diagnostics 38 # 'N\d+,\d+' means expect the first 39 # number of diagnostics 40 # on a 32-bit system; the 41 # second number on a 42 # 64-bit one 43 \s+([0-9a-f]{1,8}(?:,[0-9a-f]{1,8})*|-) # Unicode characters 44 \s+(\d+) # number of octets 45 \s+([0-9a-f]{2}(?::[0-9a-f]{2})*) # octets in hex 46 \s+(\d+|-) # number of characters 47 (?:\s+(.+))? # expected error (or comment) 48 $/x) { 49 my @hex = split(/:/, $hex); 50 is(scalar @hex, $byteslen, 'Amount of hex tallies with byteslen'); 51 my $octets = join '', map {chr hex $_} @hex; 52 is(length $octets, $byteslen, 'Number of octets tallies with byteslen'); 53 if ($okay eq 'y') { 54 my @chars = map {hex $_} split ',', $Unicode; 55 is(scalar @chars, $charslen, 'Amount of hex tallies with charslen'); 56 my @got; 57 warning_is(sub {@got = unpack 'C0U*', $octets}, undef, 58 "No warnings expected for $id"); 59 is("@got", "@chars", 'Got expected Unicode characters'); 60 } elsif ($okay eq 'n') { 61 isnt($experr, '', "Expected warning for $id provided"); 62 warnings_like(sub {unpack 'C0U*', $octets}, [qr/$experr/], 63 "Only expected warning for $id"); 64 } elsif ($okay !~ /^N-?(\d+)(?:,(\d+))?/) { 65 is($okay, 'n', "Confused test description for $id"); 66 } else { 67 my $expect32 = $1; 68 my $expect64 = $2 // $expect32; 69 my $expect = ($is64bit) ? $expect64 : $expect32; 70 my @warnings; 71 72 { 73 local $SIG{__WARN__} = sub { 74 print "# $id: @_"; 75 push @warnings, "@_"; 76 }; 77 unpack 'C0U*', $octets; 78 } 79 80 unless (is(scalar @warnings, $expect, "Expected number of warnings for $id seen")) { 81 note(join "", "Got:\n", @warnings); 82 } 83 isnt($experr, '', "Expected first warning for $id provided"); 84 85 my $message; 86 my $after = ""; 87 if ($expect64 < $expect32 && ! $is64bit) { 88 # This is needed for code points above IV_MAX 89 #if ( substr($octets, 0, 1) gt "\xfe" 90 # || ( substr($octets, 0, 1) eq "\xfe" 91 # && length $octets > 1 92 # && substr($octets, 1, 1) le "\xbf" 93 # && substr($octets, 1, 1) ge "\x80")) 94 #{ 95 like($warnings[0], qr/overflow/, "overflow warning for $id seen"); 96 shift @warnings; 97 $after .= "overflow"; 98 #} 99 } 100 101 # The data below assumes that if there is both a 'short' and 102 # 'non-continuation' malformation, the latter has precedence. But 103 # that has changed, and rather than mess with the data, this works 104 # around that. 105 if ( @warnings > 1 106 && $warnings[0] =~ /short/ 107 && $warnings[1] =~ /unexpected non-continuation/) 108 { 109 $after .= " and " if $after; 110 $after .= "short"; 111 shift @warnings; 112 } 113 $after = "after $after " if $after; 114 115 like($warnings[0], qr/$experr/, "Expected first warning ${after}for $id seen"); 116 local $::TODO; 117 if ($expect < 0) { 118 $expect = -$expect; 119 $::TODO = "Markus Kuhn states that $expect invalid sequences should be signalled"; 120 } 121 122 } 123 } else { 124 fail("unknown format '$_'"); 125 } 126} 127 128done_testing(); 129 130# This table is based on Markus Kuhn's UTF-8 Decode Stress Tester, 131# http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt, 132# version dated 2015-08-28. 133# 134# See the code that parses these lines for comments as to the column meanings 135 136__DATA__ 1371 Correct UTF-8 1381.1.1 y 3ba,1f79,3c3,3bc,3b5 11 ce:ba:e1:bd:b9:cf:83:ce:bc:ce:b5 5 1392 Boundary conditions 1402.1 First possible sequence of certain length 1412.1.1 y 0 1 00 1 1422.1.2 y 80 2 c2:80 1 1432.1.3 y 800 3 e0:a0:80 1 1442.1.4 y 10000 4 f0:90:80:80 1 1452.1.5 y 200000 5 f8:88:80:80:80 1 1462.1.6 y 4000000 6 fc:84:80:80:80:80 1 1472.2 Last possible sequence of certain length 1482.2.1 y 7f 1 7f 1 1492.2.2 y 7ff 2 df:bf 1 150# The ffff is legal by default since 872c91ae155f6880 1512.2.3 y ffff 3 ef:bf:bf 1 character 0xffff 1522.2.4 y 1fffff 4 f7:bf:bf:bf 1 1532.2.5 y 3ffffff 5 fb:bf:bf:bf:bf 1 1542.2.6 y 7fffffff 6 fd:bf:bf:bf:bf:bf 1 1552.3 Other boundary conditions 1562.3.1 y d7ff 3 ed:9f:bf 1 1572.3.2 y e000 3 ee:80:80 1 1582.3.3 y fffd 3 ef:bf:bd 1 1592.3.4 y 10ffff 4 f4:8f:bf:bf 1 1602.3.5 y 110000 4 f4:90:80:80 1 1613 Malformed sequences 1623.1 Unexpected continuation bytes 1633.1.1 n - 1 80 - unexpected continuation byte 0x80 1643.1.2 n - 1 bf - unexpected continuation byte 0xbf 1653.1.3 N2 - 2 80:bf - unexpected continuation byte 0x80 1663.1.4 N3 - 3 80:bf:80 - unexpected continuation byte 0x80 1673.1.5 N4 - 4 80:bf:80:bf - unexpected continuation byte 0x80 1683.1.6 N5 - 5 80:bf:80:bf:80 - unexpected continuation byte 0x80 1693.1.7 N6 - 6 80:bf:80:bf:80:bf - unexpected continuation byte 0x80 1703.1.8 N7 - 7 80:bf:80:bf:80:bf:80 - unexpected continuation byte 0x80 1713.1.9 N64 - 64 80:81:82:83:84:85:86:87:88:89:8a:8b:8c:8d:8e:8f:90:91:92:93:94:95:96:97:98:99:9a:9b:9c:9d:9e:9f:a0:a1:a2:a3:a4:a5:a6:a7:a8:a9:aa:ab:ac:ad:ae:af:b0:b1:b2:b3:b4:b5:b6:b7:b8:b9:ba:bb:bc:bd:be:bf - unexpected continuation byte 0x80 1723.2 Lonely start characters 1733.2.1 N34 - 64 c0:20:c1:20:c2:20:c3:20:c4:20:c5:20:c6:20:c7:20:c8:20:c9:20:ca:20:cb:20:cc:20:cd:20:ce:20:cf:20:d0:20:d1:20:d2:20:d3:20:d4:20:d5:20:d6:20:d7:20:d8:20:d9:20:da:20:db:20:dc:20:dd:20:de:20:df:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xc0 1743.2.2 N17 - 32 e0:20:e1:20:e2:20:e3:20:e4:20:e5:20:e6:20:e7:20:e8:20:e9:20:ea:20:eb:20:ec:20:ed:20:ee:20:ef:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xe0 1753.2.3 N9 - 16 f0:20:f1:20:f2:20:f3:20:f4:20:f5:20:f6:20:f7:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xf0 1763.2.4 N6 - 8 f8:20:f9:20:fa:20:fb:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xf8 1773.2.5 N4 - 4 fc:20:fd:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xfc 1783.3 Sequences with last continuation byte missing 1793.3.1 N2 - 1 c0 - 1 byte available, need 2 1803.3.2 N2 - 2 e0:80 - 2 bytes available, need 3 1813.3.3 N2 - 3 f0:80:80 - 3 bytes available, need 4 1823.3.4 N2 - 4 f8:80:80:80 - 4 bytes available, need 5 1833.3.5 N2 - 5 fc:80:80:80:80 - 5 bytes available, need 6 1843.3.6 n - 1 df - 1 byte available, need 2 1853.3.7 n - 2 ef:bf - 2 bytes available, need 3 1863.3.8 n - 3 f7:bf:bf - 3 bytes available, need 4 1873.3.9 n - 4 fb:bf:bf:bf - 4 bytes available, need 5 1883.3.10 n - 5 fd:bf:bf:bf:bf - 5 bytes available, need 6 1893.4 Concatenation of incomplete sequences 1903.4.1 N15 - 30 c0:e0:80:f0:80:80:f8:80:80:80:fc:80:80:80:80:df:ef:bf:f7:bf:bf:fb:bf:bf:bf:fd:bf:bf:bf:bf - unexpected non-continuation byte 0xe0, immediately after start byte 0xc0 1913.5 Impossible bytes (but not with Perl's extended UTF-8) 1923.5.1 n - 1 fe - 1 byte available, need 7 1933.5.2 n - 1 ff - 1 byte available, need 13 1943.5.3 N7 - 4 fe:fe:ff:ff - byte 0xfe 1954 Overlong sequences 1964.1 Examples of an overlong ASCII character 1974.1.1 n - 2 c0:af - overlong 1984.1.2 n - 3 e0:80:af - overlong 1994.1.3 n - 4 f0:80:80:af - overlong 2004.1.4 n - 5 f8:80:80:80:af - overlong 2014.1.5 n - 6 fc:80:80:80:80:af - overlong 2024.2 Maximum overlong sequences 2034.2.1 n - 2 c1:bf - overlong 2044.2.2 n - 3 e0:9f:bf - overlong 2054.2.3 n - 4 f0:8f:bf:bf - overlong 2064.2.4 n - 5 f8:87:bf:bf:bf - overlong 2074.2.5 n - 6 fc:83:bf:bf:bf:bf - overlong 2084.3 Overlong representation of the NUL character 2094.3.1 n - 2 c0:80 - overlong 2104.3.2 n - 3 e0:80:80 - overlong 2114.3.3 n - 4 f0:80:80:80 - overlong 2124.3.4 n - 5 f8:80:80:80:80 - overlong 2134.3.5 n - 6 fc:80:80:80:80:80 - overlong 2145 Illegal code positions 2155.1 Single UTF-16 surrogates 2165.1.1 y d800 3 ed:a0:80 1 UTF-16 surrogate 0xd800 2175.1.2 y db7f 3 ed:ad:bf 1 UTF-16 surrogate 0xdb7f 2185.1.3 y db80 3 ed:ae:80 1 UTF-16 surrogate 0xdb80 2195.1.4 y dbff 3 ed:af:bf 1 UTF-16 surrogate 0xdbff 2205.1.5 y dc00 3 ed:b0:80 1 UTF-16 surrogate 0xdc00 2215.1.6 y df80 3 ed:be:80 1 UTF-16 surrogate 0xdf80 2225.1.7 y dfff 3 ed:bf:bf 1 UTF-16 surrogate 0xdfff 2235.2 Paired UTF-16 surrogates 2245.2.1 y d800,dc00 6 ed:a0:80:ed:b0:80 2 UTF-16 surrogates 0xd800, dc00 2255.2.2 y d800,dfff 6 ed:a0:80:ed:bf:bf 2 UTF-16 surrogates 0xd800, dfff 2265.2.3 y db7f,dc00 6 ed:ad:bf:ed:b0:80 2 UTF-16 surrogates 0xdb7f, dc00 2275.2.4 y db7f,dfff 6 ed:ad:bf:ed:bf:bf 2 UTF-16 surrogates 0xdb7f, dfff 2285.2.5 y db80,dc00 6 ed:ae:80:ed:b0:80 2 UTF-16 surrogates 0xdb80, dc00 2295.2.6 y db80,dfff 6 ed:ae:80:ed:bf:bf 2 UTF-16 surrogates 0xdb80, dfff 2305.2.7 y dbff,dc00 6 ed:af:bf:ed:b0:80 2 UTF-16 surrogates 0xdbff, dc00 2315.2.8 y dbff,dfff 6 ed:af:bf:ed:bf:bf 2 UTF-16 surrogates 0xdbff, dfff 2325.3 Other illegal code positions 2335.3.1 y fffe 3 ef:bf:be 1 byte order mark 0xfffe 234# The ffff is legal by default since 872c91ae155f6880 2355.3.2 y ffff 3 ef:bf:bf 1 non-character 0xffff 2365.3.3 y fdd0 3 ef:b7:90 1 non-character 0xfdd0 2375.3.3 y fdd1 3 ef:b7:91 1 non-character 0xfdd1 2385.3.3 y fdd2 3 ef:b7:92 1 non-character 0xfdd2 2395.3.3 y fdd3 3 ef:b7:93 1 non-character 0xfdd3 2405.3.3 y fdd4 3 ef:b7:94 1 non-character 0xfdd4 2415.3.3 y fdd5 3 ef:b7:95 1 non-character 0xfdd5 2425.3.3 y fdd6 3 ef:b7:96 1 non-character 0xfdd6 2435.3.3 y fdd7 3 ef:b7:97 1 non-character 0xfdd7 2445.3.3 y fdd8 3 ef:b7:98 1 non-character 0xfdd8 2455.3.3 y fdd9 3 ef:b7:99 1 non-character 0xfdd9 2465.3.3 y fdda 3 ef:b7:9a 1 non-character 0xfdda 2475.3.3 y fddb 3 ef:b7:9b 1 non-character 0xfddb 2485.3.3 y fddc 3 ef:b7:9c 1 non-character 0xfddc 2495.3.3 y fddd 3 ef:b7:9d 1 non-character 0xfddd 2505.3.3 y fdde 3 ef:b7:9e 1 non-character 0xfdde 2515.3.3 y fddf 3 ef:b7:9f 1 non-character 0xfddf 2525.3.3 y fde0 3 ef:b7:a0 1 non-character 0xfde0 2535.3.3 y fde1 3 ef:b7:a1 1 non-character 0xfde1 2545.3.3 y fde2 3 ef:b7:a2 1 non-character 0xfde2 2555.3.3 y fde3 3 ef:b7:a3 1 non-character 0xfde3 2565.3.3 y fde4 3 ef:b7:a4 1 non-character 0xfde4 2575.3.3 y fde5 3 ef:b7:a5 1 non-character 0xfde5 2585.3.3 y fde6 3 ef:b7:a6 1 non-character 0xfde6 2595.3.3 y fde7 3 ef:b7:a7 1 non-character 0xfde7 2605.3.3 y fde8 3 ef:b7:a8 1 non-character 0xfde8 2615.3.3 y fde9 3 ef:b7:a9 1 non-character 0xfde9 2625.3.3 y fdea 3 ef:b7:aa 1 non-character 0xfdea 2635.3.3 y fdeb 3 ef:b7:ab 1 non-character 0xfdeb 2645.3.3 y fdec 3 ef:b7:ac 1 non-character 0xfdec 2655.3.3 y fded 3 ef:b7:ad 1 non-character 0xfded 2665.3.3 y fdee 3 ef:b7:ae 1 non-character 0xfdee 2675.3.3 y fdef 3 ef:b7:af 1 non-character 0xfdef 2685.3.4 y 1fffe 4 f0:9f:bf:be 1 non-character 0x1fffe 2695.3.4 y 1ffff 4 f0:9f:bf:bf 1 non-character 0x1ffff 2705.3.4 y 2fffe 4 f0:af:bf:be 1 non-character 0x2fffe 2715.3.4 y 2ffff 4 f0:af:bf:bf 1 non-character 0x2ffff 2725.3.4 y 3fffe 4 f0:bf:bf:be 1 non-character 0x3fffe 2735.3.4 y 3ffff 4 f0:bf:bf:bf 1 non-character 0x3ffff 2745.3.4 y 4fffe 4 f1:8f:bf:be 1 non-character 0x4fffe 2755.3.4 y 4ffff 4 f1:8f:bf:bf 1 non-character 0x4ffff 2765.3.4 y 5fffe 4 f1:9f:bf:be 1 non-character 0x5fffe 2775.3.4 y 5ffff 4 f1:9f:bf:bf 1 non-character 0x5ffff 2785.3.4 y 6fffe 4 f1:af:bf:be 1 non-character 0x6fffe 2795.3.4 y 6ffff 4 f1:af:bf:bf 1 non-character 0x6ffff 2805.3.4 y 7fffe 4 f1:bf:bf:be 1 non-character 0x7fffe 2815.3.4 y 7ffff 4 f1:bf:bf:bf 1 non-character 0x7ffff 2825.3.4 y 8fffe 4 f2:8f:bf:be 1 non-character 0x8fffe 2835.3.4 y 8ffff 4 f2:8f:bf:bf 1 non-character 0x8ffff 2845.3.4 y 9fffe 4 f2:9f:bf:be 1 non-character 0x9fffe 2855.3.4 y 9ffff 4 f2:9f:bf:bf 1 non-character 0x9ffff 2865.3.4 y afffe 4 f2:af:bf:be 1 non-character 0xafffe 2875.3.4 y affff 4 f2:af:bf:bf 1 non-character 0xaffff 2885.3.4 y bfffe 4 f2:bf:bf:be 1 non-character 0xbfffe 2895.3.4 y bffff 4 f2:bf:bf:bf 1 non-character 0xbffff 2905.3.4 y cfffe 4 f3:8f:bf:be 1 non-character 0xcfffe 2915.3.4 y cffff 4 f3:8f:bf:bf 1 non-character 0xcffff 2925.3.4 y dfffe 4 f3:9f:bf:be 1 non-character 0xdfffe 2935.3.4 y dffff 4 f3:9f:bf:bf 1 non-character 0xdffff 2945.3.4 y efffe 4 f3:af:bf:be 1 non-character 0xefffe 2955.3.4 y effff 4 f3:af:bf:bf 1 non-character 0xeffff 2965.3.4 y ffffe 4 f3:bf:bf:be 1 non-character 0xffffe 2975.3.4 y fffff 4 f3:bf:bf:bf 1 non-character 0xfffff 2985.3.4 y 10fffe 4 f4:8f:bf:be 1 non-character 0x10fffe 2995.3.4 y 10ffff 4 f4:8f:bf:bf 1 non-character 0x10ffff 300