1#!./perl 2 3BEGIN { 4 chdir 't' if -d 't'; 5 require './test.pl'; require './charset_tools.pl'; 6 set_up_inc('../lib'); 7} 8skip_all_without_perlio(); 9 10no utf8; # needed for use utf8 not griping about the raw octets 11 12 13plan(tests => 62); 14 15$| = 1; 16 17my $a_file = tempfile(); 18 19open(F,"+>:utf8",$a_file); 20print F chr(0x100).'�'; 21cmp_ok( tell(F), '==', 4, tell(F) ); 22print F "\n"; 23cmp_ok( tell(F), '>=', 5, tell(F) ); 24seek(F,0,0); 25is( getc(F), chr(0x100) ); 26is( getc(F), "�" ); 27is( getc(F), "\n" ); 28seek(F,0,0); 29binmode(F,":bytes"); 30 31# Byte representation of these characters 32my $U_100 = byte_utf8a_to_utf8n("\xc4\x80"); 33my $POUND_SIGN = byte_utf8a_to_utf8n("\xc2\xa3"); 34 35my $chr = substr($U_100, 0, 1); 36is( getc(F), $chr ); 37$chr = substr($U_100, 1, 1); 38is( getc(F), $chr ); 39$chr = substr($POUND_SIGN, 0, 1); 40is( getc(F), $chr ); 41$chr = substr($POUND_SIGN, 1, 1); 42is( getc(F), $chr ); 43is( getc(F), "\n" ); 44seek(F,0,0); 45binmode(F,":utf8"); 46is( scalar(<F>), "\x{100}�\n" ); 47seek(F,0,0); 48$buf = chr(0x200); 49$count = read(F,$buf,2,1); 50cmp_ok( $count, '==', 2 ); 51is( $buf, "\x{200}\x{100}�" ); 52close(F); 53 54{ 55 $a = chr(300); # This *is* UTF-encoded 56 $b = chr(130); # This is not. 57 58 open F, ">:utf8", $a_file or die $!; 59 print F $a,"\n"; 60 close F; 61 62 open F, "<:utf8", $a_file or die $!; 63 $x = <F>; 64 chomp($x); 65 is( $x, chr(300) ); 66 67 open F, $a_file or die $!; # Not UTF 68 binmode(F, ":bytes"); 69 $x = <F>; 70 chomp($x); 71 $chr = byte_utf8a_to_utf8n(chr(196).chr(172)); 72 is( $x, $chr ); 73 close F; 74 75 open F, ">:utf8", $a_file or die $!; 76 binmode(F); # we write a "\n" and then tell() - avoid CRLF issues. 77 binmode(F,":utf8"); # turn UTF-8-ness back on 78 print F $a; 79 my $y; 80 { my $x = tell(F); 81 { use bytes; $y = length($a);} 82 cmp_ok( $x, '==', $y ); 83 } 84 85 { # Check byte length of $b 86 use bytes; my $y = length($b); 87 cmp_ok( $y, '==', 1 ); 88 } 89 90 print F $b,"\n"; # Don't upgrade $b 91 92 { # Check byte length of $b 93 use bytes; my $y = length($b); 94 cmp_ok( $y, '==', 1 ); 95 } 96 97 { 98 my $x = tell(F); 99 { use bytes; if ($::IS_EBCDIC){$y += 2;}else{$y += 3;}} # EBCDIC ASCII 100 cmp_ok( $x, '==', $y ); 101 } 102 103 close F; 104 105 open F, $a_file or die $!; # Not UTF 106 binmode(F, ":bytes"); 107 $x = <F>; 108 chomp($x); 109 $chr = v196.172.194.130; 110 if ($::IS_EBCDIC) { $chr = v141.83.130; } # EBCDIC 111 is( $x, $chr, sprintf('(%vd)', $x) ); 112 113 open F, "<:utf8", $a_file or die $!; 114 $x = <F>; 115 chomp($x); 116 close F; 117 is( $x, chr(300).chr(130), sprintf('(%vd)', $x) ); 118 119 open F, ">", $a_file or die $!; 120 binmode(F, ":bytes:"); 121 122 # Now let's make it suffer. 123 my $w; 124 { 125 use warnings 'utf8'; 126 local $SIG{__WARN__} = sub { $w = $_[0] }; 127 print F $a; 128 ok( (!$@)); 129 like($w, qr/Wide character in print/i ); 130 } 131} 132 133# Hm. Time to get more evil. 134open F, ">:utf8", $a_file or die $!; 135print F $a; 136binmode(F, ":bytes"); 137print F chr(130)."\n"; 138close F; 139 140open F, "<", $a_file or die $!; 141binmode(F, ":bytes"); 142$x = <F>; chomp $x; 143$chr = v196.172.130; 144if ($::IS_EBCDIC) { $chr = v141.83.130; } # EBCDIC 145is( $x, $chr ); 146 147# Right. 148open F, ">:utf8", $a_file or die $!; 149print F $a; 150close F; 151open F, ">>", $a_file or die $!; 152binmode(F, ":bytes"); 153print F chr(130)."\n"; 154close F; 155 156open F, "<", $a_file or die $!; 157binmode(F, ":bytes"); 158$x = <F>; chomp $x; 159SKIP: { 160 skip("Defaulting to UTF-8 output means that we can't generate a mangled file") 161 if $UTF8_OUTPUT; 162 is( $x, $chr ); 163} 164 165# Now we have a deformed file. 166 167SKIP: { 168 if ($::IS_EBCDIC) { 169 skip("EBCDIC The file isn't deformed in UTF-EBCDIC", 2); 170 } else { 171 my @warnings; 172 open F, "<:utf8", $a_file or die $!; 173 $x = <F>; chomp $x; 174 local $SIG{__WARN__} = sub { push @warnings, $_[0]; }; 175 eval { sprintf "%vd\n", $x }; 176 is (scalar @warnings, 1); 177 like ($warnings[0], qr/Malformed UTF-8 character: \\x82 \(unexpected continuation byte 0x82, with no preceding start byte/); 178 } 179} 180 181close F; 182unlink($a_file); 183 184open F, ">:utf8", $a_file; 185@a = map { chr(1 << ($_ << 2)) } 0..5; # 0x1, 0x10, .., 0x100000 186unshift @a, chr(0); # ... and a null byte in front just for fun 187print F @a; 188close F; 189 190my $c; 191 192# read() should work on characters, not bytes 193open F, "<:utf8", $a_file; 194$a = 0; 195my $failed; 196for (@a) { 197 unless (($c = read(F, $b, 1) == 1) && 198 length($b) == 1 && 199 ord($b) == ord($_) && 200 tell(F) == ($a += bytes::length($b))) { 201 print '# ord($_) == ', ord($_), "\n"; 202 print '# ord($b) == ', ord($b), "\n"; 203 print '# length($b) == ', length($b), "\n"; 204 print '# bytes::length($b) == ', bytes::length($b), "\n"; 205 print '# tell(F) == ', tell(F), "\n"; 206 print '# $a == ', $a, "\n"; 207 print '# $c == ', $c, "\n"; 208 $failed++; 209 last; 210 } 211} 212close F; 213is($failed, undef); 214 215{ 216 # Check that warnings are on on I/O, and that they can be muffled. 217 218 local $SIG{__WARN__} = sub { $@ = shift }; 219 220 undef $@; 221 open F, ">$a_file"; 222 binmode(F, ":bytes"); 223 print F chr(0x100); 224 close(F); 225 226 like( $@, qr/Wide character in print/ ); 227 228 undef $@; 229 open F, ">:utf8", $a_file; 230 print F chr(0x100); 231 close(F); 232 233 isnt( defined $@, !0 ); 234 235 undef $@; 236 open F, ">$a_file"; 237 binmode(F, ":utf8"); 238 print F chr(0x100); 239 close(F); 240 241 isnt( defined $@, !0 ); 242 243 no warnings 'utf8'; 244 245 undef $@; 246 open F, ">$a_file"; 247 print F chr(0x100); 248 close(F); 249 250 isnt( defined $@, !0 ); 251 252 use warnings 'utf8'; 253 254 undef $@; 255 open F, ">$a_file"; 256 binmode(F, ":bytes"); 257 print F chr(0x100); 258 close(F); 259 260 like( $@, qr/Wide character in print/ ); 261} 262 263{ 264 open F, ">:bytes",$a_file; print F "\xde"; close F; 265 266 open F, "<:bytes", $a_file; 267 my $b = chr 0x100; 268 $b .= <F>; 269 is( $b, chr(0x100).chr(0xde), "21395 '.= <>' utf8 vs. bytes" ); 270 close F; 271} 272 273{ 274 open F, ">:utf8",$a_file; print F chr 0x100; close F; 275 276 open F, "<:utf8", $a_file; 277 my $b = "\xde"; 278 $b .= <F>; 279 is( $b, chr(0xde).chr(0x100), "21395 '.= <>' bytes vs. utf8" ); 280 close F; 281} 282 283{ 284 my @a = ( [ 0x007F, "bytes" ], 285 [ 0x0080, "bytes" ], 286 [ 0x0080, "utf8" ], 287 [ 0x0100, "utf8" ] ); 288 my $t = 34; 289 for my $u (@a) { 290 for my $v (@a) { 291 # print "# @$u - @$v\n"; 292 open F, ">$a_file"; 293 binmode(F, ":" . $u->[1]); 294 print F chr($u->[0]); 295 close F; 296 297 open F, "<$a_file"; 298 binmode(F, ":" . $u->[1]); 299 300 my $s = chr($v->[0]); 301 utf8::upgrade($s) if $v->[1] eq "utf8"; 302 303 $s .= <F>; 304 is( $s, chr($v->[0]) . chr($u->[0]), 'rcatline utf8' ); 305 close F; 306 $t++; 307 } 308 } 309 # last test here 49 310} 311 312{ 313 # [perl #23428] Somethings rotten in unicode semantics 314 open F, ">$a_file"; 315 binmode F; 316 $a = "A"; 317 utf8::upgrade($a); 318 syswrite(F, $a); 319 close F; 320 ok(utf8::is_utf8($a), '23428 syswrite should not downgrade scalar' ); 321} 322 323{ 324 # <FH> on a :utf8 stream should complain immediately with -w 325 # if it finds bad UTF-8 (:encoding(utf8) works this way) 326 use warnings 'utf8'; 327 undef $@; 328 local $SIG{__WARN__} = sub { $@ = shift }; 329 open F, ">$a_file"; 330 binmode F; 331 my ($chrE4, $chrF6) = (chr(0xE4), chr(0xF6)); 332 if ($::IS_EBCDIC) # EBCDIC 333 { ($chrE4, $chrF6) = (chr(0x43), chr(0xEC)); } 334 print F "foo", $chrE4, "\n"; 335 print F "foo", $chrF6, "\n"; 336 close F; 337 open F, "<:utf8", $a_file; 338 undef $@; 339 my $line = <F>; 340 my ($chrE4, $chrF6) = ("E4", "F6"); 341 if ($::IS_EBCDIC) { ($chrE4, $chrF6) = ("43", "EC"); } # EBCDIC 342 like( $@, qr/utf8 "\\x$chrE4" does not map to Unicode .+ <F> line 1/, 343 "<:utf8 readline must warn about bad utf8"); 344 undef $@; 345 $line .= <F>; 346 like( $@, qr/utf8 "\\x$chrF6" does not map to Unicode .+ <F> line 2/, 347 "<:utf8 rcatline must warn about bad utf8"); 348 close F; 349} 350 351{ 352 # fixed record reads 353 open F, ">:utf8", $a_file; 354 print F "foo\xE4"; 355 print F "bar\xFE"; 356 print F "\xC0\xC8\xCC\xD2"; 357 print F "a\xE4ab"; 358 print F "a\xE4a"; 359 close F; 360 open F, "<:utf8", $a_file; 361 local $/ = \4; 362 my $line = <F>; 363 is($line, "foo\xE4", "readline with \$/ = \\4"); 364 $line .= <F>; 365 is($line, "foo\xE4bar\xFE", "rcatline with \$/ = \\4"); 366 $line = <F>; 367 is($line, "\xC0\xC8\xCC\xD2", "readline with several encoded characters"); 368 $line = <F>; 369 is($line, "a\xE4ab", "readline with another boundary condition"); 370 $line = <F>; 371 is($line, "a\xE4a", "readline with boundary condition"); 372 close F; 373 374 # badly encoded at EOF 375 open F, ">:raw", $a_file; 376 print F "foo\xEF\xAC"; # truncated \x{FB04} small ligature ffl 377 close F; 378 379 use warnings 'utf8'; 380 open F, "<:utf8", $a_file; 381 undef $@; 382 local $SIG{__WARN__} = sub { $@ = shift }; 383 $line = <F>; 384 385 like( $@, qr/utf8 "\\xEF" does not map to Unicode .+ <F> chunk 1/, 386 "<:utf8 readline (fixed) must warn about bad utf8"); 387 close F; 388} 389 390# getc should reset the utf8 flag and not be affected by previous 391# return values 392SKIP: { 393 skip "no PerlIO::scalar on miniperl", 2, if is_miniperl(); 394 open my $fh, "<:raw", \($buf = chr 255); 395 open my $uh, "<:utf8", \($uuf = $U_100); 396 for([$uh,chr 256], [$fh,chr 255]) { 397 is getc $$_[0], $$_[1], 398 'getc returning non-utf8 after utf8'; 399 } 400} 401