1#!./perl 2 3BEGIN { 4 chdir 't' if -d 't'; 5 require './test.pl'; 6 set_up_inc('../lib'); 7 require './charset_tools.pl'; 8} 9skip_all_without_perlio(); 10 11no utf8; # needed for use utf8 not griping about the raw octets 12 13 14plan(tests => 62); 15 16$| = 1; 17 18my $a_file = tempfile(); 19 20open(F,"+>:utf8",$a_file); 21print F chr(0x100).'�'; 22cmp_ok( tell(F), '==', 4, tell(F) ); 23print F "\n"; 24cmp_ok( tell(F), '>=', 5, tell(F) ); 25seek(F,0,0); 26is( getc(F), chr(0x100) ); 27is( getc(F), "�" ); 28is( getc(F), "\n" ); 29seek(F,0,0); 30binmode(F,":bytes"); 31 32# Byte representation of these characters 33my $U_100 = byte_utf8a_to_utf8n("\xc4\x80"); 34my $POUND_SIGN = byte_utf8a_to_utf8n("\xc2\xa3"); 35 36my $chr = substr($U_100, 0, 1); 37is( getc(F), $chr ); 38$chr = substr($U_100, 1, 1); 39is( getc(F), $chr ); 40$chr = substr($POUND_SIGN, 0, 1); 41is( getc(F), $chr ); 42$chr = substr($POUND_SIGN, 1, 1); 43is( getc(F), $chr ); 44is( getc(F), "\n" ); 45seek(F,0,0); 46binmode(F,":utf8"); 47is( scalar(<F>), "\x{100}�\n" ); 48seek(F,0,0); 49$buf = chr(0x200); 50$count = read(F,$buf,2,1); 51cmp_ok( $count, '==', 2 ); 52is( $buf, "\x{200}\x{100}�" ); 53close(F); 54 55{ 56 $a = chr(300); # This *is* UTF-encoded 57 $b = chr(130); # This is not. 58 59 open F, ">:utf8", $a_file or die $!; 60 print F $a,"\n"; 61 close F; 62 63 open F, "<:utf8", $a_file or die $!; 64 $x = <F>; 65 chomp($x); 66 is( $x, chr(300) ); 67 68 open F, $a_file or die $!; # Not UTF 69 binmode(F, ":bytes"); 70 $x = <F>; 71 chomp($x); 72 $chr = byte_utf8a_to_utf8n(chr(196).chr(172)); 73 is( $x, $chr ); 74 close F; 75 76 open F, ">:utf8", $a_file or die $!; 77 binmode(F); # we write a "\n" and then tell() - avoid CRLF issues. 78 binmode(F,":utf8"); # turn UTF-8-ness back on 79 print F $a; 80 my $y; 81 { my $x = tell(F); 82 { use bytes; $y = length($a);} 83 cmp_ok( $x, '==', $y ); 84 } 85 86 { # Check byte length of $b 87 use bytes; my $y = length($b); 88 cmp_ok( $y, '==', 1 ); 89 } 90 91 print F $b,"\n"; # Don't upgrade $b 92 93 { # Check byte length of $b 94 use bytes; my $y = length($b); 95 cmp_ok( $y, '==', 1 ); 96 } 97 98 { 99 my $x = tell(F); 100 { use bytes; if ($::IS_EBCDIC){$y += 2;}else{$y += 3;}} # EBCDIC ASCII 101 cmp_ok( $x, '==', $y ); 102 } 103 104 close F; 105 106 open F, $a_file or die $!; # Not UTF 107 binmode(F, ":bytes"); 108 $x = <F>; 109 chomp($x); 110 $chr = v196.172.194.130; 111 if ($::IS_EBCDIC) { $chr = v141.83.130; } # EBCDIC 112 is( $x, $chr, sprintf('(%vd)', $x) ); 113 114 open F, "<:utf8", $a_file or die $!; 115 $x = <F>; 116 chomp($x); 117 close F; 118 is( $x, chr(300).chr(130), sprintf('(%vd)', $x) ); 119 120 open F, ">", $a_file or die $!; 121 binmode(F, ":bytes:"); 122 123 # Now let's make it suffer. 124 my $w; 125 { 126 use warnings 'utf8'; 127 local $SIG{__WARN__} = sub { $w = $_[0] }; 128 print F $a; 129 ok( (!$@)); 130 like($w, qr/Wide character in print/i ); 131 } 132} 133 134# Hm. Time to get more evil. 135open F, ">:utf8", $a_file or die $!; 136print F $a; 137binmode(F, ":bytes"); 138print F chr(130)."\n"; 139close F; 140 141open F, "<", $a_file or die $!; 142binmode(F, ":bytes"); 143$x = <F>; chomp $x; 144$chr = v196.172.130; 145if ($::IS_EBCDIC) { $chr = v141.83.130; } # EBCDIC 146is( $x, $chr ); 147 148# Right. 149open F, ">:utf8", $a_file or die $!; 150print F $a; 151close F; 152open F, ">>", $a_file or die $!; 153binmode(F, ":bytes"); 154print F chr(130)."\n"; 155close F; 156 157open F, "<", $a_file or die $!; 158binmode(F, ":bytes"); 159$x = <F>; chomp $x; 160SKIP: { 161 skip("Defaulting to UTF-8 output means that we can't generate a mangled file") 162 if $UTF8_OUTPUT; 163 is( $x, $chr ); 164} 165 166# Now we have a deformed file. 167 168SKIP: { 169 if ($::IS_EBCDIC) { 170 skip("EBCDIC The file isn't deformed in UTF-EBCDIC", 2); 171 } else { 172 my @warnings; 173 open F, "<:utf8", $a_file or die $!; 174 $x = <F>; chomp $x; 175 local $SIG{__WARN__} = sub { push @warnings, $_[0]; }; 176 eval { sprintf "%vd\n", $x }; 177 is (scalar @warnings, 1); 178 like ($warnings[0], qr/Malformed UTF-8 character: \\x82 \(unexpected continuation byte 0x82, with no preceding start byte/); 179 } 180} 181 182close F; 183unlink($a_file); 184 185open F, ">:utf8", $a_file; 186@a = map { chr(1 << ($_ << 2)) } 0..5; # 0x1, 0x10, .., 0x100000 187unshift @a, chr(0); # ... and a null byte in front just for fun 188print F @a; 189close F; 190 191my $c; 192 193# read() should work on characters, not bytes 194open F, "<:utf8", $a_file; 195$a = 0; 196my $failed; 197for (@a) { 198 unless (($c = read(F, $b, 1) == 1) && 199 length($b) == 1 && 200 ord($b) == ord($_) && 201 tell(F) == ($a += bytes::length($b))) { 202 print '# ord($_) == ', ord($_), "\n"; 203 print '# ord($b) == ', ord($b), "\n"; 204 print '# length($b) == ', length($b), "\n"; 205 print '# bytes::length($b) == ', bytes::length($b), "\n"; 206 print '# tell(F) == ', tell(F), "\n"; 207 print '# $a == ', $a, "\n"; 208 print '# $c == ', $c, "\n"; 209 $failed++; 210 last; 211 } 212} 213close F; 214is($failed, undef); 215 216{ 217 # Check that warnings are on on I/O, and that they can be muffled. 218 219 local $SIG{__WARN__} = sub { $@ = shift }; 220 221 undef $@; 222 open F, ">$a_file"; 223 binmode(F, ":bytes"); 224 print F chr(0x100); 225 close(F); 226 227 like( $@, qr/Wide character in print/ ); 228 229 undef $@; 230 open F, ">:utf8", $a_file; 231 print F chr(0x100); 232 close(F); 233 234 isnt( defined $@, !0 ); 235 236 undef $@; 237 open F, ">$a_file"; 238 binmode(F, ":utf8"); 239 print F chr(0x100); 240 close(F); 241 242 isnt( defined $@, !0 ); 243 244 no warnings 'utf8'; 245 246 undef $@; 247 open F, ">$a_file"; 248 print F chr(0x100); 249 close(F); 250 251 isnt( defined $@, !0 ); 252 253 use warnings 'utf8'; 254 255 undef $@; 256 open F, ">$a_file"; 257 binmode(F, ":bytes"); 258 print F chr(0x100); 259 close(F); 260 261 like( $@, qr/Wide character in print/ ); 262} 263 264{ 265 open F, ">:bytes",$a_file; print F "\xde"; close F; 266 267 open F, "<:bytes", $a_file; 268 my $b = chr 0x100; 269 $b .= <F>; 270 is( $b, chr(0x100).chr(0xde), "21395 '.= <>' utf8 vs. bytes" ); 271 close F; 272} 273 274{ 275 open F, ">:utf8",$a_file; print F chr 0x100; close F; 276 277 open F, "<:utf8", $a_file; 278 my $b = "\xde"; 279 $b .= <F>; 280 is( $b, chr(0xde).chr(0x100), "21395 '.= <>' bytes vs. utf8" ); 281 close F; 282} 283 284{ 285 my @a = ( [ 0x007F, "bytes" ], 286 [ 0x0080, "bytes" ], 287 [ 0x0080, "utf8" ], 288 [ 0x0100, "utf8" ] ); 289 my $t = 34; 290 for my $u (@a) { 291 for my $v (@a) { 292 # print "# @$u - @$v\n"; 293 open F, ">$a_file"; 294 binmode(F, ":" . $u->[1]); 295 print F chr($u->[0]); 296 close F; 297 298 open F, "<$a_file"; 299 binmode(F, ":" . $u->[1]); 300 301 my $s = chr($v->[0]); 302 utf8::upgrade($s) if $v->[1] eq "utf8"; 303 304 $s .= <F>; 305 is( $s, chr($v->[0]) . chr($u->[0]), 'rcatline utf8' ); 306 close F; 307 $t++; 308 } 309 } 310 # last test here 49 311} 312 313{ 314 # [perl #23428] Somethings rotten in unicode semantics 315 open F, ">$a_file"; 316 binmode F; 317 $a = "A"; 318 utf8::upgrade($a); 319 syswrite(F, $a); 320 close F; 321 ok(utf8::is_utf8($a), '23428 syswrite should not downgrade scalar' ); 322} 323 324{ 325 # <FH> on a :utf8 stream should complain immediately with -w 326 # if it finds bad UTF-8 (:encoding(utf8) works this way) 327 use warnings 'utf8'; 328 undef $@; 329 local $SIG{__WARN__} = sub { $@ = shift }; 330 open F, ">$a_file"; 331 binmode F; 332 my ($chrE4, $chrF6) = (chr(0xE4), chr(0xF6)); 333 if ($::IS_EBCDIC) # EBCDIC 334 { ($chrE4, $chrF6) = (chr(0x43), chr(0xEC)); } 335 print F "foo", $chrE4, "\n"; 336 print F "foo", $chrF6, "\n"; 337 close F; 338 open F, "<:utf8", $a_file; 339 undef $@; 340 my $line = <F>; 341 my ($chrE4, $chrF6) = ("E4", "F6"); 342 if ($::IS_EBCDIC) { ($chrE4, $chrF6) = ("43", "EC"); } # EBCDIC 343 like( $@, qr/utf8 "\\x$chrE4" does not map to Unicode .+ <F> line 1/, 344 "<:utf8 readline must warn about bad utf8"); 345 undef $@; 346 $line .= <F>; 347 like( $@, qr/utf8 "\\x$chrF6" does not map to Unicode .+ <F> line 2/, 348 "<:utf8 rcatline must warn about bad utf8"); 349 close F; 350} 351 352{ 353 # fixed record reads 354 open F, ">:utf8", $a_file; 355 print F "foo\xE4"; 356 print F "bar\xFE"; 357 print F "\xC0\xC8\xCC\xD2"; 358 print F "a\xE4ab"; 359 print F "a\xE4a"; 360 close F; 361 open F, "<:utf8", $a_file; 362 local $/ = \4; 363 my $line = <F>; 364 is($line, "foo\xE4", "readline with \$/ = \\4"); 365 $line .= <F>; 366 is($line, "foo\xE4bar\xFE", "rcatline with \$/ = \\4"); 367 $line = <F>; 368 is($line, "\xC0\xC8\xCC\xD2", "readline with several encoded characters"); 369 $line = <F>; 370 is($line, "a\xE4ab", "readline with another boundary condition"); 371 $line = <F>; 372 is($line, "a\xE4a", "readline with boundary condition"); 373 close F; 374 375 # badly encoded at EOF 376 open F, ">:raw", $a_file; 377 print F "foo\xEF\xAC"; # truncated \x{FB04} small ligature ffl 378 close F; 379 380 use warnings 'utf8'; 381 open F, "<:utf8", $a_file; 382 undef $@; 383 local $SIG{__WARN__} = sub { $@ = shift }; 384 $line = <F>; 385 386 like( $@, qr/utf8 "\\xEF" does not map to Unicode .+ <F> chunk 1/, 387 "<:utf8 readline (fixed) must warn about bad utf8"); 388 close F; 389} 390 391# getc should reset the utf8 flag and not be affected by previous 392# return values 393SKIP: { 394 skip "no PerlIO::scalar on miniperl", 2, if is_miniperl(); 395 open my $fh, "<:raw", \($buf = chr 255); 396 open my $uh, "<:utf8", \($uuf = $U_100); 397 for([$uh,chr 256], [$fh,chr 255]) { 398 is getc $$_[0], $$_[1], 399 'getc returning non-utf8 after utf8'; 400 } 401} 402