1%% 2%% %CopyrightBegin% 3%% 4%% Copyright Ericsson AB 2010-2017. All Rights Reserved. 5%% 6%% Licensed under the Apache License, Version 2.0 (the "License"); 7%% you may not use this file except in compliance with the License. 8%% You may obtain a copy of the License at 9%% 10%% http://www.apache.org/licenses/LICENSE-2.0 11%% 12%% Unless required by applicable law or agreed to in writing, software 13%% distributed under the License is distributed on an "AS IS" BASIS, 14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15%% See the License for the specific language governing permissions and 16%% limitations under the License. 17%% 18%% %CopyrightEnd% 19%% 20-module(leex_SUITE). 21 22%-define(debug, true). 23 24-include_lib("stdlib/include/erl_compile.hrl"). 25-include_lib("kernel/include/file.hrl"). 26 27-ifdef(debug). 28-define(line, put(line, ?LINE), ). 29-define(config(X,Y), foo). 30-define(datadir, "leex_SUITE_data"). 31-define(privdir, "leex_SUITE_priv"). 32-define(t, test_server). 33-else. 34-include_lib("common_test/include/ct.hrl"). 35-define(datadir, ?config(data_dir, Config)). 36-define(privdir, ?config(priv_dir, Config)). 37-endif. 38 39-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 40 init_per_group/2,end_per_group/2, 41 init_per_testcase/2, end_per_testcase/2]). 42 43-export([ 44 file/1, compile/1, syntax/1, 45 46 pt/1, man/1, ex/1, ex2/1, not_yet/1, 47 line_wrap/1, 48 otp_10302/1, otp_11286/1, unicode/1, otp_13916/1, otp_14285/1]). 49 50% Default timetrap timeout (set in init_per_testcase). 51-define(default_timeout, ?t:minutes(1)). 52 53init_per_testcase(_Case, Config) -> 54 ?line Dog = ?t:timetrap(?default_timeout), 55 [{watchdog, Dog} | Config]. 56 57end_per_testcase(_Case, Config) -> 58 Dog = ?config(watchdog, Config), 59 test_server:timetrap_cancel(Dog), 60 ok. 61 62suite() -> [{ct_hooks,[ts_install_cth]}]. 63 64all() -> 65 [{group, checks}, {group, examples}, {group, tickets}, {group, bugs}]. 66 67groups() -> 68 [{checks, [], [file, compile, syntax]}, 69 {examples, [], [pt, man, ex, ex2, not_yet, unicode]}, 70 {tickets, [], [otp_10302, otp_11286, otp_13916, otp_14285]}, 71 {bugs, [], [line_wrap]}]. 72 73init_per_suite(Config) -> 74 Config. 75 76end_per_suite(_Config) -> 77 ok. 78 79init_per_group(_GroupName, Config) -> 80 Config. 81 82end_per_group(_GroupName, Config) -> 83 Config. 84 85 86 87file(doc) -> 88 "Bad files and options."; 89file(suite) -> []; 90file(Config) when is_list(Config) -> 91 Dir = ?privdir, 92 Ret = [return, {report, false}], 93 ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = 94 leex:file("not_a_file", Ret), 95 ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = 96 leex:file("not_a_file", [{return,true}]), 97 ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = 98 leex:file("not_a_file", [{report,false},return_errors]), 99 ?line error = leex:file("not_a_file"), 100 ?line error = leex:file("not_a_file", [{return,false},report]), 101 ?line error = leex:file("not_a_file", [return_warnings,{report,false}]), 102 103 Filename = filename:join(Dir, "file.xrl"), 104 file:delete(Filename), 105 106 ?line {'EXIT', {badarg, _}} = (catch leex:file({foo})), 107 ?line {'EXIT', {badarg, _}} = 108 (catch leex:file(Filename, {parserfile,{foo}})), 109 ?line {'EXIT', {badarg, _}} = 110 (catch leex:file(Filename, {includefile,{foo}})), 111 112 ?line {'EXIT', {badarg, _}} = (catch leex:file(Filename, no_option)), 113 ?line {'EXIT', {badarg, _}} = 114 (catch leex:file(Filename, [return | report])), 115 ?line {'EXIT', {badarg, _}} = 116 (catch leex:file(Filename, {return,foo})), 117 ?line {'EXIT', {badarg, _}} = 118 (catch leex:file(Filename, includefile)), 119 120 Mini = <<"Definitions.\n" 121 "D = [0-9]\n" 122 "Rules.\n" 123 "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" 124 "Erlang code.\n">>, 125 ?line ok = file:write_file(Filename, Mini), 126 ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = 127 leex:file(Filename, [{scannerfile,"//"} | Ret]), 128 ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = 129 leex:file(Filename, [{includefile,"//"} | Ret]), 130 ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = 131 leex:file(Filename, [{includefile,"/ /"} | Ret]), 132 133 LeexPre = filename:join(Dir, "leexinc.hrl"), 134 ?line ok = file:write_file(LeexPre, <<"syntax error.\n">>), 135 PreErrors = run_test(Config, Mini, LeexPre), 136 ?line {errors, 137 [{1,_,["syntax error before: ","error"]}, 138 {3,_,undefined_module}], 139 []} = 140 extract(LeexPre, PreErrors), 141 file:delete(LeexPre), 142 143 Ret2 = [return, report_errors, report_warnings, verbose], 144 Scannerfile = filename:join(Dir, "file.erl"), 145 ?line ok = file:write_file(Scannerfile, <<"nothing">>), 146 ?line unwritable(Scannerfile), 147 ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = 148 leex:file(Filename, Ret2), 149 ?line writable(Scannerfile), 150 file:delete(Scannerfile), 151 152 Dotfile = filename:join(Dir, "file.dot"), 153 ?line ok = file:write_file(Dotfile, <<"nothing">>), 154 ?line unwritable(Dotfile), 155 ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = 156 leex:file(Filename, [dfa_graph | Ret2]), 157 ?line writable(Dotfile), 158 file:delete(Dotfile), 159 160 ok = file:delete(Scannerfile), 161 Warn = <<"Definitions.1998\n" 162 "D = [0-9]\n" 163 "Rules.\n" 164 "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" 165 "Erlang code.\n">>, 166 ok = file:write_file(Filename, Warn), 167 error = leex:file(Filename, [warnings_as_errors]), 168 false = filelib:is_regular(Scannerfile), 169 error = leex:file(Filename, [return_warnings,warnings_as_errors]), 170 false = filelib:is_regular(Scannerfile), 171 {error,_,[{Filename,[{1,leex,ignored_characters}]}]} = 172 leex:file(Filename, [return_errors,warnings_as_errors]), 173 false = filelib:is_regular(Scannerfile), 174 {ok,Scannerfile,[{Filename,[{1,leex,ignored_characters}]}]} = 175 leex:file(Filename, [return_warnings]), 176 true = filelib:is_regular(Scannerfile), 177 178 file:delete(Filename), 179 ok. 180 181compile(doc) -> 182 "Check of compile/3."; 183compile(suite) -> []; 184compile(Config) when is_list(Config) -> 185 Dir = ?privdir, 186 Filename = filename:join(Dir, "file.xrl"), 187 Scannerfile = filename:join(Dir, "file.erl"), 188 Mini = <<"Definitions.\n" 189 "D = [0-9]\n" 190 "Rules.\n" 191 "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" 192 "Erlang code.\n">>, 193 ?line ok = file:write_file(Filename, Mini), 194 ?line error = leex:compile(Filename, "//", #options{}), 195 ?line ok = leex:compile(Filename, Scannerfile, #options{}), 196 file:delete(Scannerfile), 197 file:delete(Filename), 198 ok. 199 200syntax(doc) -> 201 "Syntax checks."; 202syntax(suite) -> []; 203syntax(Config) when is_list(Config) -> 204 Dir = ?privdir, 205 Filename = filename:join(Dir, "file.xrl"), 206 Ret = [return, {report, true}], 207 ?line ok = file:write_file(Filename, 208 <<"Definitions.\n" 209 "D = [0-9]\n" 210 "%% comment\n" 211 "Rules.\n" 212 "{L}+ : {token,{word,TokenLine,TokenChars}}.\n 213 ">>), 214 ?line {error,[{_,[{7,leex,missing_code}]}],[]} = leex:file(Filename, Ret), 215 ?line ok = file:write_file(Filename, 216 <<"Definitions.\n" 217 "D = [0-9]\n" 218 "Rules.\n" 219 "{L}+ : \n">>), 220 ?line {error,[{_,[{5,leex,missing_code}]}],[]} = leex:file(Filename, Ret), 221 ?line ok = file:write_file(Filename, 222 <<"Definitions.\n" 223 "D = [0-9]\n" 224 "Rules.\n" 225 "[] :">>), 226 ?line {error,[{_,[{4,leex,{regexp,_}}]}],[]} = 227 leex:file(Filename, Ret), 228 ?line ok = file:write_file(Filename, 229 <<"Definitions.\n" 230 "D = [0-9]\n" 231 "Rules.\n" 232 "{L}+ : .\n" 233 "[] : ">>), 234 ?line {error,[{_,[{5,leex,{regexp,_}}]}],[]} = 235 leex:file(Filename, Ret), 236 ?line ok = file:write_file(Filename, 237 <<"Definitions.\n" 238 "D = [0-9]\n" 239 "Rules.\n" 240 "[] : .\n">>), 241 ?line {error,[{_,[{4,leex,{regexp,_}}]}],[]} = 242 leex:file(Filename, Ret), 243 ?line ok = file:write_file(Filename, 244 <<"Definitions.\n" 245 "D = [0-9]\n" 246 "Rules.\n" 247 "{L}+ ">>), 248 ?line {error,[{_,[{5,leex,bad_rule}]}],[]} = 249 leex:file(Filename, Ret), 250 ?line ok = file:write_file(Filename, 251 <<"Definitions.\n" 252 "D = [0-9]\n" 253 "Rules.\n" 254 "{L}+ ; ">>), 255 ?line {error,[{_,[{4,leex,bad_rule}]}],[]} = 256 leex:file(Filename, Ret), 257 ?line ok = file:write_file(Filename, 258 <<"Definitions.\n" 259 "D = [0-9]\n" 260 "Rules.\n" 261 "[] : '99\n">>), 262 ?line {error,[{_,[{4,erl_scan,_}]}],[]} = leex:file(Filename, Ret), 263 ?line ok = file:write_file(Filename, 264 <<"Definitions.\n" 265 "D = [0-9]\n" 266 "Rules.\n">>), 267 ?line {error,[{_,[{3,leex,empty_rules}]}],[]} = leex:file(Filename, Ret), 268 ?line ok = file:write_file(Filename, 269 <<"Definitions.\n" 270 "D = [0-9]\n" 271 "Rules.\n" 272 "Erlang code.\n">>), 273 ?line {error,[{_,[{4,leex,empty_rules}]}],[]} = leex:file(Filename, Ret), 274 ?line ok = file:write_file(Filename, 275 <<"Definitions.\n" 276 "D = [0-9]\n">>), 277 ?line {error,[{_,[{2,leex,missing_rules}]}],[]} = leex:file(Filename, Ret), 278 ?line ok = file:write_file(Filename, 279 <<"Definitions.\n" 280 "D = [0-9]\n" 281 "Erlang code.\n">>), 282 ?line {error,[{_,[{3,leex,missing_rules}]}],[]} = leex:file(Filename, Ret), 283 ?line ok = file:write_file(Filename, 284 <<"">>), 285 %% This is a weird line: 286 ?line {error,[{_,[{0,leex,missing_defs}]}],[]} = leex:file(Filename, Ret), 287 ?line ok = file:write_file(Filename, 288 <<"Rules.\n">>), 289 ?line {error,[{_,[{1,leex,missing_defs}]}],[]} = leex:file(Filename, Ret), 290 291 %% Check that correct line number is used in messages. 292 ErlFile = filename:join(Dir, "file.erl"), 293 Ret1 = [{scannerfile,ErlFile}|Ret], 294 ?line ok = file:write_file(Filename, 295 <<"Definitions.\n" 296 "D = [0-9]\n" 297 "Rules.\n" 298 "{L}+ : {token,\n" 299 " {word,TokenLine,TokenChars,\n" 300 " DDDD}}.\n" % unbound 301 "Erlang code.\n" 302 "an error.\n">>), % syntax error 303 ?line {ok, _, []} = leex:file(Filename, Ret1), 304 ?line {error, 305 [{_,[{8,_,["syntax error before: ","error"]}]}, 306 {_,[{6,_,{unbound_var,'DDDD'}}]}], 307 []} = 308 compile:file(ErlFile, [basic_validation, return]), 309 310 %% Ignored characters 311 ?line ok = file:write_file(Filename, 312 <<"Definitions. D = [0-9]\n" 313 "Rules. [a-z] : .\n" 314 "1 : skip_token.\n" 315 "Erlang code. f() -> a.\n">>), 316 ?line {ok,_,[{_, 317 [{1,leex,ignored_characters}, 318 {2,leex,ignored_characters}, 319 {4,leex,ignored_characters}]}]} = 320 leex:file(Filename, Ret), 321 322 ?line ok = file:write_file(Filename, 323 <<"Definitions.\n" 324 "D = [0-9]\n" 325 "Rules.\n" 326 "{L}+\\ : token.\n">>), 327 ?line {error,[{_,[{4,leex,{regexp,{unterminated,"\\"}}}]}],[]} = 328 leex:file(Filename, Ret), 329 ?line ok = file:write_file(Filename, 330 <<"Definitions.\n" 331 "D = [0-9]\n" 332 "Rules.\n" 333 "{L}+\\x : token.\n">>), 334 ?line {error,[{_,[{4,leex,{regexp,{illegal_char,"\\x"}}}]}],[]} = 335 leex:file(Filename, Ret), 336 ?line ok = file:write_file(Filename, 337 <<"Definitions.\n" 338 "D = [0-9]\n" 339 "Rules.\n" 340 "{L}+\\x{ : token.\n">>), 341 ?line {error,[{_,[{4,leex,{regexp,{unterminated,"\\x{"}}}]}],[]} = 342 leex:file(Filename, Ret), 343 ?line ok = file:write_file(Filename, 344 <<"Definitions.\n" 345 "D = [0-9]\n" 346 "Rules.\n" 347 "[^ab : token.\n">>), 348 ?line {error,[{_,[{4,leex,{regexp,{unterminated,"["}}}]}],[]} = 349 leex:file(Filename, Ret), 350 ?line ok = file:write_file(Filename, 351 <<"Definitions.\n" 352 "D = [0-9]\n" 353 "Rules.\n" 354 "(a : token.\n">>), 355 ?line {error,[{_,[{4,leex,{regexp,{unterminated,"("}}}]}],[]} = 356 leex:file(Filename, Ret), 357 ?line ok = file:write_file(Filename, 358 <<"Definitions.\n" 359 "D = [0-9]\n" 360 "Rules.\n" 361 "[b-a] : token.\n">>), 362 ?line {error,[{_,[{4,leex,{regexp,{char_class,"b-a"}}}]}],[]} = 363 leex:file(Filename, Ret), 364 365 ?line ok = file:write_file(Filename, 366 <<"Definitions.\n" 367 "D = [0-9]\n" 368 "Rules.\n" 369 "\\x{333333333333333333333333} : token.\n">>), 370 ?line {error,[{_,[{4,leex,{regexp, 371 {illegal_char, 372 "\\x{333333333333333333333333}"}}}]}],[]} = 373 leex:file(Filename, Ret), 374 ok. 375 376 377pt(doc) -> 378 "Pushing back characters."; 379pt(suite) -> []; 380pt(Config) when is_list(Config) -> 381 %% Needs more testing... 382 Ts = [{pt_1, 383 <<"Definitions.\n" 384 "D = [0-9]\n" 385 "L = [a-z]\n" 386 387 "Rules.\n" 388 "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" 389 "abc{D}+ : {skip_token,\"sture\" ++ string:substr(TokenChars, 4)}.\n" 390 "{D}+ : {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" 391 "\\s : .\n" 392 "\\r\\n : {end_token,{crlf,TokenLine}}.\n" 393 394 "Erlang code.\n" 395 "-export([t/0]).\n" 396 "t() -> 397 {ok,[{word,1,\"sture\"},{integer,1,123}],1} = 398 string(\"abc123\"), ok. ">>, 399 default, 400 ok}], 401 402 ?line run(Config, Ts), 403 ok. 404 405unicode(suite) -> 406 []; 407unicode(Config) when is_list(Config) -> 408 Ts = [{unicode_1, 409 <<"%% -*- coding: utf-8 -*-\n" 410 "Definitions.\n" 411 "RTLarrow = (â†)\n" 412 "Rules.\n" 413 "{RTLarrow} : {token,{\"â†\",TokenLine}}.\n" 414 "Erlang code.\n" 415 "-export([t/0]).\n" 416 "t() -> {ok, [{\"â†\", 1}], 1} = string(\"â†\"), ok.">>, 417 default, 418 ok}], 419 420 ?line run(Config, Ts), 421 ok. 422 423man(doc) -> 424 "Examples from the manpage."; 425man(suite) -> []; 426man(Config) when is_list(Config) -> 427 Ts = [{man_1, 428 <<"Definitions.\n" 429 "Rules.\n" 430 "[a-z][0-9a-zA-Z_]* :\n" 431 " {token,{atom,TokenLine,list_to_atom(TokenChars)}}.\n" 432 "[A-Z_][0-9a-zA-Z_]* :\n" 433 " {token,{var,TokenLine,list_to_atom(TokenChars)}}.\n" 434 "(\\+|-)?[0-9]+\\.[0-9]+((E|e)(\\+|-)?[0-9]+)? : \n" 435 " {token,{float,TokenLine,list_to_float(TokenChars)}}.\n" 436 "\\s : skip_token.\n" 437 "Erlang code.\n" 438 "-export([t/0]).\n" 439 "t() ->\n" 440 " {ok,[{float,1,3.14},{atom,1,atom},{var,1,'V314'}],1} =\n" 441 " string(\"3.14atom V314\"),\n" 442 " ok.\n">>, 443 default, 444 ok}, 445 446 {man_2, 447 <<"Definitions.\n" 448 "D = [0-9]\n" 449 "Rules.\n" 450 "{D}+ :\n" 451 " {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" 452 "{D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? :\n" 453 " {token,{float,TokenLine,list_to_float(TokenChars)}}.\n" 454 "\\s : skip_token.\n" 455 "Erlang code.\n" 456 "-export([t/0]).\n" 457 "t() ->\n" 458 " {ok,[{float,1,3.14},{integer,1,314}],1} = \n" 459 " string(\"3.14 314\"),\n" 460 " ok.\n">>, 461 default, 462 ok}], 463 464 ?line run(Config, Ts), 465 ok. 466 467ex(doc) -> 468 "Examples."; 469ex(suite) -> []; 470ex(Config) when is_list(Config) -> 471 Ts = [{ex_1, 472 <<"Definitions.\n" 473 "D = [0-543-705-982]\n" 474 "Rules.\n" 475 "{D}+ :\n" 476 " {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" 477 "[^235]+ :\n" 478 " {token,{list_to_atom(TokenChars),TokenLine}}.\n" 479 "Erlang code.\n" 480 "-export([t/0]).\n" 481 "t() ->\n" 482 " {ok,[{integer,1,12},{' c\\na',1},{integer,2,34},{b789a,2}],2} =\n" 483 " string(\"12 c\\na34b789a\"),\n" 484 " ok.\n">>, 485 default, 486 ok}, 487 488 {ex_2, 489 <<"Definitions.\n" 490 "L = [a-z]\n" 491 "D = [0-9]\n" 492 "Rules.\n" 493 "{L}+ : {token,chars}.\n" 494 "zyx{D}+ : {token,zyx}.\n" 495 "\\s : skip_token.\n" 496 "Erlang code.\n" 497 "-export([t/0]).\n" 498 "t() ->\n" 499 " {ok,[chars,zyx],1} = string(\"abcdef zyx123\"),\n" 500 " ok.\n">>, 501 default, 502 ok}, 503 504 {ex_3, 505 <<"Definitions.\n" 506 "NL = [\\n]\n" 507 "Rules.\n" 508 "{NL}* : {token,newlines}.\n" 509 "Erlang code.\n" 510 "-export([t/0]).\n" 511 "t() ->\n" 512 " {ok,[],1} = string(\"\"), ok.\n">>, % string("a") would loop... 513 default, 514 ok}, 515 516 {ex_4, 517 <<"Definitions.\n" 518 "SP1 = [\\n-\\s]\n" 519 "SP0 = [\\000-\\n]\n" 520 "Rules.\n" 521 "{SP0}+ : {token,{small,TokenChars}}.\n" 522 "{SP1}+ : {token,{big,TokenChars}}.\n" 523 "Erlang code.\n" 524 "-export([t/0]).\n" 525 "t() ->\n" 526 " string(\"\\x00\\n\\s\\n\\n\"),\n" 527 " ok.\n">>, 528 default, 529 ok}, 530 531 {ex_5, 532 <<"Definitions.\n" 533 "L = [a-z]\n" 534 "W = [\\s\\b\\n\\r\\t\\e\\v\\d\\f]\n" 535 "Rules.\n" 536 "\\[{L}+(,{L}+)*\\] : {token,{list,TokenChars}}.\n" 537 "\"{L}+\" : {token,{string,TokenChars}}.\n" 538 "\\$. : {token,{char,TokenChars}}.\n" 539 "{W}+ : {token,{white,TokenChars}}.\n" 540 "ff\\f+ : {token,{form,TokenChars}}.\n" 541 "\\$\\^+\\\\+ : {token,{other,TokenChars}}.\n" 542 "Erlang code.\n" 543 "-export([t/0]).\n" 544 "t() ->\n" 545 " {ok,[{white,\"\\b\\f\"}],1} = string(\"\\b\\f\"),\n" 546 " {ok,[{form,\"ff\\f\"}],1} = string(\"ff\\f\"),\n" 547 " {ok,[{string,\"\\\"foo\\\"\"}],1} = string(\"\\\"foo\\\"\"),\n" 548 " {ok,[{char,\"$.\"}],1} = string(\"$\\.\"),\n" 549 " {ok,[{list,\"[a,b,c]\"}],1} = string(\"[a,b,c]\"),\n" 550 " {ok,[{other,\"$^\\\\\"}],1} = string(\"$^\\\\\"),\n" 551 " ok.\n">>, 552 default, 553 ok}, 554 555 {ex_6, 556 <<"Definitions.\n" 557 "L = [a-z]\n" 558 "Rules.\n" 559 "L}+ : {token,{TokenChars,#r.f}}.\n" 560 "Erlang code.\n" 561 "-record(r, {f}).\n" 562 "-export([t/0]).\n" 563 "t() ->\n" 564 " string(\"abc\"),\n" 565 " ok.\n">>, 566 default, 567 ok}, 568 569 {ex_7, %% Assumes regexp can handle \x 570 <<"Definitions.\n" 571 "H1 = \\x11\\x{ab}\n" 572 "H2 = [\\x{30}\\x{ac}]\n" 573 "Rules.\n" 574 "{H1}{H2}+ : {token,{hex,TokenChars}}.\n" 575 "Erlang code.\n" 576 "-export([t/0]).\n" 577 "t() ->\n" 578 " {ok,[{hex,[17,171,48,172]}],1} =\n" 579 " string(\"\\x{11}\\xab0\\xac\"),\n" 580 " ok.\n">>, 581 default, 582 ok}], 583 584 ?line run(Config, Ts), 585 ok. 586 587ex2(doc) -> 588 "More examples."; 589ex2(suite) -> []; 590ex2(Config) when is_list(Config) -> 591 Xrl = 592 <<" 593%%% File : erlang_scan.xrl 594%%% Author : Robert Virding 595%%% Purpose : Token definitions for Erlang. 596 597Definitions. 598O = [0-7] 599D = [0-9] 600H = [0-9a-fA-F] 601U = [A-Z] 602L = [a-z] 603A = ({U}|{L}|{D}|_|@) 604WS = ([\\000-\\s]|%.*) 605 606Rules. 607{D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? : 608 {token,{float,TokenLine,list_to_float(TokenChars)}}. 609{D}+#{H}+ : base(TokenLine, TokenChars). 610{D}+ : {token,{integer,TokenLine,list_to_integer(TokenChars)}}. 611{L}{A}* : Atom = list_to_atom(TokenChars), 612 {token,case reserved_word(Atom) of 613 true -> {Atom,TokenLine}; 614 false -> {atom,TokenLine,Atom} 615 end}. 616'(\\\\\\^.|\\\\.|[^'])*' : 617 %% Strip quotes. 618 S = lists:sublist(TokenChars, 2, TokenLen - 2), 619 case catch list_to_atom(string_gen(S)) of 620 {'EXIT',_} -> {error,\"illegal atom \" ++ TokenChars}; 621 Atom -> {token,{atom,TokenLine,Atom}} 622 end. 623({U}|_){A}* : {token,{var,TokenLine,list_to_atom(TokenChars)}}. 624\"(\\\\\\^.|\\\\.|[^\"])*\" : 625 %% Strip quotes. 626 S = lists:sublist(TokenChars, 2, TokenLen - 2), 627 {token,{string,TokenLine,string_gen(S)}}. 628\\$(\\\\{O}{O}{O}|\\\\\\^.|\\\\.|.) : 629 {token,{char,TokenLine,cc_convert(TokenChars)}}. 630-> : {token,{'->',TokenLine}}. 631:- : {token,{':-',TokenLine}}. 632\\|\\| : {token,{'||',TokenLine}}. 633<- : {token,{'<-',TokenLine}}. 634\\+\\+ : {token,{'++',TokenLine}}. 635-- : {token,{'--',TokenLine}}. 636=/= : {token,{'=/=',TokenLine}}. 637== : {token,{'==',TokenLine}}. 638=:= : {token,{'=:=',TokenLine}}. 639/= : {token,{'/=',TokenLine}}. 640>= : {token,{'>=',TokenLine}}. 641=< : {token,{'=<',TokenLine}}. 642<= : {token,{'<=',TokenLine}}. 643<< : {token,{'<<',TokenLine}}. 644>> : {token,{'>>',TokenLine}}. 645:: : {token,{'::',TokenLine}}. 646[]()[}{|!?/;:,.*+#<>=-] : 647 {token,{list_to_atom(TokenChars),TokenLine}}. 648\\.{WS} : {end_token,{dot,TokenLine}}. 649{WS}+ : skip_token. 650 651Erlang code. 652 653-export([reserved_word/1]). 654 655%% reserved_word(Atom) -> Bool 656%% return 'true' if Atom is an Erlang reserved word, else 'false'. 657 658reserved_word('after') -> true; 659reserved_word('begin') -> true; 660reserved_word('case') -> true; 661reserved_word('try') -> true; 662reserved_word('cond') -> true; 663reserved_word('catch') -> true; 664reserved_word('andalso') -> true; 665reserved_word('orelse') -> true; 666reserved_word('end') -> true; 667reserved_word('fun') -> true; 668reserved_word('if') -> true; 669reserved_word('let') -> true; 670reserved_word('of') -> true; 671reserved_word('receive') -> true; 672reserved_word('when') -> true; 673reserved_word('bnot') -> true; 674reserved_word('not') -> true; 675reserved_word('div') -> true; 676reserved_word('rem') -> true; 677reserved_word('band') -> true; 678reserved_word('and') -> true; 679reserved_word('bor') -> true; 680reserved_word('bxor') -> true; 681reserved_word('bsl') -> true; 682reserved_word('bsr') -> true; 683reserved_word('or') -> true; 684reserved_word('xor') -> true; 685reserved_word('spec') -> true; 686reserved_word(_) -> false. 687 688base(L, Cs) -> 689 H = string:chr(Cs, $#), 690 case list_to_integer(string:substr(Cs, 1, H-1)) of 691 B when B > 16 -> {error,\"illegal base\"}; 692 B -> 693 case base(string:substr(Cs, H+1), B, 0) of 694 error -> {error,\"illegal based number\"}; 695 N -> {token,{integer,L,N}} 696 end 697 end. 698 699base([C|Cs], Base, SoFar) when C >= $0, C =< $9, C < Base + $0 -> 700 Next = SoFar * Base + (C - $0), 701 base(Cs, Base, Next); 702base([C|Cs], Base, SoFar) when C >= $a, C =< $f, C < Base + $a - 10 -> 703 Next = SoFar * Base + (C - $a + 10), 704 base(Cs, Base, Next); 705base([C|Cs], Base, SoFar) when C >= $A, C =< $F, C < Base + $A - 10 -> 706 Next = SoFar * Base + (C - $A + 10), 707 base(Cs, Base, Next); 708base([_|_], _, _) -> error; %Unknown character 709base([], _, N) -> N. 710 711cc_convert([$$,$\\\\|Cs]) -> 712 hd(string_escape(Cs)); 713cc_convert([$$,C]) -> C. 714 715string_gen([$\\\\|Cs]) -> 716 string_escape(Cs); 717string_gen([C|Cs]) -> 718 [C|string_gen(Cs)]; 719string_gen([]) -> []. 720 721string_escape([O1,O2,O3|S]) when 722 O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> 723 [(O1*8 + O2)*8 + O3 - 73*$0|string_gen(S)]; 724string_escape([$^,C|Cs]) -> 725 [C band 31|string_gen(Cs)]; 726string_escape([C|Cs]) when C >= $\\000, C =< $\\s -> 727 string_gen(Cs); 728string_escape([C|Cs]) -> 729 [escape_char(C)|string_gen(Cs)]. 730 731escape_char($n) -> $\\n; %\\n = LF 732escape_char($r) -> $\\r; %\\r = CR 733escape_char($t) -> $\\t; %\\t = TAB 734escape_char($v) -> $\\v; %\\v = VT 735escape_char($b) -> $\\b; %\\b = BS 736escape_char($f) -> $\\f; %\\f = FF 737escape_char($e) -> $\\e; %\\e = ESC 738escape_char($s) -> $\\s; %\\s = SPC 739escape_char($d) -> $\\d; %\\d = DEL 740escape_char(C) -> C. 741 ">>, 742 Dir = ?privdir, 743 XrlFile = filename:join(Dir, "erlang_scan.xrl"), 744 ?line ok = file:write_file(XrlFile, Xrl), 745 ErlFile = filename:join(Dir, "erlang_scan.erl"), 746 ?line {ok, _} = leex:file(XrlFile, []), 747 ?line {ok, _} = compile:file(ErlFile, [{outdir,Dir}]), 748 code:purge(erlang_scan), 749 AbsFile = filename:rootname(ErlFile, ".erl"), 750 code:load_abs(AbsFile, erlang_scan), 751 752 F = fun(Cont, Chars, Location) -> 753 erlang_scan:tokens(Cont, Chars, Location) 754 end, 755 F1 = fun(Cont, Chars, Location) -> 756 erlang_scan:token(Cont, Chars, Location) 757 end, 758 fun() -> 759 S = "ab cd. ", 760 {ok, Ts, 1} = scan_tokens_1(S, F, 1), 761 {ok, Ts, 1} = scan_token_1(S, F1, 1), 762 {ok, Ts, 1} = scan_tokens(S, F, 1), 763 {ok, Ts, 1} = erlang_scan:string(S, 1) 764 end(), 765 fun() -> 766 S = "'ab\n cd'. ", 767 {ok, Ts, 2} = scan_tokens_1(S, F, 1), 768 {ok, Ts, 2} = scan_token_1(S, F1, 1), 769 {ok, Ts, 2} = scan_tokens(S, F, 1), 770 {ok, Ts, 2} = erlang_scan:string(S, 1) 771 end(), 772 fun() -> 773 S = "99. ", 774 {ok, Ts, 1} = scan_tokens_1(S, F, 1), 775 {ok, Ts, 1} = scan_token_1(S, F1, 1), 776 {ok, Ts, 1} = scan_tokens(S, F, 1), 777 {ok, Ts, 1} = erlang_scan:string(S, 1) 778 end(), 779 {ok,[{integer,1,99},{dot,1}],1} = erlang_scan:string("99. "), 780 fun() -> 781 Atom = "'" ++ lists:duplicate(1000,$a) ++ "'", 782 S = Atom ++ ". ", 783 Reason = "illegal atom " ++ Atom, 784 Err = {error,{1,erlang_scan,{user,Reason}},1}, 785 {done,Err,[]} = scan_tokens_1(S, F, 1), 786 {done,Err,[]} = scan_token_1(S, F1, 1), 787 {done,Err,[]} = scan_tokens(S, F, 1), 788 Err = erlang_scan:string(S, 1) 789 end(), 790 fun() -> 791 S = "\x{aaa}. ", 792 Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, 793 {done,Err,[]} = scan_tokens_1(S, F, 1), 794 {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty 795 {done,Err,[]} = scan_tokens(S, F, 1), 796 Err = erlang_scan:string(S, 1) 797 end(), 798 fun() -> 799 S = "\x{aaa} + 1. 34", 800 Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, 801 {done,Err,[]} = scan_tokens_1(S, F, 1), 802 {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty 803 {done,Err,"34"} = scan_tokens(S, F, 1), 804 Err = erlang_scan:string(S, 1) 805 end(), 806 fun() -> 807 S = "\x{aaa} \x{bbb}. 34", 808 Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, 809 {done,Err,[]} = scan_tokens_1(S, F, 1), 810 {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty 811 {done,Err,"34"} = scan_tokens(S, F, 1), 812 Err = erlang_scan:string(S, 1) 813 end(), 814 fun() -> 815 S = "\x{aaa} 18#34. 34", 816 Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, 817 {done,Err,[]} = scan_tokens_1(S, F, 1), 818 {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty 819 {done,Err,"34"} = scan_tokens(S, F, 1), 820 Err = erlang_scan:string(S, 1) 821 end(), 822 fun() -> 823 S = "\x{aaa}"++eof, 824 Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, 825 {done,Err,eof} = scan_tokens_1(S, F, 1), 826 {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty 827 {done,Err,eof} = scan_tokens(S, F, 1), 828 Err = erlang_scan:string(S, 1) 829 end(), 830 ok. 831 832scan_tokens(String, Fun, Location) -> 833 scan_tokens(String, Fun, Location, []). 834 835scan_tokens(String, Fun, Location, Rs) -> 836 case Fun([], String, Location) of 837 {done, {error,_,_}, _} = Error -> 838 Error; 839 {done, {ok,Ts,End}, ""} -> 840 {ok, lists:append(lists:reverse([Ts|Rs])), End}; 841 {done, {ok,Ts,End}, Rest} -> 842 scan_tokens(Rest, Fun, End, [Ts|Rs]) 843 end. 844 845scan_tokens_1(String, Fun, Location) -> 846 scan_tokens_1({more, []}, String, Fun, Location, []). 847 848scan_tokens_1({done, {error, _, _}, _}=Error, _Cs, _Fun, _Location, _Rs) -> 849 Error; 850scan_tokens_1({done, {ok,Ts,End}, ""}, "", _Fun, _Location, Rs) -> 851 {ok,lists:append(lists:reverse([Ts|Rs])),End}; 852scan_tokens_1({done, {ok,Ts,End}, Rest}, Cs, Fun, _Location, Rs) -> 853 scan_tokens_1({more,[]}, Rest++Cs, Fun, End, [Ts|Rs]); 854scan_tokens_1({more, Cont}, [C | Cs], Fun, Loc, Rs) -> 855 R = Fun(Cont, [C], Loc), 856 scan_tokens_1(R, Cs, Fun, Loc, Rs); 857scan_tokens_1({more, Cont}, eof, Fun, Loc, Rs) -> 858 R = Fun(Cont, eof, Loc), 859 scan_tokens_1(R, eof, Fun, Loc, Rs). 860 861scan_token_1(String, Fun, Location) -> 862 scan_token_1({more, []}, String, Fun, Location, []). 863 864scan_token_1({done, {error, _, _}, _}=Error, _Cs, _Fun, _Location, _Rs) -> 865 Error; 866scan_token_1({done, {ok,Ts,End}, ""}, "", _Fun, _Location, Rs) -> 867 {ok,lists:reverse([Ts|Rs]),End}; 868scan_token_1({done, {ok,Ts,End}, Rest}, Cs, Fun, _Location, Rs) -> 869 scan_token_1({more,[]}, Rest++Cs, Fun, End, [Ts|Rs]); 870scan_token_1({more, Cont}, [C | Cs], Fun, Loc, Rs) -> 871 R = Fun(Cont, [C], Loc), 872 scan_token_1(R, Cs, Fun, Loc, Rs). 873 874%% End of ex2 875 876line_wrap(doc) -> "Much more examples."; 877line_wrap(suite) -> []; 878line_wrap(Config) when is_list(Config) -> 879 Xrl = 880 <<" 881Definitions. 882Rules. 883[a]+[\\n]*= : {token, {first, TokenLine}}. 884[a]+ : {token, {second, TokenLine}}. 885[\\s\\r\\n\\t]+ : skip_token. 886Erlang code. 887 ">>, 888 Dir = ?privdir, 889 XrlFile = filename:join(Dir, "test_line_wrap.xrl"), 890 ?line ok = file:write_file(XrlFile, Xrl), 891 ErlFile = filename:join(Dir, "test_line_wrap.erl"), 892 {ok, _} = leex:file(XrlFile, []), 893 {ok, _} = compile:file(ErlFile, [{outdir,Dir}]), 894 code:purge(test_line_wrap), 895 AbsFile = filename:rootname(ErlFile, ".erl"), 896 code:load_abs(AbsFile, test_line_wrap), 897 fun() -> 898 S = "aaa\naaa", 899 {ok,[{second,1},{second,2}],2} = test_line_wrap:string(S) 900 end(), 901 fun() -> 902 S = "aaa\naaa", 903 {ok,[{second,3},{second,4}],4} = test_line_wrap:string(S, 3) 904 end(), 905 fun() -> 906 {done,{ok,{second,1},1},"\na"} = test_line_wrap:token([], "a\na"), 907 {more,Cont1} = test_line_wrap:token([], "\na"), 908 {done,{ok,{second,2},2},eof} = test_line_wrap:token(Cont1, eof) 909 end(), 910 fun() -> 911 {more,Cont1} = test_line_wrap:tokens([], "a\na"), 912 {done,{ok,[{second,1},{second,2}],2},eof} = test_line_wrap:tokens(Cont1, eof) 913 end(), 914 ok. 915 916%% End of line_wrap 917 918not_yet(doc) -> 919 "Not yet implemented."; 920not_yet(suite) -> []; 921not_yet(Config) when is_list(Config) -> 922 Dir = ?privdir, 923 Filename = filename:join(Dir, "file.xrl"), 924 Ret = [return, {report, true}], 925 ?line ok = file:write_file(Filename, 926 <<"Definitions.\n" 927 "Rules.\n" 928 "$ : .\n" 929 "Erlang code.\n">>), 930 ?line {error,[{_,[{3,leex,{regexp,_}}]}],[]} = 931 leex:file(Filename, Ret), 932 ?line ok = file:write_file(Filename, 933 <<"Definitions.\n" 934 "Rules.\n" 935 "^ : .\n" 936 "Erlang code.\n">>), 937 ?line {error,[{_,[{3,leex,{regexp,_}}]}],[]} = 938 leex:file(Filename, Ret), 939 940 ok. 941 942otp_10302(doc) -> 943 "OTP-10302. Unicode characters scanner/parser."; 944otp_10302(suite) -> []; 945otp_10302(Config) when is_list(Config) -> 946 Dir = ?privdir, 947 Filename = filename:join(Dir, "file.xrl"), 948 Ret = [return, {report, true}], 949 950 ok = file:write_file(Filename,<< 951 "%% coding: UTF-8\n" 952 "ä" 953 >>), 954 {error,[{_,[{2,leex,cannot_parse}]}],[]} = 955 leex:file(Filename, Ret), 956 957 ok = file:write_file(Filename,<< 958 "%% coding: UTF-8\n" 959 "Definitions.\n" 960 "ä" 961 >>), 962 {error,[{_,[{3,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret), 963 964 ok = file:write_file(Filename,<< 965 "%% coding: UTF-8\n" 966 "Definitions.\n" 967 "A = a\n" 968 "L = [{A}-{Z}]\n" 969 "Z = z\n" 970 "Rules.\n" 971 "{L}+ : {token,{list_to_atom(TokenChars),Häpp}}.\n" 972 >>), 973 {error,[{_,[{7,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret), 974 975 ok = file:write_file(Filename,<< 976 "%% coding: UTF-8\n" 977 "Definitions.\n" 978 "A = a\n" 979 "L = [{A}-{Z}]\n" 980 "Z = z\n" 981 "Rules.\n" 982 "{L}+ : {token,{list_to_atom(TokenChars)}}.\n" 983 "Erlang code.\n" 984 "-export([t/0]).\n" 985 "t() ->\n" 986 " Häpp\n" 987 >>), 988 {error,[{_,[{11,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret), 989 990 Mini = <<"Definitions.\n" 991 "D = [0-9]\n" 992 "Rules.\n" 993 "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" 994 "Erlang code.\n">>, 995 LeexPre = filename:join(Dir, "leexinc.hrl"), 996 ?line ok = file:write_file(LeexPre, <<"%% coding: UTF-8\n ä">>), 997 PreErrors = run_test(Config, Mini, LeexPre), 998 {error,[{IncludeFile,[{2,leex,cannot_parse}]}],[]} = PreErrors, 999 "leexinc.hrl" = filename:basename(IncludeFile), 1000 1001 Ts = [{uni_1, 1002 <<"%% coding: UTF-8\n" 1003 "Definitions.\n" 1004 "A = a\n" 1005 "L = [{A}-{Z}]\n" 1006 "Z = z\n" 1007 "Rules.\n" 1008 "{L}+ : {token,{list_to_atom(TokenChars),\n" 1009 "begin Häpp = foo, Häpp end," 1010 " 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n" 1011 "Erlang code.\n" 1012 "-export([t/0]).\n" 1013 "t() ->\n" 1014 " %% Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"\n" 1015 " {ok, [R], 1} = string(\"tip\"),\n" 1016 " {tip,foo,'Häpp',[1024,66],[246,114,110,95,1024]} = R,\n" 1017 " Häpp = foo,\n" 1018 " {tip, Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n" 1019 " ok.\n">>, 1020 default, 1021 ok}, 1022 {uni_2, 1023 <<"%% coding: Latin-1\n" 1024 "Definitions.\n" 1025 "A = a\n" 1026 "L = [{A}-{Z}]\n" 1027 "Z = z\n" 1028 "Rules.\n" 1029 "{L}+ : {token,{list_to_atom(TokenChars),\n" 1030 "begin Häpp = foo, Häpp end," 1031 " 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n" 1032 "Erlang code.\n" 1033 "-export([t/0]).\n" 1034 "t() ->\n" 1035 " %% Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"\n" 1036 " {ok, [R], 1} = string(\"tip\"),\n" 1037 " {tip,foo,'Häpp',[1024,66],[195,182,114,110,95,208,128]} = R,\n" 1038 " Häpp = foo,\n" 1039 " {tip, Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n" 1040 " ok.\n">>, 1041 default, 1042 ok}], 1043 run(Config, Ts), 1044 1045 ok. 1046 1047otp_11286(doc) -> 1048 "OTP-11286. A Unicode filename bug; both Leex and Yecc."; 1049otp_11286(suite) -> []; 1050otp_11286(Config) when is_list(Config) -> 1051 Node = start_node(otp_11286, "+fnu"), 1052 Dir = ?privdir, 1053 UName = [1024] ++ "u", 1054 UDir = filename:join(Dir, UName), 1055 _ = rpc:call(Node, file, make_dir, [UDir]), 1056 1057 %% Note: Cannot use UName as filename since the filename is used 1058 %% as module name. To be fixed in R18. 1059 Filename = filename:join(UDir, 'OTP-11286.xrl'), 1060 Scannerfile = filename:join(UDir, 'OTP-11286.erl'), 1061 Options = [return, {scannerfile, Scannerfile}], 1062 1063 Mini1 = <<"%% coding: utf-8\n" 1064 "Definitions.\n" 1065 "D = [0-9]\n" 1066 "Rules.\n" 1067 "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" 1068 "Erlang code.\n">>, 1069 ok = rpc:call(Node, file, write_file, [Filename, Mini1]), 1070 {ok, _, []} = rpc:call(Node, leex, file, [Filename, Options]), 1071 {ok,_,_} = rpc:call(Node, compile, file, 1072 [Scannerfile,[basic_validation,return]]), 1073 1074 Mini2 = <<"Definitions.\n" 1075 "D = [0-9]\n" 1076 "Rules.\n" 1077 "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" 1078 "Erlang code.\n">>, 1079 ok = rpc:call(Node, file, write_file, [Filename, Mini2]), 1080 {ok, _, []} = rpc:call(Node, leex, file, [Filename, Options]), 1081 {ok,_,_} = rpc:call(Node, compile, file, 1082 [Scannerfile,[basic_validation,return]]), 1083 1084 Mini3 = <<"%% coding: latin-1\n" 1085 "Definitions.\n" 1086 "D = [0-9]\n" 1087 "Rules.\n" 1088 "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" 1089 "Erlang code.\n">>, 1090 ok = rpc:call(Node, file, write_file, [Filename, Mini3]), 1091 {ok, _, []} = rpc:call(Node, leex, file, [Filename, Options]), 1092 {ok,_,_} = rpc:call(Node, compile, file, 1093 [Scannerfile,[basic_validation,return]]), 1094 1095 true = test_server:stop_node(Node), 1096 ok. 1097 1098otp_13916(doc) -> 1099 "OTP-13916. Leex rules with newlines result in bad line numbers"; 1100otp_13916(suite) -> []; 1101otp_13916(Config) when is_list(Config) -> 1102 Ts = [{otp_13916_1, 1103 <<"Definitions.\n" 1104 "W = [a-zA-Z0-9]\n" 1105 "S = [\\s\\t]\n" 1106 "B = [\\n\\r]\n" 1107 "Rules.\n" 1108 "%% mark line break(s) and empty lines by token 'break'\n" 1109 "%% in order to use as delimiters\n" 1110 "{B}({S}*{B})+ : {token, {break, TokenLine}}.\n" 1111 "{B} : {token, {break, TokenLine}}.\n" 1112 "{S}+ : {token, {blank, TokenLine, TokenChars}}.\n" 1113 "{W}+ : {token, {word, TokenLine, TokenChars}}.\n" 1114 "Erlang code.\n" 1115 "-export([t/0]).\n" 1116 "t() ->\n" 1117 " {ok,[{break,1},{blank,4,\" \"},{word,4,\"breaks\"}],4} =\n" 1118 " string(\"\\n\\n \\n breaks\"),\n" 1119 " {ok,[{break,1},{word,4,\"works\"}],4} =\n" 1120 " string(\"\\n\\n \\nworks\"),\n" 1121 " {ok,[{break,1},{word,4,\"L4\"},{break,4},\n" 1122 " {word,5,\"L5\"},{break,5},{word,7,\"L7\"}], 7} =\n" 1123 " string(\"\\n\\n \\nL4\\nL5\\n\\nL7\"),\n" 1124 " {ok,[{break,1},{blank,4,\" \"},{word,4,\"L4\"},\n" 1125 " {break,4},{blank,5,\" \"},{word,5,\"L5\"},\n" 1126 " {break,5},{blank,7,\" \"},{word,7,\"L7\"}], 7} =\n" 1127 " string(\"\\n\\n \\n L4\\n L5\\n\\n L7\"),\n" 1128 " ok.\n">>, 1129 default, 1130 ok}], 1131 ?line run(Config, Ts), 1132 ok. 1133 1134otp_14285(Config) -> 1135 Dir = ?privdir, 1136 Filename = filename:join(Dir, "file.xrl"), 1137 1138 Ts = [{otp_14285_1, 1139 <<"%% encoding: latin-1\n" 1140 "Definitions.\n" 1141 "A = a\n" 1142 "Z = z\n" 1143 "L = [{A}-{Z}]\n" 1144 "U = [\\x{400}]\n" 1145 "Rules.\n" 1146 "{L}+ : {token,l}.\n" 1147 "{U}+ : {token,'\\x{400}'}.\n" 1148 "Erlang code.\n" 1149 "-export([t/0]).\n" 1150 "t() ->\n" 1151 " {ok,['\\x{400}'],1} = string(\"\\x{400}\"), ok.\n">>, 1152 default, 1153 ok}, 1154 {otp_14285_2, 1155 <<"%% encoding: UTF-8\n" 1156 "Definitions.\n" 1157 "A = a\n" 1158 "Z = z\n" 1159 "L = [{A}-{Z}]\n" 1160 "U = [\x{400}]\n" 1161 "Rules.\n" 1162 "{L}+ : {token,l}.\n" 1163 "{U}+ : {token,'\x{400}'}.\n" 1164 "Erlang code.\n" 1165 "-export([t/0]).\n" 1166 "t() ->\n" 1167 " {ok,['\x{400}'],1} = string(\"\x{400}\"), ok.\n">>, 1168 default, 1169 ok}], 1170 run(Config, Ts), 1171 ok. 1172 1173start_node(Name, Args) -> 1174 [_,Host] = string:tokens(atom_to_list(node()), "@"), 1175 ct:log("Trying to start ~w@~s~n", [Name,Host]), 1176 case test_server:start_node(Name, peer, [{args,Args}]) of 1177 {error,Reason} -> 1178 test_server:fail(Reason); 1179 {ok,Node} -> 1180 ct:log("Node ~p started~n", [Node]), 1181 Node 1182 end. 1183 1184unwritable(Fname) -> 1185 {ok, Info} = file:read_file_info(Fname), 1186 Mode = Info#file_info.mode - 8#00200, 1187 ok = file:write_file_info(Fname, Info#file_info{mode = Mode}). 1188 1189writable(Fname) -> 1190 {ok, Info} = file:read_file_info(Fname), 1191 Mode = Info#file_info.mode bor 8#00200, 1192 ok = file:write_file_info(Fname, Info#file_info{mode = Mode}). 1193 1194run(Config, Tests) -> 1195 F = fun({N,P,Pre,E}) -> 1196 case catch run_test(Config, P, Pre) of 1197 E -> 1198 ok; 1199 Bad -> 1200 ?t:format("~nTest ~p failed. Expected~n ~p~n" 1201 "but got~n ~p~n", [N, E, Bad]), 1202 fail() 1203 end 1204 end, 1205 lists:foreach(F, Tests). 1206 1207run_test(Config, Def, Pre) -> 1208 %% io:format("testing ~s~n", [binary_to_list(Def)]), 1209 DefFile = 'leex_test.xrl', 1210 Filename = 'leex_test.erl', 1211 DataDir = ?privdir, 1212 XrlFile = filename:join(DataDir, DefFile), 1213 ErlFile = filename:join(DataDir, Filename), 1214 Opts = [return, warn_unused_vars,{outdir,DataDir}], 1215 ok = file:write_file(XrlFile, Def), 1216 LOpts = [return, {report, false} | 1217 case Pre of 1218 default -> 1219 []; 1220 _ -> 1221 [{includefile,Pre}] 1222 end], 1223 XOpts = [verbose, dfa_graph], % just to get some code coverage... 1224 LRet = leex:file(XrlFile, XOpts ++ LOpts), 1225 case LRet of 1226 {ok, _Outfile, _LWs} -> 1227 CRet = compile:file(ErlFile, Opts), 1228 case CRet of 1229 {ok, _M, _Ws} -> 1230 AbsFile = filename:rootname(ErlFile, ".erl"), 1231 Mod = leex_test, 1232 code:purge(Mod), 1233 code:load_abs(AbsFile, Mod), 1234 Mod:t(); 1235 %% warnings(ErlFile, Ws); 1236 {error, [{ErlFile,Es}], []} -> {error, Es, []}; 1237 {error, [{ErlFile,Es}], [{ErlFile,Ws}]} -> {error, Es, Ws}; 1238 Error -> Error 1239 end; 1240 {error, [{XrlFile,LEs}], []} -> {error, LEs, []}; 1241 {error, [{XrlFile,LEs}], [{XrlFile,LWs}]} -> {error, LEs, LWs}; 1242 LError -> LError 1243 end. 1244 1245extract(File, {error, Es, Ws}) -> 1246 {errors, extract(File, Es), extract(File, Ws)}; 1247extract(File, Ts) -> 1248 lists:append([T || {F, T} <- Ts, F =:= File]). 1249 1250fail() -> 1251 ?t:fail(). 1252