1%% 2%% %CopyrightBegin% 3%% 4%% Copyright Ericsson AB 2004-2018. All Rights Reserved. 5%% 6%% Licensed under the Apache License, Version 2.0 (the "License"); 7%% you may not use this file except in compliance with the License. 8%% You may obtain a copy of the License at 9%% 10%% http://www.apache.org/licenses/LICENSE-2.0 11%% 12%% Unless required by applicable law or agreed to in writing, software 13%% distributed under the License is distributed on an "AS IS" BASIS, 14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15%% See the License for the specific language governing permissions and 16%% limitations under the License. 17%% 18%% %CopyrightEnd% 19%% 20-module(qlc). 21 22%%% Purpose: Main API module qlc. Functions for evaluation. 23%%% Other files: 24%%% qlc_pt. Implements the parse transform. 25 26%% External exports 27 28%% Avoid warning for local function error/1 clashing with autoimported BIF. 29-compile({no_auto_import,[error/1]}). 30-export([parse_transform/2, transform_from_evaluator/2]). 31 32-export([q/1, q/2]). 33 34-export([eval/1, e/1, eval/2, e/2, fold/3, fold/4]). 35-export([cursor/1, cursor/2, 36 next_answers/1, next_answers/2, 37 delete_cursor/1]). 38-export([append/1, append/2]). 39 40-export([sort/1, sort/2, keysort/2, keysort/3]). 41 42-export([table/2]). 43 44-export([info/1, info/2]). 45 46-export([string_to_handle/1, string_to_handle/2, string_to_handle/3]). 47 48-export([format_error/1]). 49 50%% Exported to qlc_pt.erl only: 51-export([template_state/0, aux_name/3, name_suffix/2, vars/1, 52 var_ufold/2, var_fold/3, all_selections/1]). 53 54-dialyzer(no_improper_lists). 55 56%% When cache=list lists bigger than ?MAX_LIST_SIZE bytes are put on 57%% file. Also used when merge join finds big equivalence classes. 58-define(MAX_LIST_SIZE, 512*1024). 59 60-record(qlc_append, % qlc:append/1,2 61 {hl 62 }). 63 64-record(qlc_table, % qlc:table/2 65 {trav_fun, % traverse fun 66 trav_MS, % boolean(); true iff traverse fun takes a match spec 67 pre_fun, 68 post_fun, 69 info_fun, 70 format_fun, 71 lookup_fun, 72 parent_fun, 73 key_equality, % '==' | '=:=' | undefined (--R12B-5) 74 lu_vals, % undefined | {Position,Values}; values to be looked up 75 ms = no_match_spec 76 % match specification; [T || P <- Tab, Fs] 77 }). 78 79-record(qlc_sort, % qlc:sort/1,2 and qlc:keysort/2,3 80 {h, 81 keypos, % sort | {keysort, KeyPos} 82 unique, 83 compressed, % [] | [compressed] 84 order, 85 fs_opts, % file_sorter options 86 tmpdir_usage = allowed, % allowed | not_allowed 87 % | warning_msg | error_msg | info_msg 88 tmpdir 89 }). 90 91%% Also in qlc_pt.erl. 92-record(qlc_lc, % qlc:q/1,2 93 {lc, 94 opt % #qlc_opt 95 }). 96 97-record(qlc_list, % a prepared list 98 {l, 99 ms = no_match_spec 100 }). 101 102-record(qlc_join, % a prepared join 103 {kind, % {merge, KeyEquality} | 104 % {lookup, KeyEquality, LookupFun} 105 opt, % #qlc_opt from q/2. 106 h1, q1, c1, % to be traversed by "lookup join" 107 h2, q2, c2 % to be looked up by "lookup join" 108 }). 109 110%%% A query cursor is a tuple {qlc_cursor, Cursor} where Cursor is a pair 111%%% {CursorPid, OwnerPid}. 112 113-record(qlc_cursor, {c}). 114 115-record(qlc_opt, 116 {unique = false, % boolean() 117 cache = false, % boolean() | list (true~ets, false~no) 118 max_lookup = -1, % int() >= 0 | -1 (represents infinity) 119 join = any, % any | nested_loop | merge | lookup 120 tmpdir = "", % global tmpdir 121 lookup = any, % any | boolean() 122 max_list = ?MAX_LIST_SIZE, % int() >= 0 123 tmpdir_usage = allowed % allowed | not_allowed 124 % | warning_msg | error_msg | info_msg 125 }). 126 127-record(setup, {parent}). 128 129-define(THROWN_ERROR, {?MODULE, throw_error, _, _}). 130 131-export_type([query_cursor/0, query_handle/0]). 132 133%%% A query handle is a tuple {qlc_handle, Handle} where Handle is one 134%%% of #qlc_append, #qlc_table, #qlc_sort, and #qlc_lc. 135 136-record(qlc_handle, {h}). 137 138get_handle(#qlc_handle{h = #qlc_lc{opt = {qlc_opt, U, C, M}}=H}) -> 139 %% R11B-0. 140 H#qlc_lc{opt = #qlc_opt{unique = U, cache = C, max_lookup = M}}; 141get_handle(#qlc_handle{h = H}) -> 142 H; 143get_handle(L) when is_list(L) -> 144 L; 145get_handle(_) -> 146 badarg. 147 148%%% 149%%% Exported functions 150%%% 151 152-type(query_list_comprehension() :: term()). 153-opaque(query_cursor() :: {qlc_cursor, term()}). 154-opaque(query_handle() :: {qlc_handle, term()}). 155-type(query_handle_or_list() :: query_handle() | list()). 156-type(answers() :: [answer()]). 157-type(answer() :: term()). 158-type(abstract_expr() :: erl_parse:abstract_expr()). 159-type(match_expression() :: ets:match_spec()). 160-type(spawn_options() :: default | [proc_lib:spawn_option()]). 161-type(sort_options() :: [sort_option()] | sort_option()). 162-type(sort_option() :: {compressed, boolean()} 163 | {no_files, no_files()} 164 | {order, order()} 165 | {size, pos_integer()} 166 | {tmpdir, tmp_directory()} 167 | {unique, boolean()}). 168-type(order() :: ascending | descending | order_fun()). 169-type(order_fun() :: fun((term(), term()) -> boolean())). 170-type(tmp_directory() :: [] | file:name()). 171-type(no_files() :: pos_integer()). % > 1 172-type(key_pos() :: pos_integer() | [pos_integer()]). 173-type(max_list_size() :: non_neg_integer()). 174-type(cache() :: ets | list | no). 175-type(tmp_file_usage() :: allowed | not_allowed | info_msg 176 | warning_msg | error_msg). 177 178-spec(append(QHL) -> QH when 179 QHL :: [query_handle_or_list()], 180 QH :: query_handle()). 181append(QHs) -> 182 Hs = [case get_handle(QH) of 183 badarg -> erlang:error(badarg, [QHs]); 184 H -> H 185 end || QH <- QHs], 186 #qlc_handle{h = #qlc_append{hl = Hs}}. 187 188-spec(append(QH1, QH2) -> QH3 when 189 QH1 :: query_handle_or_list(), 190 QH2 :: query_handle_or_list(), 191 QH3 :: query_handle()). 192append(QH1, QH2) -> 193 Hs = [case get_handle(QH) of 194 badarg -> erlang:error(badarg, [QH1, QH2]); 195 H -> H 196 end || QH <- [QH1, QH2]], 197 #qlc_handle{h = #qlc_append{hl = Hs}}. 198 199-spec(cursor(QH) -> Cursor when 200 QH :: query_handle_or_list(), 201 Cursor :: query_cursor()). 202cursor(QH) -> 203 cursor(QH, []). 204 205-spec(cursor(QH, Options) -> Cursor when 206 QH :: query_handle_or_list(), 207 Options :: [Option] | Option, 208 Option :: {cache_all, cache()} | cache_all 209 | {max_list_size, max_list_size()} 210 | {spawn_options, spawn_options()} 211 | {tmpdir_usage, tmp_file_usage()} 212 | {tmpdir, tmp_directory()} 213 | {unique_all, boolean()} | unique_all, 214 Cursor :: query_cursor()). 215cursor(QH, Options) -> 216 case {options(Options, [unique_all, cache_all, tmpdir, 217 spawn_options, max_list_size, 218 tmpdir_usage]), 219 get_handle(QH)} of 220 {B1, B2} when B1 =:= badarg; B2 =:= badarg -> 221 erlang:error(badarg, [QH, Options]); 222 {[GUnique, GCache, TmpDir, SpawnOptions0, MaxList, TmpUsage], H} -> 223 SpawnOptions = spawn_options(SpawnOptions0), 224 case cursor_process(H, GUnique, GCache, TmpDir, 225 SpawnOptions, MaxList, TmpUsage) of 226 Pid when is_pid(Pid) -> 227 #qlc_cursor{c = {Pid, self()}}; 228 Error -> 229 Error 230 end 231 end. 232 233-spec(delete_cursor(QueryCursor) -> ok when 234 QueryCursor :: query_cursor()). 235delete_cursor(#qlc_cursor{c = {_, Owner}}=C) when Owner =/= self() -> 236 erlang:error(not_cursor_owner, [C]); 237delete_cursor(#qlc_cursor{c = {Pid, _}}) -> 238 stop_cursor(Pid); 239delete_cursor(T) -> 240 erlang:error(badarg, [T]). 241 242-spec(e(QH) -> Answers | Error when 243 QH :: query_handle_or_list(), 244 Answers :: answers(), 245 Error :: {error, module(), Reason}, 246 Reason :: file_sorter:reason()). 247e(QH) -> 248 eval(QH, []). 249 250-spec(e(QH, Options) -> Answers | Error when 251 QH :: query_handle_or_list(), 252 Options :: [Option] | Option, 253 Option :: {cache_all, cache()} | cache_all 254 | {max_list_size, max_list_size()} 255 | {tmpdir_usage, tmp_file_usage()} 256 | {tmpdir, tmp_directory()} 257 | {unique_all, boolean()} | unique_all, 258 Answers :: answers(), 259 Error :: {error, module(), Reason}, 260 Reason :: file_sorter:reason()). 261e(QH, Options) -> 262 eval(QH, Options). 263 264-spec(eval(QH) -> Answers | Error when 265 QH :: query_handle_or_list(), 266 Answers :: answers(), 267 Error :: {error, module(), Reason}, 268 Reason :: file_sorter:reason()). 269eval(QH) -> 270 eval(QH, []). 271 272-spec(eval(QH, Options) -> Answers | Error when 273 QH :: query_handle_or_list(), 274 Answers :: answers(), 275 Options :: [Option] | Option, 276 Option :: {cache_all, cache()} | cache_all 277 | {max_list_size, max_list_size()} 278 | {tmpdir_usage, tmp_file_usage()} 279 | {tmpdir, tmp_directory()} 280 | {unique_all, boolean()} | unique_all, 281 Error :: {error, module(), Reason}, 282 Reason :: file_sorter:reason()). 283eval(QH, Options) -> 284 case {options(Options, [unique_all, cache_all, tmpdir, max_list_size, 285 tmpdir_usage]), 286 get_handle(QH)} of 287 {B1, B2} when B1 =:= badarg; B2 =:= badarg -> 288 erlang:error(badarg, [QH, Options]); 289 {[GUnique, GCache, TmpDir, MaxList, TmpUsage], Handle} -> 290 try 291 Prep = prepare_qlc(Handle, [], GUnique, GCache, 292 TmpDir, MaxList, TmpUsage), 293 case setup_qlc(Prep, #setup{parent = self()}) of 294 {L, Post, _LocalPost} when is_list(L) -> 295 post_funs(Post), 296 L; 297 {Objs, Post, _LocalPost} when is_function(Objs) -> 298 try 299 collect(Objs) 300 after 301 post_funs(Post) 302 end 303 end 304 catch throw:Term:Stacktrace -> 305 case Stacktrace of 306 [?THROWN_ERROR | _] -> 307 Term; 308 _ -> 309 erlang:raise(throw, Term, Stacktrace) 310 end 311 end 312 end. 313 314-spec(fold(Function, Acc0, QH) -> 315 Acc1 | Error when 316 QH :: query_handle_or_list(), 317 Function :: fun((answer(), AccIn) -> AccOut), 318 Acc0 :: term(), 319 Acc1 :: term(), 320 AccIn :: term(), 321 AccOut :: term(), 322 Error :: {error, module(), Reason}, 323 Reason :: file_sorter:reason()). 324fold(Fun, Acc0, QH) -> 325 fold(Fun, Acc0, QH, []). 326 327-spec(fold(Function, Acc0, QH, Options) -> 328 Acc1 | Error when 329 QH :: query_handle_or_list(), 330 Function :: fun((answer(), AccIn) -> AccOut), 331 Acc0 :: term(), 332 Acc1 :: term(), 333 AccIn :: term(), 334 AccOut :: term(), 335 Options :: [Option] | Option, 336 Option :: {cache_all, cache()} | cache_all 337 | {max_list_size, max_list_size()} 338 | {tmpdir_usage, tmp_file_usage()} 339 | {tmpdir, tmp_directory()} 340 | {unique_all, boolean()} | unique_all, 341 Error :: {error, module(), Reason}, 342 Reason :: file_sorter:reason()). 343fold(Fun, Acc0, QH, Options) -> 344 case {options(Options, [unique_all, cache_all, tmpdir, max_list_size, 345 tmpdir_usage]), 346 get_handle(QH)} of 347 {B1, B2} when B1 =:= badarg; B2 =:= badarg -> 348 erlang:error(badarg, [Fun, Acc0, QH, Options]); 349 {[GUnique, GCache, TmpDir, MaxList, TmpUsage], Handle} -> 350 try 351 Prep = prepare_qlc(Handle, not_a_list, GUnique, GCache, 352 TmpDir, MaxList, TmpUsage), 353 case setup_qlc(Prep, #setup{parent = self()}) of 354 {Objs, Post, _LocalPost} when is_function(Objs); 355 is_list(Objs) -> 356 try 357 fold_loop(Fun, Objs, Acc0) 358 after 359 post_funs(Post) 360 end 361 end 362 catch throw:Term:Stacktrace -> 363 case Stacktrace of 364 [?THROWN_ERROR | _] -> 365 Term; 366 _ -> 367 erlang:raise(throw, Term, Stacktrace) 368 end 369 end 370 end. 371 372-spec(format_error(Error) -> Chars when 373 Error :: {error, module(), term()}, 374 Chars :: io_lib:chars()). 375format_error(not_a_query_list_comprehension) -> 376 io_lib:format("argument is not a query list comprehension", []); 377format_error({used_generator_variable, V}) -> 378 io_lib:format("generated variable ~w must not be used in list expression", 379 [V]); 380format_error(binary_generator) -> 381 io_lib:format("cannot handle binary generators", []); 382format_error(too_complex_join) -> 383 io_lib:format("cannot handle join of three or more generators efficiently", 384 []); 385format_error(too_many_joins) -> 386 io_lib:format("cannot handle more than one join efficiently", []); 387format_error(nomatch_pattern) -> 388 io_lib:format("pattern cannot possibly match", []); 389format_error(nomatch_filter) -> 390 io_lib:format("filter evaluates to 'false'", []); 391format_error({Line, Mod, Reason}) when is_integer(Line) -> 392 io_lib:format("~p: ~ts~n", 393 [Line, lists:flatten(Mod:format_error(Reason))]); 394%% file_sorter errors 395format_error({bad_object, FileName}) -> 396 io_lib:format("the temporary file \"~ts\" holding answers is corrupt", 397 [FileName]); 398format_error(bad_object) -> 399 io_lib:format("the keys could not be extracted from some term", []); 400format_error({file_error, FileName, Reason}) -> 401 io_lib:format("\"~ts\": ~tp~n",[FileName, file:format_error(Reason)]); 402format_error({premature_eof, FileName}) -> 403 io_lib:format("\"~ts\": end-of-file was encountered inside some binary term", 404 [FileName]); 405format_error({tmpdir_usage, Why}) -> 406 io_lib:format("temporary file was needed for ~w~n", [Why]); 407format_error({error, Module, Reason}) -> 408 Module:format_error(Reason); 409format_error(E) -> 410 io_lib:format("~tp~n", [E]). 411 412-spec(info(QH) -> Info when 413 QH :: query_handle_or_list(), 414 Info :: abstract_expr() | string()). 415info(QH) -> 416 info(QH, []). 417 418-spec(info(QH, Options) -> Info when 419 QH :: query_handle_or_list(), 420 Options :: [Option] | Option, 421 Option :: EvalOption | ReturnOption, 422 EvalOption :: {cache_all, cache()} | cache_all 423 | {max_list_size, max_list_size()} 424 | {tmpdir_usage, tmp_file_usage()} 425 | {tmpdir, tmp_directory()} 426 | {unique_all, boolean()} | unique_all, 427 ReturnOption :: {depth, Depth} 428 | {flat, boolean()} 429 | {format, Format} 430 | {n_elements, NElements}, 431 Depth :: infinity | non_neg_integer(), 432 Format :: abstract_code | string, 433 NElements :: infinity | pos_integer(), 434 Info :: abstract_expr() | string()). 435info(QH, Options) -> 436 case {options(Options, [unique_all, cache_all, flat, format, n_elements, 437 depth, tmpdir, max_list_size, tmpdir_usage]), 438 get_handle(QH)} of 439 {B1, B2} when B1 =:= badarg; B2 =:= badarg -> 440 erlang:error(badarg, [QH, Options]); 441 {[GUnique, GCache, Flat, Format, NElements, 442 Depth, TmpDir, MaxList, TmpUsage], 443 H} -> 444 try 445 Prep = prepare_qlc(H, [], GUnique, GCache, 446 TmpDir, MaxList, TmpUsage), 447 Info = le_info(Prep, {NElements,Depth}), 448 AbstractCode = abstract(Info, Flat, NElements, Depth), 449 case Format of 450 abstract_code -> 451 abstract_code(AbstractCode); 452 string -> 453 Hook = fun({special, _Line, String}, _I, _P, _F) -> 454 String 455 end, 456 lists:flatten(erl_pp:expr(AbstractCode, 0, Hook)); 457 debug -> % Not documented. Intended for testing only. 458 Info 459 end 460 catch throw:Term:Stacktrace -> 461 case Stacktrace of 462 [?THROWN_ERROR | _] -> 463 Term; 464 _ -> 465 erlang:raise(throw, Term, Stacktrace) 466 end 467 end 468 end. 469 470-spec(keysort(KeyPos, QH1) -> QH2 when 471 KeyPos :: key_pos(), 472 QH1 :: query_handle_or_list(), 473 QH2 :: query_handle()). 474keysort(KeyPos, QH) -> 475 keysort(KeyPos, QH, []). 476 477-spec(keysort(KeyPos, QH1, SortOptions) -> QH2 when 478 KeyPos :: key_pos(), 479 SortOptions :: sort_options(), 480 QH1 :: query_handle_or_list(), 481 QH2 :: query_handle()). 482keysort(KeyPos, QH, Options) -> 483 case {is_keypos(KeyPos), 484 options(Options, [tmpdir, order, unique, compressed, 485 size, no_files]), 486 get_handle(QH)} of 487 {true, [TmpDir, Order, Unique,Compressed | _], H} when H =/= badarg -> 488 #qlc_handle{h = #qlc_sort{h = H, keypos = {keysort,KeyPos}, 489 unique = Unique, 490 compressed = Compressed, 491 order = Order, 492 fs_opts = listify(Options), 493 tmpdir = TmpDir}}; 494 _ -> 495 erlang:error(badarg, [KeyPos, QH, Options]) 496 end. 497 498-define(DEFAULT_NUM_OF_ANSWERS, 10). 499 500-spec(next_answers(QueryCursor) -> 501 Answers | Error when 502 QueryCursor :: query_cursor(), 503 Answers :: answers(), 504 Error :: {error, module(), Reason}, 505 Reason :: file_sorter:reason()). 506next_answers(C) -> 507 next_answers(C, ?DEFAULT_NUM_OF_ANSWERS). 508 509-spec(next_answers(QueryCursor, NumberOfAnswers) -> 510 Answers | Error when 511 QueryCursor :: query_cursor(), 512 Answers :: answers(), 513 NumberOfAnswers :: all_remaining | pos_integer(), 514 Error :: {error, module(), Reason}, 515 Reason :: file_sorter:reason()). 516next_answers(#qlc_cursor{c = {_, Owner}}=C, 517 NumOfAnswers) when Owner =/= self() -> 518 erlang:error(not_cursor_owner, [C, NumOfAnswers]); 519next_answers(#qlc_cursor{c = {Pid, _}}=C, NumOfAnswers) -> 520 N = case NumOfAnswers of 521 all_remaining -> -1; 522 _ when is_integer(NumOfAnswers), NumOfAnswers > 0 -> NumOfAnswers; 523 _ -> erlang:error(badarg, [C, NumOfAnswers]) 524 end, 525 next_loop(Pid, [], N); 526next_answers(T1, T2) -> 527 erlang:error(badarg, [T1, T2]). 528 529-spec(parse_transform(Forms, Options) -> Forms2 when 530 Forms :: [erl_parse:abstract_form()], 531 Forms2 :: [erl_parse:abstract_form()], 532 Options :: [Option], 533 Option :: type_checker | compile:option()). 534 535parse_transform(Forms, Options) -> 536 qlc_pt:parse_transform(Forms, Options). 537 538%% The funcspecs qlc:q/1 and qlc:q/2 are known by erl_eval.erl and 539%% erl_lint.erl. 540-spec(q(QLC) -> QH when 541 QLC :: query_list_comprehension(), 542 QH :: query_handle()). 543q(QLC_lc) -> 544 q(QLC_lc, []). 545 546-spec(q(QLC, Options) -> QH when 547 QH :: query_handle(), 548 Options :: [Option] | Option, 549 Option :: {max_lookup, MaxLookup} 550 | {cache, cache()} | cache 551 | {join, Join} 552 | {lookup, Lookup} 553 | {unique, boolean()} | unique, 554 MaxLookup :: non_neg_integer() | infinity, 555 Join :: any | lookup | merge | nested_loop, 556 Lookup :: boolean() | any, 557 QLC :: query_list_comprehension()). 558q(#qlc_lc{}=QLC_lc, Options) -> 559 case options(Options, [unique, cache, max_lookup, join, lookup]) of 560 [Unique, Cache, Max, Join, Lookup] -> 561 Opt = #qlc_opt{unique = Unique, cache = Cache, 562 max_lookup = Max, join = Join, lookup = Lookup}, 563 #qlc_handle{h = QLC_lc#qlc_lc{opt = Opt}}; 564 _ -> 565 erlang:error(badarg, [QLC_lc, Options]) 566 end; 567q(T1, T2) -> 568 erlang:error(badarg, [T1, T2]). 569 570-spec(sort(QH1) -> QH2 when 571 QH1 :: query_handle_or_list(), 572 QH2 :: query_handle()). 573sort(QH) -> 574 sort(QH, []). 575 576-spec(sort(QH1, SortOptions) -> QH2 when 577 SortOptions :: sort_options(), 578 QH1 :: query_handle_or_list(), 579 QH2 :: query_handle()). 580sort(QH, Options) -> 581 case {options(Options, [tmpdir, order, unique, compressed, 582 size, no_files]), get_handle(QH)} of 583 {B1, B2} when B1 =:= badarg; B2 =:= badarg -> 584 erlang:error(badarg, [QH, Options]); 585 {[TD, Order, Unique, Compressed | _], H} -> 586 #qlc_handle{h = #qlc_sort{h = H, keypos = sort, unique = Unique, 587 compressed = Compressed, order = Order, 588 fs_opts = listify(Options), 589 tmpdir = TD}} 590 end. 591 592%% Note that the generated code is evaluated by (the slow) erl_eval. 593-spec(string_to_handle(QueryString) -> QH | Error when 594 QueryString :: string(), 595 QH :: query_handle(), 596 Error :: {error, module(), Reason}, 597 Reason :: erl_parse:error_info() | erl_scan:error_info()). 598string_to_handle(Str) -> 599 string_to_handle(Str, []). 600 601-spec(string_to_handle(QueryString, Options) -> QH | Error when 602 QueryString :: string(), 603 Options :: [Option] | Option, 604 Option :: {max_lookup, MaxLookup} 605 | {cache, cache()} | cache 606 | {join, Join} 607 | {lookup, Lookup} 608 | {unique, boolean()} | unique, 609 MaxLookup :: non_neg_integer() | infinity, 610 Join :: any | lookup | merge | nested_loop, 611 Lookup :: boolean() | any, 612 QH :: query_handle(), 613 Error :: {error, module(), Reason}, 614 Reason :: erl_parse:error_info() | erl_scan:error_info()). 615string_to_handle(Str, Options) -> 616 string_to_handle(Str, Options, erl_eval:new_bindings()). 617 618-spec(string_to_handle(QueryString, Options, Bindings) -> QH | Error when 619 QueryString :: string(), 620 Options :: [Option] | Option, 621 Option :: {max_lookup, MaxLookup} 622 | {cache, cache()} | cache 623 | {join, Join} 624 | {lookup, Lookup} 625 | {unique, boolean()} | unique, 626 MaxLookup :: non_neg_integer() | infinity, 627 Join :: any | lookup | merge | nested_loop, 628 Lookup :: boolean() | any, 629 Bindings :: erl_eval:binding_struct(), 630 QH :: query_handle(), 631 Error :: {error, module(), Reason}, 632 Reason :: erl_parse:error_info() | erl_scan:error_info()). 633string_to_handle(Str, Options, Bindings) when is_list(Str) -> 634 case options(Options, [unique, cache, max_lookup, join, lookup]) of 635 badarg -> 636 erlang:error(badarg, [Str, Options, Bindings]); 637 [Unique, Cache, MaxLookup, Join, Lookup] -> 638 case erl_scan:string(Str, 1, [text]) of 639 {ok, Tokens, _} -> 640 ScanRes = 641 case erl_eval:extended_parse_exprs(Tokens) of 642 {ok, [Expr0], SBs} -> 643 {ok, Expr0, SBs}; 644 {ok, _ExprList, _SBs} -> 645 erlang:error(badarg, 646 [Str, Options, Bindings]); 647 E -> 648 E 649 end, 650 case ScanRes of 651 {ok, Expr, XBs} -> 652 Bs1 = merge_binding_structs(Bindings, XBs), 653 case qlc_pt:transform_expression(Expr, Bs1) of 654 {ok, {call, _, _QlcQ, Handle}} -> 655 {value, QLC_lc, _} = 656 erl_eval:exprs(Handle, Bs1), 657 O = #qlc_opt{unique = Unique, 658 cache = Cache, 659 max_lookup = MaxLookup, 660 join = Join, 661 lookup = Lookup}, 662 #qlc_handle{h = QLC_lc#qlc_lc{opt = O}}; 663 {not_ok, [{error, Error} | _]} -> 664 error(Error) 665 end; 666 {error, ErrorInfo} -> 667 error(ErrorInfo) 668 end; 669 {error, ErrorInfo, _EndLine} -> 670 error(ErrorInfo) 671 end 672 end; 673string_to_handle(T1, T2, T3) -> 674 erlang:error(badarg, [T1, T2, T3]). 675 676-spec(table(TraverseFun, Options) -> QH when 677 TraverseFun :: TraverseFun0 | TraverseFun1, 678 TraverseFun0 :: fun(() -> TraverseResult), 679 TraverseFun1 :: fun((match_expression()) -> TraverseResult), 680 TraverseResult :: Objects | term(), 681 Objects :: [] | [term() | ObjectList], 682 ObjectList :: TraverseFun0 | Objects, 683 Options :: [Option] | Option, 684 Option :: {format_fun, FormatFun} 685 | {info_fun, InfoFun} 686 | {lookup_fun, LookupFun} 687 | {parent_fun, ParentFun} 688 | {post_fun, PostFun} 689 | {pre_fun, PreFun} 690 | {key_equality, KeyComparison}, 691 FormatFun :: undefined | fun((SelectedObjects) -> FormatedTable), 692 SelectedObjects :: all 693 | {all, NElements, DepthFun} 694 | {match_spec, match_expression()} 695 | {lookup, Position, Keys} 696 | {lookup, Position, Keys, NElements, DepthFun}, 697 NElements :: infinity | pos_integer(), 698 DepthFun :: fun((term()) -> term()), 699 FormatedTable :: {Mod, Fun, Args} 700 | abstract_expr() 701 | string(), 702 InfoFun :: undefined | fun((InfoTag) -> InfoValue), 703 InfoTag :: indices | is_unique_objects | keypos | num_of_objects, 704 InfoValue :: undefined | term(), 705 LookupFun :: undefined | fun((Position, Keys) -> LookupResult), 706 LookupResult :: [term()] | term(), 707 ParentFun :: undefined | fun(() -> ParentFunValue), 708 PostFun :: undefined | fun(() -> term()), 709 PreFun :: undefined | fun((PreArgs) -> term()), 710 PreArgs :: [PreArg], 711 PreArg :: {parent_value, ParentFunValue} | {stop_fun, StopFun}, 712 ParentFunValue :: undefined | term(), 713 StopFun :: undefined | fun(() -> term()), 714 KeyComparison :: '=:=' | '==', 715 Position :: pos_integer(), 716 Keys :: [term()], 717 Mod :: atom(), 718 Fun :: atom(), 719 Args :: [term()], 720 QH :: query_handle()). 721table(TraverseFun, Options) when is_function(TraverseFun) -> 722 case {is_function(TraverseFun, 0), 723 IsFun1 = is_function(TraverseFun, 1)} of 724 {false, false} -> 725 erlang:error(badarg, [TraverseFun, Options]); 726 _ -> 727 case options(Options, [pre_fun, post_fun, info_fun, format_fun, 728 lookup_fun, parent_fun, key_equality]) of 729 [PreFun, PostFun, InfoFun, FormatFun, LookupFun, ParentFun, 730 KeyEquality] -> 731 T = #qlc_table{trav_fun = TraverseFun, pre_fun = PreFun, 732 post_fun = PostFun, info_fun = InfoFun, 733 parent_fun = ParentFun, 734 trav_MS = IsFun1, 735 format_fun = FormatFun, 736 lookup_fun = LookupFun, 737 key_equality = KeyEquality}, 738 #qlc_handle{h = T}; 739 badarg -> 740 erlang:error(badarg, [TraverseFun, Options]) 741 end 742 end; 743table(T1, T2) -> 744 erlang:error(badarg, [T1, T2]). 745 746-spec(transform_from_evaluator(LC, Bs) -> Return when 747 LC :: abstract_expr(), 748 Bs :: erl_eval:binding_struct(), 749 Return :: {ok, abstract_expr()} 750 | {not_ok, {error, module(), Reason :: term()}}). 751 752transform_from_evaluator(LC, Bs0) -> 753 qlc_pt:transform_from_evaluator(LC, Bs0). 754 755-define(TEMPLATE_STATE, 1). 756 757template_state() -> 758 ?TEMPLATE_STATE. 759 760aux_name(Name, N, AllNames) -> 761 {VN, _} = aux_name1(Name, N, AllNames), 762 VN. 763 764name_suffix(A, Suff) -> 765 list_to_atom(lists:concat([A, Suff])). 766 767vars(E) -> 768 var_ufold(fun({var,_L,V}) -> V end, E). 769 770var_ufold(F, E) -> 771 ordsets:from_list(var_fold(F, [], E)). 772 773all_selections([]) -> 774 [[]]; 775all_selections([{I,Cs} | ICs]) -> 776 [[{I,C} | L] || C <- Cs, L <- all_selections(ICs)]. 777 778%%% 779%%% Local functions 780%%% 781 782merge_binding_structs(Bs1, Bs2) -> 783 lists:foldl(fun({N, V}, Bs) -> erl_eval:add_binding(N, V, Bs) 784 end, Bs1, erl_eval:bindings(Bs2)). 785 786aux_name1(Name, N, AllNames) -> 787 SN = name_suffix(Name, N), 788 case sets:is_element(SN, AllNames) of 789 true -> aux_name1(Name, N + 1, AllNames); 790 false -> {SN, N} 791 end. 792 793var_fold(F, A, {var,_,V}=Var) when V =/= '_' -> 794 [F(Var) | A]; 795var_fold(F, A, T) when is_tuple(T) -> 796 var_fold(F, A, tuple_to_list(T)); 797var_fold(F, A, [E | Es]) -> 798 var_fold(F, var_fold(F, A, E), Es); 799var_fold(_F, A, _T) -> 800 A. 801 802options(Options, Keys) when is_list(Options) -> 803 options(Options, Keys, []); 804options(Option, Keys) -> 805 options([Option], Keys, []). 806 807options(Options0, [Key | Keys], L) when is_list(Options0) -> 808 Options = case lists:member(Key, Options0) of 809 true -> 810 [atom_option(Key) | lists:delete(Key, Options0)]; 811 false -> 812 Options0 813 end, 814 V = case lists:keyfind(Key, 1, Options) of 815 {format_fun, U=undefined} -> 816 {ok, U}; 817 {info_fun, U=undefined} -> 818 {ok, U}; 819 {lookup_fun, U=undefined} -> 820 {ok, U}; 821 {parent_fun, U=undefined} -> 822 {ok, U}; 823 {post_fun, U=undefined} -> 824 {ok, U}; 825 {pre_fun, U=undefined} -> 826 {ok, U}; 827 {info_fun, Fun} when is_function(Fun, 1) -> 828 {ok, Fun}; 829 {pre_fun, Fun} when is_function(Fun, 1) -> 830 {ok, Fun}; 831 {post_fun, Fun} when is_function(Fun, 0) -> 832 {ok, Fun}; 833 {lookup_fun, Fun} when is_function(Fun, 2) -> 834 {ok, Fun}; 835 {max_lookup, Max} when is_integer(Max), Max >= 0 -> 836 {ok, Max}; 837 {max_lookup, infinity} -> 838 {ok, -1}; 839 {format_fun, Fun} when is_function(Fun, 1) -> 840 {ok, Fun}; 841 {parent_fun, Fun} when is_function(Fun, 0) -> 842 {ok, Fun}; 843 {key_equality, KE='=='} -> 844 {ok, KE}; 845 {key_equality, KE='=:='} -> 846 {ok, KE}; 847 {join, J=any} -> 848 {ok, J}; 849 {join, J=nested_loop} -> 850 {ok, J}; 851 {join, J=merge} -> 852 {ok, J}; 853 {join, J=lookup} -> 854 {ok, J}; 855 {lookup, LookUp} when is_boolean(LookUp); LookUp =:= any -> 856 {ok, LookUp}; 857 {max_list_size, Max} when is_integer(Max), Max >= 0 -> 858 {ok, Max}; 859 {tmpdir_usage, TmpUsage} when TmpUsage =:= allowed; 860 TmpUsage =:= not_allowed; 861 TmpUsage =:= info_msg; 862 TmpUsage =:= warning_msg; 863 TmpUsage =:= error_msg -> 864 {ok, TmpUsage}; 865 {unique, Unique} when is_boolean(Unique) -> 866 {ok, Unique}; 867 {cache, Cache} when is_boolean(Cache); Cache =:= list -> 868 {ok, Cache}; 869 {cache, ets} -> 870 {ok, true}; 871 {cache, no} -> 872 {ok, false}; 873 {unique_all, UniqueAll} when is_boolean(UniqueAll) -> 874 {ok, UniqueAll}; 875 {cache_all, CacheAll} when is_boolean(CacheAll); 876 CacheAll =:= list -> 877 {ok, CacheAll}; 878 {cache_all, ets} -> 879 {ok, true}; 880 {cache_all, no} -> 881 {ok, false}; 882 {spawn_options, default} -> 883 {ok, default}; 884 {spawn_options, SpawnOptions} -> 885 case is_proper_list(SpawnOptions) of 886 true -> 887 {ok, SpawnOptions}; 888 false -> 889 badarg 890 end; 891 {flat, Flat} when is_boolean(Flat) -> 892 {ok, Flat}; 893 {format, Format} when Format =:= string; 894 Format =:= abstract_code; 895 Format =:= debug -> 896 {ok, Format}; 897 {n_elements, NElements} when NElements =:= infinity; 898 is_integer(NElements), 899 NElements > 0 -> 900 {ok, NElements}; 901 {depth, Depth} when Depth =:= infinity; 902 is_integer(Depth), Depth >= 0 -> 903 {ok, Depth}; 904 {order, Order} when is_function(Order, 2); 905 (Order =:= ascending); 906 (Order =:= descending) -> 907 {ok, Order}; 908 {compressed, Comp} when Comp -> 909 {ok, [compressed]}; 910 {compressed, Comp} when not Comp -> 911 {ok, []}; 912 {tmpdir, T} -> 913 {ok, T}; 914 {size, Size} when is_integer(Size), Size > 0 -> 915 {ok, Size}; 916 {no_files, NoFiles} when is_integer(NoFiles), NoFiles > 1 -> 917 {ok, NoFiles}; 918 {Key, _} -> 919 badarg; 920 false -> 921 Default = default_option(Key), 922 {ok, Default} 923 end, 924 case V of 925 badarg -> 926 badarg; 927 {ok, Value} -> 928 NewOptions = lists:keydelete(Key, 1, Options), 929 options(NewOptions, Keys, [Value | L]) 930 end; 931options([], [], L) -> 932 lists:reverse(L); 933options(_Options, _, _L) -> 934 badarg. 935 936default_option(pre_fun) -> undefined; 937default_option(post_fun) -> undefined; 938default_option(info_fun) -> undefined; 939default_option(format_fun) -> undefined; 940default_option(lookup_fun) -> undefined; 941default_option(max_lookup) -> -1; 942default_option(join) -> any; 943default_option(lookup) -> any; 944default_option(parent_fun) -> undefined; 945default_option(key_equality) -> '=:='; 946default_option(spawn_options) -> default; 947default_option(flat) -> true; 948default_option(format) -> string; 949default_option(n_elements) -> infinity; 950default_option(depth) -> infinity; 951default_option(max_list_size) -> ?MAX_LIST_SIZE; 952default_option(tmpdir_usage) -> allowed; 953default_option(cache) -> false; 954default_option(cache_all) -> false; 955default_option(unique) -> false; 956default_option(unique_all) -> false; 957default_option(order) -> ascending; % default values from file_sorter.erl 958default_option(compressed) -> []; 959default_option(tmpdir) -> ""; 960default_option(size) -> 524288; 961default_option(no_files) -> 16. 962 963atom_option(cache) -> {cache, true}; 964atom_option(unique) -> {unique, true}; 965atom_option(cache_all) -> {cache_all, true}; 966atom_option(unique_all) -> {unique_all, true}; 967atom_option(lookup) -> {lookup, true}; 968atom_option(flat) -> {flat, true}; 969atom_option(Key) -> Key. 970 971is_proper_list([_ | L]) -> 972 is_proper_list(L); 973is_proper_list(L) -> 974 L =:= []. 975 976spawn_options(default) -> 977 [link]; 978spawn_options(SpawnOptions) -> 979 lists:delete(monitor, 980 case lists:member(link, SpawnOptions) of 981 true -> 982 SpawnOptions; 983 false -> 984 [link | SpawnOptions] 985 end). 986 987is_keypos(Keypos) when is_integer(Keypos), Keypos > 0 -> 988 true; 989is_keypos([]) -> 990 false; 991is_keypos(L) -> 992 is_keyposs(L). 993 994is_keyposs([Kp | Kps]) when is_integer(Kp), Kp > 0 -> 995 is_keyposs(Kps); 996is_keyposs(Kps) -> 997 Kps =:= []. 998 999listify(L) when is_list(L) -> 1000 L; 1001listify(T) -> 1002 [T]. 1003 1004%% Optimizations to be carried out. 1005-record(optz, 1006 {unique = false, % boolean() 1007 cache = false, % boolean() | list 1008 join_option = any, % constraint set by the 'join' option 1009 fast_join = no, % no | #qlc_join. 'no' means nested loop. 1010 opt % #qlc_opt 1011 }). 1012 1013%% Prepared #qlc_lc. 1014-record(qlc, 1015 {lcf, % fun() -> Val 1016 codef, 1017 qdata, % with evaluated list expressions 1018 init_value, 1019 optz % #optz 1020 }). 1021 1022%% Prepared simple #qlc_lc. 1023-record(simple_qlc, 1024 {p, % atom(), pattern variable 1025 le, 1026 line :: erl_anno:anno(), 1027 init_value, 1028 optz % #optz 1029 }). 1030 1031-record(prepared, 1032 {qh, % #qlc_append | #qlc_table | #qlc | #simple_qlc | 1033 % #qlc_sort | list() 1034 sorted = no, % yes | no | ascending | descending 1035 sort_info = [], % 1036 sort_info2 = [], % 'sort_info' updated with pattern info; qh is LE 1037 lu_skip_quals = [], % qualifiers to skip due to lookup 1038 join = {[],[]}, % {Lookup, Merge} 1039 n_objs = undefined, % for join (not used yet) 1040 is_unique_objects = false, % boolean() 1041 is_cached = false % boolean() (true means 'ets' or 'list') 1042 }). 1043 1044%%% Cursor process functions. 1045 1046cursor_process(H, GUnique, GCache, TmpDir, SpawnOptions, MaxList, TmpUsage) -> 1047 Parent = self(), 1048 Setup = #setup{parent = Parent}, 1049 CF = fun() -> 1050 %% Unless exit/2 is trapped no cleanup can be done. 1051 %% The user is assumed not to set the flag to false. 1052 process_flag(trap_exit, true), 1053 MonRef = erlang:monitor(process, Parent), 1054 {Objs, Post, _LocalPost} = 1055 try 1056 Prep = prepare_qlc(H, not_a_list, GUnique, GCache, 1057 TmpDir, MaxList, TmpUsage), 1058 setup_qlc(Prep, Setup) 1059 catch Class:Reason:Stacktrace -> 1060 Parent ! {self(), 1061 {caught, Class, Reason, Stacktrace}}, 1062 exit(normal) 1063 end, 1064 Parent ! {self(), ok}, 1065 wait_for_request(Parent, MonRef, Post), 1066 reply(Parent, MonRef, Post, Objs) 1067 end, 1068 Pid = spawn_opt(CF, SpawnOptions), 1069 parent_fun(Pid, Parent). 1070 1071%% Expect calls from tables calling the parent_fun and finally an 'ok'. 1072parent_fun(Pid, Parent) -> 1073 receive 1074 {Pid, ok} -> Pid; 1075 {TPid, {parent_fun, Fun}} -> 1076 V = try 1077 {value, Fun()} 1078 catch Class:Reason:Stacktrace -> 1079 {parent_fun_caught, Class, Reason, Stacktrace} 1080 end, 1081 TPid ! {Parent, V}, 1082 parent_fun(Pid, Parent); 1083 {Pid, {caught, throw, Error, [?THROWN_ERROR | _]}} -> 1084 Error; 1085 {Pid, {caught, Class, Reason, Stacktrace}} -> 1086 erlang:raise(Class, Reason, Stacktrace) 1087 end. 1088 1089reply(Parent, MonRef, Post, []) -> 1090 no_more(Parent, MonRef, Post); 1091reply(Parent, MonRef, Post, [Answer | Cont]) -> 1092 Parent ! {self(), {answer, Answer}}, 1093 wait_for_request(Parent, MonRef, Post), 1094 reply(Parent, MonRef, Post, Cont); 1095reply(Parent, MonRef, Post, Cont) -> 1096 Reply = try 1097 if 1098 is_function(Cont) -> 1099 Cont(); 1100 true -> 1101 throw_error(Cont) 1102 end 1103 catch 1104 Class:Reason:Stacktrace -> 1105 post_funs(Post), 1106 Message = {caught, Class, Reason, Stacktrace}, 1107 Parent ! {self(), Message}, 1108 exit(normal) 1109 end, 1110 reply(Parent, MonRef, Post, Reply). 1111 1112no_more(Parent, MonRef, Post) -> 1113 Parent ! {self(), no_more}, 1114 wait_for_request(Parent, MonRef, Post), 1115 no_more(Parent, MonRef, Post). 1116 1117wait_for_request(Parent, MonRef, Post) -> 1118 receive 1119 {Parent, stop} -> 1120 post_funs(Post), 1121 exit(normal); 1122 {Parent, more} -> 1123 ok; 1124 {'EXIT', Parent, _Reason} -> 1125 post_funs(Post), 1126 exit(normal); 1127 {'DOWN', MonRef, process, Parent, _Info} -> 1128 post_funs(Post), 1129 exit(normal); 1130 {'EXIT', Pid, _Reason} when Pid =:= self() -> 1131 %% Trapped signal. The cursor ignores it... 1132 wait_for_request(Parent, MonRef, Post); 1133 Other -> 1134 error_logger:error_msg( 1135 "The qlc cursor ~w received an unexpected message:\n~tp\n", 1136 [self(), Other]), 1137 wait_for_request(Parent, MonRef, Post) 1138 end. 1139 1140%%% End of cursor process functions. 1141 1142abstract_code({special, Line, String}) -> 1143 {string, Line, String}; 1144abstract_code(Tuple) when is_tuple(Tuple) -> 1145 list_to_tuple(abstract_code(tuple_to_list(Tuple))); 1146abstract_code([H | T]) -> 1147 [abstract_code(H) | abstract_code(T)]; 1148abstract_code(Term) -> 1149 Term. 1150 1151%% Also in qlc_pt.erl. 1152-define(Q, q). 1153-define(QLC_Q(L1, L2, L3, L4, LC, Os), 1154 {call,L1,{remote,L2,{atom,L3,?MODULE},{atom,L4,?Q}},[LC | Os]}). 1155 1156abstract(Info, false=_Flat, NElements, Depth) -> 1157 abstract(Info, NElements, Depth); 1158abstract(Info, true=_Flat, NElements, Depth) -> 1159 Abstract = abstract(Info, NElements, Depth), 1160 Vars = abstract_vars(Abstract), 1161 {_, Body0, Expr} = flatten_abstr(Abstract, 1, Vars, []), 1162 case Body0 of 1163 [] -> 1164 Expr; 1165 [{match,_,Expr,Q}] -> 1166 Q; 1167 [{match,_,Expr,Q} | Body] -> 1168 {block, anno0(), lists:reverse(Body, [Q])}; 1169 _ -> 1170 {block, anno0(), lists:reverse(Body0, [Expr])} 1171 end. 1172 1173abstract(Info, NElements, Depth) -> 1174 abstract1(Info, NElements, Depth, anno1()). 1175 1176abstract1({qlc, E0, Qs0, Opt}, NElements, Depth, A) -> 1177 Qs = lists:map(fun({generate, P, LE}) -> 1178 {generate, A, binary_to_term(P), 1179 abstract1(LE, NElements, Depth, A)}; 1180 (F) -> 1181 binary_to_term(F) 1182 end, Qs0), 1183 E = binary_to_term(E0), 1184 Os = case Opt of 1185 [] -> []; 1186 _ -> [abstract_term(Opt, 1)] 1187 end, 1188 ?QLC_Q(A, A, A, A, {lc,A,E,Qs}, Os); 1189abstract1({table, {M, F, As0}}, _NElements, _Depth, Anno) 1190 when is_atom(M), is_atom(F), is_list(As0) -> 1191 As = [abstract_term(A, 1) || A <- As0], 1192 {call, Anno, {remote, Anno, {atom, Anno, M}, {atom, Anno, F}}, As}; 1193abstract1({table, TableDesc}, _NElements, _Depth, _A) -> 1194 case io_lib:deep_char_list(TableDesc) of 1195 true -> 1196 {ok, Tokens, _} = 1197 erl_scan:string(lists:flatten(TableDesc++"."), 1, [text]), 1198 {ok, Es, Bs} = 1199 erl_eval:extended_parse_exprs(Tokens), 1200 [Expr] = erl_eval:subst_values_for_vars(Es, Bs), 1201 special(Expr); 1202 false -> % abstract expression 1203 TableDesc 1204 end; 1205abstract1({append, Infos}, NElements, Depth, A) -> 1206 As = lists:foldr(fun(Info, As0) -> 1207 {cons,A,abstract1(Info, NElements, Depth, A), 1208 As0} 1209 end, {nil, A}, Infos), 1210 {call, A, {remote, A, {atom, A, ?MODULE}, {atom, A, append}}, [As]}; 1211abstract1({sort, Info, SortOptions}, NElements, Depth, A) -> 1212 {call, A, {remote, A, {atom, A, ?MODULE}, {atom, A, sort}}, 1213 [abstract1(Info, NElements, Depth, A), abstract_term(SortOptions, 1)]}; 1214abstract1({keysort, Info, Kp, SortOptions}, NElements, Depth, A) -> 1215 {call, A, {remote, A, {atom, A, ?MODULE}, {atom, A, keysort}}, 1216 [abstract_term(Kp, 1), abstract1(Info, NElements, Depth, A), 1217 abstract_term(SortOptions, 1)]}; 1218abstract1({list,L,MS}, NElements, Depth, A) -> 1219 {call, A, {remote, A, {atom, A, ets}, {atom, A, match_spec_run}}, 1220 [abstract1(L, NElements, Depth, A), 1221 {call, A, {remote, A, {atom, A, ets}, {atom, A, match_spec_compile}}, 1222 [abstract_term(depth(MS, Depth), 1)]}]}; 1223abstract1({list, L}, NElements, Depth, _A) when NElements =:= infinity; 1224 NElements >= length(L) -> 1225 abstract_term(depth(L, Depth), 1); 1226abstract1({list, L}, NElements, Depth, _A) -> 1227 abstract_term(depth(lists:sublist(L, NElements), Depth) ++ '...', 1). 1228 1229special({value, _, Thing}) -> 1230 abstract_term(Thing); 1231special(Tuple) when is_tuple(Tuple) -> 1232 list_to_tuple(special(tuple_to_list(Tuple))); 1233special([E|Es]) -> 1234 [special(E)|special(Es)]; 1235special(Expr) -> 1236 Expr. 1237 1238depth(List, infinity) -> 1239 List; 1240depth(List, Depth) -> 1241 [depth1(E, Depth) || E <- List]. 1242 1243depth_fun(infinity = _Depth) -> 1244 fun(E) -> E end; 1245depth_fun(Depth) -> 1246 fun(E) -> depth1(E, Depth) end. 1247 1248depth1([]=L, _D) -> 1249 L; 1250depth1(_Term, 0) -> 1251 '...'; 1252depth1(Tuple, D) when is_tuple(Tuple) -> 1253 depth_tuple(Tuple, tuple_size(Tuple), 1, D - 1, []); 1254depth1(List, D) when is_list(List) -> 1255 if 1256 D =:= 1 -> 1257 ['...']; 1258 true -> 1259 depth_list(List, D - 1) 1260 end; 1261depth1(Binary, D) when byte_size(Binary) > D - 1 -> 1262 D1 = D - 1, 1263 <<Bin:D1/bytes,_/bytes>> = Binary, 1264 <<Bin/bytes,"...">>; 1265depth1(T, _Depth) -> 1266 T. 1267 1268depth_list([]=L, _D) -> 1269 L; 1270depth_list(_L, 0) -> 1271 '...'; 1272depth_list([E | Es], D) -> 1273 [depth1(E, D) | depth_list(Es, D - 1)]. 1274 1275depth_tuple(_Tuple, Sz, I, _D, L) when I > Sz -> 1276 list_to_tuple(lists:reverse(L)); 1277depth_tuple(_L, _Sz, _I, 0, L) -> 1278 list_to_tuple(lists:reverse(L, ['...'])); 1279depth_tuple(Tuple, Sz, I, D, L) -> 1280 E = depth1(element(I, Tuple), D), 1281 depth_tuple(Tuple, Sz, I + 1, D - 1, [E | L]). 1282 1283abstract_term(Term) -> 1284 abstract_term(Term, 0). 1285 1286abstract_term(Term, Line) -> 1287 abstr_term(Term, anno(Line)). 1288 1289abstr_term(Tuple, Line) when is_tuple(Tuple) -> 1290 {tuple,Line,[abstr_term(E, Line) || E <- tuple_to_list(Tuple)]}; 1291abstr_term([_ | _]=L, Line) -> 1292 case io_lib:char_list(L) of 1293 true -> 1294 erl_parse:abstract(L, erl_anno:line(Line)); 1295 false -> 1296 abstr_list(L, Line) 1297 end; 1298abstr_term(Fun, Line) when is_function(Fun) -> 1299 case erl_eval:fun_data(Fun) of 1300 {fun_data, _Bs, Cs} -> 1301 {'fun', Line, {clauses, Cs}}; 1302 {named_fun_data, _Bs, Name, Cs} -> 1303 {named_fun, Line, Name, Cs}; 1304 false -> 1305 {name, Name} = erlang:fun_info(Fun, name), 1306 {arity, Arity} = erlang:fun_info(Fun, arity), 1307 case erlang:fun_info(Fun, type) of 1308 {type, external} -> 1309 {module, Module} = erlang:fun_info(Fun, module), 1310 {'fun', Line, {function, 1311 {atom,Line,Module}, 1312 {atom,Line,Name}, 1313 {integer,Line,Arity}}}; 1314 {type, local} -> 1315 {'fun', Line, {function,Name,Arity}} 1316 end 1317 end; 1318abstr_term(PPR, Line) when is_pid(PPR); is_port(PPR); is_reference(PPR) -> 1319 {special, Line, lists:flatten(io_lib:write(PPR))}; 1320abstr_term(Map, Line) when is_map(Map) -> 1321 {map,Line, 1322 [{map_field_assoc,Line,abstr_term(K, Line),abstr_term(V, Line)} || 1323 {K,V} <- maps:to_list(Map)]}; 1324abstr_term(Simple, Line) -> 1325 erl_parse:abstract(Simple, erl_anno:line(Line)). 1326 1327abstr_list([H | T], Line) -> 1328 {cons, Line, abstr_term(H, Line), abstr_list(T, Line)}; 1329abstr_list(T, Line) -> 1330 abstr_term(T, Line). 1331 1332%% Since generator pattern variables cannot be used in list 1333%% expressions, it is OK to flatten out QLCs using temporary 1334%% variables. 1335flatten_abstr(?QLC_Q(L1, L2, L3, L4, LC0, Os), VN0, Vars, Body0) -> 1336 {lc,L,E,Qs0} = LC0, 1337 F = fun({generate,Ln,P,LE0}, {VN1,Body1}) -> 1338 {VN2,Body2,LE} = flatten_abstr(LE0, VN1, Vars, Body1), 1339 {{generate,Ln,P,LE}, {VN2,Body2}}; 1340 (Fil, VN_Body) -> 1341 {Fil, VN_Body} 1342 end, 1343 {Qs, {VN3,Body}} = lists:mapfoldl(F, {VN0,Body0}, Qs0), 1344 LC = {lc,L,E,Qs}, 1345 {V, VN} = aux_name1('V', VN3, Vars), 1346 Var = {var, L1, V}, 1347 QLC = ?QLC_Q(L1, L2, L3, L4, LC, Os), 1348 {VN + 1, [{match, L1, Var, QLC} | Body], Var}; 1349flatten_abstr(T0, VN0, Vars, Body0) when is_tuple(T0) -> 1350 {VN, Body, L} = flatten_abstr(tuple_to_list(T0), VN0, Vars, Body0), 1351 {VN, Body, list_to_tuple(L)}; 1352flatten_abstr([E0 | Es0], VN0, Vars, Body0) -> 1353 {VN1, Body1, E} = flatten_abstr(E0, VN0, Vars, Body0), 1354 {VN, Body, Es} = flatten_abstr(Es0, VN1, Vars, Body1), 1355 {VN, Body, [E | Es]}; 1356flatten_abstr(E, VN, _Vars, Body) -> 1357 {VN, Body, E}. 1358 1359abstract_vars(Abstract) -> 1360 sets:from_list(ordsets:to_list(vars(Abstract))). 1361 1362collect([]=L) -> 1363 L; 1364collect([Answer | Cont]) -> 1365 [Answer | collect(Cont)]; 1366collect(Cont) -> 1367 case Cont() of 1368 Answers when is_list(Answers) -> 1369 collect(Answers); 1370 Term -> 1371 throw_error(Term) 1372 end. 1373 1374fold_loop(Fun, [Obj | Cont], Acc) -> 1375 fold_loop(Fun, Cont, Fun(Obj, Acc)); 1376fold_loop(_Fun, [], Acc) -> 1377 Acc; 1378fold_loop(Fun, Cont, Acc) -> 1379 case Cont() of 1380 Objects when is_list(Objects) -> 1381 fold_loop(Fun, Objects, Acc); 1382 Term -> 1383 Term 1384 end. 1385 1386next_loop(Pid, L, N) when N =/= 0 -> 1387 case monitor_request(Pid, more) of 1388 no_more -> 1389 lists:reverse(L); 1390 {answer, Answer} -> 1391 next_loop(Pid, [Answer | L], N - 1); 1392 {caught, throw, Error, [?THROWN_ERROR | _]} -> 1393 Error; 1394 {caught, Class, Reason, Stacktrace} -> 1395 {current_stacktrace, CurrentStacktrace} = 1396 erlang:process_info(self(), current_stacktrace), 1397 erlang:raise(Class, Reason, Stacktrace ++ CurrentStacktrace); 1398 error -> 1399 erlang:error({qlc_cursor_pid_no_longer_exists, Pid}) 1400 end; 1401next_loop(_Pid, L, _N) -> 1402 lists:reverse(L). 1403 1404stop_cursor(Pid) -> 1405 erlang:monitor(process, Pid), 1406 unlink(Pid), 1407 receive 1408 {'EXIT',Pid,_Reason} -> % Simply ignore the error. 1409 receive 1410 {'DOWN',_,process,Pid,_} -> ok 1411 end 1412 after 0 -> 1413 Pid ! {self(),stop}, 1414 receive 1415 {'DOWN',_,process,Pid,_} -> ok 1416 end 1417 end. 1418 1419monitor_request(Pid, Req) -> 1420 Ref = erlang:monitor(process, Pid), 1421 Pid ! {self(), Req}, 1422 receive 1423 {'DOWN', Ref, process, Pid, _Info} -> 1424 receive 1425 {'EXIT', Pid, _Reason} -> ok 1426 after 1 -> ok end, 1427 error; 1428 {'EXIT', Pid, _Reason} -> 1429 receive 1430 {'DOWN', _, process, Pid, _} -> error 1431 end; 1432 {Pid, Reply} -> 1433 erlang:demonitor(Ref, [flush]), 1434 Reply 1435 end. 1436 1437%% Marker for skipped filter or unused generator. 1438-define(SKIP, (-1)). 1439 1440%% Qual = {gen, LE} | fil 1441-define(qual_data(QNum, GoToIndex, State, Qual), 1442 {QNum, GoToIndex, State, Qual}). 1443 1444-record(join, % generated by qlc_pt 1445 {op, q1, q2, wh1, wh2, cs_fun}). % op is unused 1446 1447%% le_info/1 returns an intermediate information format only used for 1448%% testing purposes. Changes will happen without notice. 1449%% 1450%% QueryDesc = {qlc, TemplateDesc, [QualDesc], [QueryOpt]} 1451%% | {table, TableDesc} 1452%% | {append, [QueryDesc]} 1453%% | {sort, QueryDesc, [sort_option()]} 1454%% | {keysort, key_pos(), QueryDesc, [sort_option()]} 1455%% | {list, list()} 1456%% | {list, QueryDesc, match_expression()} 1457%% TableDesc = {Mod, Fun, Args} 1458%% | erl_parse:abstract_expr() 1459%% | string() 1460%% Mod = module() 1461%% Fun = atom() 1462%% Args = [term()] 1463%% QualDesc = FilterDesc 1464%% | {generate, PatternDesc, QueryDesc} 1465%% QueryOpt = {cache, boolean()} | cache 1466%% | {unique, boolean()} | unique 1467%% FilterDesc = PatternDesc = TemplateDesc = binary() 1468 1469le_info(#prepared{qh = #simple_qlc{le = LE, p = P, line = L, optz = Optz}}, 1470 InfOpt) -> 1471 QVar = term_to_binary({var, L, P}), 1472 {qlc, QVar, [{generate, QVar, le_info(LE, InfOpt)}], opt_info(Optz)}; 1473le_info(#prepared{qh = #qlc{codef = CodeF, qdata = Qdata, optz = Optz}}, 1474 InfOpt) -> 1475 Code = CodeF(), 1476 TemplateState = template_state(), 1477 E = element(TemplateState, Code), 1478 QualInfo0 = qual_info(Qdata, Code, InfOpt), 1479 QualInfo1 = case Optz#optz.fast_join of 1480 #qlc_join{} = Join -> 1481 join_info(Join, QualInfo0, Qdata, Code); 1482 no -> 1483 QualInfo0 1484 end, 1485 QualInfo = [I || I <- QualInfo1, I =/= skip], 1486 {qlc, E, QualInfo, opt_info(Optz)}; 1487le_info(#prepared{qh = #qlc_table{format_fun = FormatFun, trav_MS = TravMS, 1488 ms = MS, lu_vals = LuVals}}, InfOpt) -> 1489 {NElements, Depth} = InfOpt, 1490 %% The 'depth' option applies to match specifications as well. 1491 %% This is for limiting imported variables (parameters). 1492 DepthFun = depth_fun(Depth), 1493 case LuVals of 1494 _ when FormatFun =:= undefined -> 1495 {table, {'$MOD', '$FUN', []}}; 1496 {Pos, Vals} -> 1497 Formated = try FormatFun({lookup, Pos, Vals, NElements, DepthFun}) 1498 catch _:_ -> FormatFun({lookup, Pos, Vals}) 1499 end, 1500 if 1501 MS =:= no_match_spec -> 1502 {table, Formated}; 1503 true -> 1504 {list, {table, Formated}, depth(MS, Depth)} 1505 end; 1506 _ when TravMS, is_list(MS) -> 1507 {table, FormatFun({match_spec, depth(MS, Depth)})}; 1508 _ when MS =:= no_match_spec -> 1509 try {table, FormatFun({all, NElements, DepthFun})} 1510 catch _:_ -> {table, FormatFun(all)} 1511 end 1512 end; 1513le_info(#prepared{qh = #qlc_append{hl = HL}}, InfOpt) -> 1514 {append, [le_info(H, InfOpt) || H <- HL]}; 1515le_info(#prepared{qh = #qlc_sort{h = H, keypos = sort, 1516 fs_opts = SortOptions0, tmpdir = TmpDir}}, 1517 InfOpt) -> 1518 SortOptions = sort_options_global_tmp(SortOptions0, TmpDir), 1519 {sort, le_info(H, InfOpt), SortOptions}; 1520le_info(#prepared{qh = #qlc_sort{h = H, keypos = {keysort, Kp}, 1521 fs_opts = SortOptions0, tmpdir = TmpDir}}, 1522 InfOpt) -> 1523 SortOptions = sort_options_global_tmp(SortOptions0, TmpDir), 1524 {keysort, le_info(H, InfOpt), Kp, SortOptions}; 1525le_info(#prepared{qh = #qlc_list{l = L, ms = no_match_spec}}, _InfOpt) -> 1526 {list, L}; 1527le_info(#prepared{qh = #qlc_list{l = L, ms = MS}},_InfOpt) when is_list(L) -> 1528 {list, {list, L}, MS}; 1529le_info(#prepared{qh = #qlc_list{l = L, ms = MS}}, InfOpt) -> 1530 {list, le_info(L, InfOpt), MS}. 1531 1532qual_info([?qual_data(_QNum, _GoI, ?SKIP, fil) | Qdata], Code, InfOpt) -> 1533 %% see skip_lookup_filters() 1534 [skip | qual_info(Qdata, Code, InfOpt)]; 1535qual_info([?qual_data(QNum, _GoI, _SI, fil) | Qdata], Code, InfOpt) -> 1536 [element(QNum + 1, Code) | qual_info(Qdata, Code, InfOpt)]; 1537qual_info([?qual_data(_QNum, _GoI, _SI, {gen,#join{}}) | Qdata], 1538 Code, InfOpt) -> 1539 [skip | qual_info(Qdata, Code, InfOpt)]; 1540qual_info([?qual_data(QNum, _GoI, _SI, {gen,LE}) | Qdata], Code, InfOpt) -> 1541 [{generate,element(QNum + 1, Code),le_info(LE, InfOpt)} | 1542 qual_info(Qdata, Code, InfOpt)]; 1543qual_info([], _Code, _InfOpt) -> 1544 []. 1545 1546join_info(Join, QInfo, Qdata, Code) -> 1547 #qlc_join{kind = Kind, q1 = QNum1a, c1 = C1, q2 = QNum2a, c2 = C2, 1548 opt = Opt} = Join, 1549 {?qual_data(JQNum,_,_,_), Rev, QNum1, QNum2, _WH1, _WH2, CsFun} = 1550 find_join_data(Qdata, QNum1a, QNum2a), 1551 {Cs1_0, Cs2_0, Compat} = CsFun(), 1552 [Cs1, Cs2] = case Compat of 1553 [] -> % --R12B-5 1554 [[{C,[{V,'=:='} || V <- Vs]} || {C,Vs} <- CVs] || 1555 CVs <- [Cs1_0, Cs2_0]]; 1556 _ -> % 'v1', R13A -- 1557 %% Only compared constants (==). 1558 [Cs1_0, Cs2_0] 1559 end, 1560 L = anno0(), 1561 G1_0 = {var,L,'G1'}, G2_0 = {var,L,'G2'}, 1562 JP = element(JQNum + 1, Code), 1563 %% Create code for wh1 and wh2 in #join{}: 1564 {{I1,G1}, {I2,G2}, QInfoL} = 1565 case Kind of 1566 {merge, _} -> 1567 {JG1,QInfo1} = join_merge_info(QNum1, QInfo, Code, G1_0, Cs1), 1568 {JG2,QInfo2} = join_merge_info(QNum2, QInfo, Code, G2_0, Cs2), 1569 {JG1, JG2, QInfo1 ++ QInfo2}; 1570 _ when Rev -> 1571 {JG2,QInfo2} = join_merge_info(QNum2, QInfo, Code, G2_0, Cs2), 1572 {J1, QInfo1} = join_lookup_info(QNum1, QInfo, G1_0), 1573 {{J1,G1_0}, JG2, QInfo2 ++ [QInfo1]}; 1574 _ -> 1575 {JG1,QInfo1} = join_merge_info(QNum1, QInfo, Code, G1_0, Cs1), 1576 {J2, QInfo2} = join_lookup_info(QNum2, QInfo, G2_0), 1577 {JG1, {J2,G2_0}, QInfo1 ++ [QInfo2]} 1578 end, 1579 {JOptVal, JOp} = kind2op(Kind), 1580 JOpt = [{join, JOptVal}] ++ opt_info(join_unique_cache(Opt)), 1581 JFil = term_to_binary({op,L,JOp, 1582 {call,L,{atom,L,element},[{integer,L,C1},G1]}, 1583 {call,L,{atom,L,element},[{integer,L,C2},G2]}}), 1584 P = term_to_binary({cons, L, G1, G2}), 1585 JInfo = {generate, JP, {qlc, P, QInfoL ++ [JFil], JOpt}}, 1586 {Before, [I1 | After]} = lists:split(QNum1 - 1, QInfo), 1587 Before ++ [JInfo] ++ lists:delete(I2, After). 1588 1589kind2op({merge, _KE}) -> {merge, '=='}; 1590kind2op({lookup, KE, _LU_fun}) -> {lookup, KE}. 1591 1592%% qlc:q(P0 || P0 = Pattern <- H1, ConstFilters), 1593%% where "P0" is a fresh variable and ConstFilters are filters that 1594%% test constant values of pattern columns. 1595join_merge_info(QNum, QInfo, Code, G, ExtraConstants) -> 1596 {generate, _, LEInfo}=I = lists:nth(QNum, QInfo), 1597 P = binary_to_term(element(QNum + 1, Code)), 1598 case {P, ExtraConstants} of 1599 {{var, _, _}, []} -> 1600 %% No need to introduce a QLC. 1601 TP = term_to_binary(G), 1602 I2 = {generate, TP, LEInfo}, 1603 {{I,G}, [I2]}; 1604 _ -> 1605 {EPV, M} = 1606 case P of 1607 {var, _, _} -> 1608 %% No need to introduce a pattern variable. 1609 {P, P}; 1610 _ -> 1611 {PV, _} = aux_name1('P', 0, abstract_vars(P)), 1612 L = erl_anno:new(0), 1613 V = {var, L, PV}, 1614 {V, {match, L, V, P}} 1615 end, 1616 DQP = term_to_binary(EPV), 1617 LEI = {generate, term_to_binary(M), LEInfo}, 1618 TP = term_to_binary(G), 1619 CFs = [begin 1620 A = anno0(), 1621 Call = {call,A,{atom,A,element},[{integer,A,Col},EPV]}, 1622 F = list2op([{op,A,Op,abstract_term(Con),Call} 1623 || {Con,Op} <- ConstOps], 'or', A), 1624 term_to_binary(F) 1625 end || 1626 {Col,ConstOps} <- ExtraConstants], 1627 {{I,G}, [{generate, TP, {qlc, DQP, [LEI | CFs], []}}]} 1628 end. 1629 1630list2op([E], _Op, _Anno) -> 1631 E; 1632list2op([E | Es], Op, Anno) -> 1633 {op,Anno,Op,E,list2op(Es, Op, Anno)}. 1634 1635join_lookup_info(QNum, QInfo, G) -> 1636 {generate, _, LEInfo}=I = lists:nth(QNum, QInfo), 1637 TP = term_to_binary(G), 1638 {I, {generate, TP, LEInfo}}. 1639 1640opt_info(#optz{unique = Unique, cache = Cache0, join_option = JoinOption}) -> 1641 %% No 'nested_loop' options are added here, even if there are 1642 %% nested loops to carry out, unless a 'nested_loop' was given as 1643 %% option. The reason is that the qlc module does not know about 1644 %% all instances of nested loops. 1645 Cache = if 1646 Cache0 -> ets; 1647 true -> Cache0 1648 end, 1649 [{T,V} || {T,V} <- [{cache,Cache},{unique,Unique}], 1650 V =/= default_option(T)] ++ 1651 [{T,V} || {T,V} <- [{join,JoinOption}], V =:= nested_loop]. 1652 1653prepare_qlc(H, InitialValue, GUnique, GCache, TmpDir, MaxList, TmpUsage) -> 1654 GOpt = #qlc_opt{unique = GUnique, cache = GCache, 1655 tmpdir = TmpDir, max_list = MaxList, 1656 tmpdir_usage = TmpUsage}, 1657 case opt_le(prep_le(H, GOpt), 1) of 1658 #prepared{qh = #qlc{} = QLC}=Prep -> 1659 Prep#prepared{qh = QLC#qlc{init_value = InitialValue}}; 1660 #prepared{qh = #simple_qlc{}=SimpleQLC}=Prep -> 1661 Prep#prepared{qh = SimpleQLC#simple_qlc{init_value = InitialValue}}; 1662 Prep -> 1663 Prep 1664 end. 1665 1666%%% The options given to append, q and table (unique and cache) as well 1667%%% as the type of expression (list, table, append, qlc...) are 1668%%% analyzed by prep_le. The results are is_unique_objects and 1669%%% is_cached. It is checked that the evaluation (in the Erlang sense) 1670%%% of list expressions yields qlc handles. 1671 1672prep_le(#qlc_lc{lc = LC_fun, opt = #qlc_opt{} = Opt0}=H, GOpt) -> 1673 #qlc_opt{unique = GUnique, cache = GCache, 1674 tmpdir = TmpDir, max_list = MaxList, 1675 tmpdir_usage = TmpUsage} = GOpt, 1676 Unique = Opt0#qlc_opt.unique or GUnique, 1677 Cache = if 1678 not GCache -> Opt0#qlc_opt.cache; 1679 true -> GCache 1680 end, 1681 Opt = Opt0#qlc_opt{unique = Unique, cache = Cache, 1682 tmpdir = TmpDir, max_list = MaxList, 1683 tmpdir_usage = TmpUsage}, 1684 prep_qlc_lc(LC_fun(), Opt, GOpt, H); 1685prep_le(#qlc_table{info_fun = IF}=T, GOpt) -> 1686 {SortInfo, Sorted} = table_sort_info(T), 1687 IsUnique = grd(IF, is_unique_objects), 1688 Prep = #prepared{qh = T, sort_info = SortInfo, sorted = Sorted, 1689 is_unique_objects = IsUnique}, 1690 Opt = if 1691 IsUnique or not GOpt#qlc_opt.unique, 1692 T#qlc_table.ms =:= no_match_spec -> 1693 GOpt#qlc_opt{cache = false}; 1694 true -> 1695 GOpt 1696 end, 1697 may_create_simple(Opt, Prep); 1698prep_le(#qlc_append{hl = HL}, GOpt) -> 1699 case lists:flatmap(fun(#prepared{qh = #qlc_list{l = []}}) -> []; 1700 (#prepared{qh = #qlc_append{hl = HL1}}) -> HL1; 1701 (H) -> [H] end, 1702 [prep_le(H, GOpt) || H <- HL]) of 1703 []=Nil -> 1704 short_list(Nil); 1705 [Prep] -> 1706 Prep; 1707 PrepL -> 1708 Cache = lists:all(fun(#prepared{is_cached = IsC}) -> IsC =/= false 1709 end, PrepL), 1710 %% The handles in hl are replaced by prepared handles: 1711 Prep = #prepared{qh = #qlc_append{hl = PrepL}, is_cached = Cache}, 1712 may_create_simple(GOpt, Prep) 1713 end; 1714prep_le(#qlc_sort{h = H0}=Q0, GOpt) -> 1715 %% The handle h is replaced by a prepared handle: 1716 Q = Q0#qlc_sort{h = prep_le(H0, GOpt)}, 1717 prep_sort(Q, GOpt); 1718prep_le([_, _ | _]=L, GOpt) -> 1719 Prep = #prepared{qh = #qlc_list{l = L}, is_cached = true}, 1720 Opt = if 1721 not GOpt#qlc_opt.unique -> 1722 GOpt#qlc_opt{cache = false}; 1723 true -> GOpt 1724 end, 1725 may_create_simple(Opt, Prep); 1726prep_le(L, _GOpt) when is_list(L) -> 1727 short_list(L); 1728prep_le(T, _GOpt) -> 1729 erlang:error({unsupported_qlc_handle, #qlc_handle{h = T}}). 1730 1731eval_le(LE_fun, GOpt) -> 1732 case LE_fun() of 1733 {error, ?MODULE, _} = Error -> 1734 throw_error(Error); 1735 R -> 1736 case get_handle(R) of 1737 badarg -> 1738 erlang:error(badarg, [R]); 1739 H -> 1740 prep_le(H, GOpt) 1741 end 1742 end. 1743 1744prep_qlc_lc({simple_v1, PVar, LE_fun, L}, Opt, GOpt, _H) -> 1745 check_lookup_option(Opt, false), 1746 prep_simple_qlc(PVar, anno(L), eval_le(LE_fun, GOpt), Opt); 1747prep_qlc_lc({qlc_v1, QFun, CodeF, Qdata0, QOpt}, Opt, GOpt, _H) -> 1748 F = fun(?qual_data(_QNum, _GoI, _SI, fil)=QualData, ModGens) -> 1749 {QualData, ModGens}; 1750 (?qual_data(_QNum, _GoI, _SI, {gen, #join{}})=QualData, ModGens) -> 1751 {QualData, ModGens}; 1752 (?qual_data(QNum, GoI, SI, {gen, LE_fun}), ModGens0) -> 1753 Prep1 = eval_le(LE_fun, GOpt), 1754 {Prep, ModGens} = 1755 prep_generator(QNum, Prep1, QOpt, Opt, ModGens0), 1756 {?qual_data(QNum, GoI, SI, {gen, Prep}), ModGens} 1757 end, 1758 {Qdata, ModGens} = lists:mapfoldl(F, [], Qdata0), 1759 SomeLookUp = lists:keymember(true, 2, ModGens), 1760 check_lookup_option(Opt, SomeLookUp), 1761 case ModGens of 1762 [{_QNum, _LookUp, all, OnePrep}] -> 1763 check_join_option(Opt), 1764 OnePrep; 1765 _ -> 1766 Prep0 = prep_qlc(QFun, CodeF, Qdata, QOpt, Opt), 1767 LU_SkipQuals = 1768 lists:flatmap(fun({QNum,_LookUp,Fs,_Prep}) -> [{QNum,Fs}] 1769 end, ModGens), 1770 Prep1 = Prep0#prepared{lu_skip_quals = LU_SkipQuals}, 1771 prep_join(Prep1, QOpt, Opt) 1772 end; 1773prep_qlc_lc(_, _Opt, _GOpt, H) -> 1774 erlang:error({unsupported_qlc_handle, #qlc_handle{h = H}}). 1775 1776prep_generator(QNum, Prep0, QOpt, Opt, ModGens) -> 1777 PosFun = fun(KeyEquality) -> pos_fun(KeyEquality, QOpt, QNum) end, 1778 MSFs = case match_specs(QOpt, QNum) of 1779 undefined -> 1780 {no_match_spec, []}; 1781 {_, _}=MSFs0 -> 1782 MSFs0 1783 end, 1784 #prepared{qh = LE} = Prep0, 1785 case prep_gen(LE, Prep0, PosFun, MSFs, Opt) of 1786 {replace, Fs, LookUp, Prep} -> 1787 {Prep, [{QNum,LookUp,Fs,Prep} | ModGens]}; 1788 {skip, SkipFils, LookUp, Prep} -> 1789 {Prep, [{QNum,LookUp,SkipFils,Prep} | ModGens]}; 1790 {no, _Fs, _LookUp, Prep} -> 1791 {Prep, ModGens} 1792 end. 1793 1794pos_fun(undefined, QOpt, QNum) -> 1795 {'=:=', constants(QOpt, QNum)}; %% --R12B-5 1796pos_fun('=:=', QOpt, QNum) -> 1797 {'=:=', constants(QOpt, QNum)}; 1798pos_fun('==', QOpt, QNum) -> 1799 try {'==', equal_constants(QOpt, QNum)} % R13A-- 1800 catch _:_ -> {'=:=', constants(QOpt, QNum)} 1801 end. 1802 1803prep_gen(#qlc_table{lu_vals = LuV0, ms = MS0, trav_MS = TravMS, 1804 info_fun = IF, lookup_fun = LU_fun, 1805 key_equality = KeyEquality}=LE0, 1806 Prep0, PosFun0, {MS, Fs}, Opt) -> 1807 PosFun = PosFun0(KeyEquality), 1808 {LuV, {STag,SkipFils}} = find_const_positions(IF, LU_fun, PosFun, Opt), 1809 LU = LuV =/= false, 1810 if 1811 LuV0 =/= undefined; MS0 =/= no_match_spec -> 1812 {no, [], false, Prep0}; 1813 MS =/= no_match_spec, LU -> 1814 MS1 = if 1815 Fs =:= SkipFils; STag =:= Fs -> 1816 %% The guard of the match specification 1817 %% is covered by the lookup. 1818 case MS of 1819 [{'$1',_Guard,['$1']}] -> % no transformation 1820 no_match_spec; 1821 [{Head,_Guard,Body}] -> 1822 [{Head,[],Body}] % true guard 1823 end; 1824 true -> 1825 MS 1826 end, 1827 Prep = Prep0#prepared{qh = LE0#qlc_table{lu_vals = LuV,ms = MS1}}, 1828 {replace, Fs, LU, Prep}; 1829 LU -> 1830 Prep = Prep0#prepared{qh = LE0#qlc_table{lu_vals = LuV}}, 1831 {skip, SkipFils, LU, Prep}; 1832 TravMS, MS =/= no_match_spec -> 1833 Prep = Prep0#prepared{qh = LE0#qlc_table{ms = MS}, 1834 is_unique_objects = false}, 1835 {replace, Fs, false, may_create_simple(Opt, Prep)}; 1836 true -> 1837 {no, [], false, Prep0} 1838 end; 1839prep_gen(#qlc_list{l = []}, Prep0, _PosFun, {_MS, Fs}, _Opt) -> 1840 %% unique and cached 1841 {replace, Fs, false, Prep0}; 1842prep_gen(#qlc_list{ms = no_match_spec}=LE0, Prep0, _PosFun, {MS, Fs}, Opt) 1843 when MS =/= no_match_spec -> 1844 Prep = Prep0#prepared{qh = LE0#qlc_list{ms = MS}, 1845 is_cached = false}, 1846 {replace, Fs, false, may_create_simple(Opt, Prep)}; 1847prep_gen(#qlc_list{}, Prep0, _PosFun, {MS, Fs}, Opt) 1848 when MS =/= no_match_spec -> 1849 ListMS = #qlc_list{l = Prep0, ms = MS}, 1850 LE = #prepared{qh = ListMS, is_cached = false}, 1851 {replace, Fs, false, may_create_simple(Opt, LE)}; 1852prep_gen(_LE0, Prep0, _PosFun, _MSFs, _Opt) -> 1853 {no, [], false, Prep0}. 1854 1855-define(SIMPLE_QVAR, 'SQV'). 1856 1857may_create_simple(#qlc_opt{unique = Unique, cache = Cache} = Opt, 1858 #prepared{is_cached = IsCached, 1859 is_unique_objects = IsUnique} = Prep) -> 1860 if 1861 Unique and not IsUnique; 1862 (Cache =/= false) and not IsCached -> 1863 prep_simple_qlc(?SIMPLE_QVAR, anno(1), Prep, Opt); 1864 true -> 1865 Prep 1866 end. 1867 1868prep_simple_qlc(PVar, Line, LE, Opt) -> 1869 check_join_option(Opt), 1870 #prepared{is_cached = IsCached, 1871 sort_info = SortInfo, sorted = Sorted, 1872 is_unique_objects = IsUnique} = LE, 1873 #qlc_opt{unique = Unique, cache = Cache} = Opt, 1874 Cachez = if 1875 Unique -> Cache; 1876 not IsCached -> Cache; 1877 true -> false 1878 end, 1879 Optz = #optz{unique = Unique and not IsUnique, 1880 cache = Cachez, opt = Opt}, 1881 QLC = #simple_qlc{p = PVar, le = LE, line = Line, 1882 init_value = not_a_list, optz = Optz}, 1883 %% LE#prepared.join is not copied 1884 #prepared{qh = QLC, is_unique_objects = IsUnique or Unique, 1885 sort_info = SortInfo, sorted = Sorted, 1886 is_cached = IsCached or (Cachez =/= false)}. 1887 1888prep_sort(#qlc_sort{h = #prepared{sorted = yes}=Prep}, _GOpt) -> 1889 Prep; 1890prep_sort(#qlc_sort{h = #prepared{is_unique_objects = IsUniqueObjs}}=Q, 1891 GOpt) -> 1892 S1 = sort_unique(IsUniqueObjs, Q), 1893 S2 = sort_tmpdir(S1, GOpt), 1894 S = S2#qlc_sort{tmpdir_usage = GOpt#qlc_opt.tmpdir_usage}, 1895 {SortInfo, Sorted} = sort_sort_info(S), 1896 #prepared{qh = S, is_cached = true, sort_info = SortInfo, 1897 sorted = Sorted, 1898 is_unique_objects = S#qlc_sort.unique or IsUniqueObjs}. 1899 1900prep_qlc(QFun, CodeF, Qdata0, QOpt, Opt) -> 1901 #qlc_opt{unique = Unique, cache = Cache, join = Join} = Opt, 1902 Optz = #optz{unique = Unique, cache = Cache, 1903 join_option = Join, opt = Opt}, 1904 {Qdata, SortInfo} = qlc_sort_info(Qdata0, QOpt), 1905 QLC = #qlc{lcf = QFun, codef = CodeF, qdata = Qdata, 1906 init_value = not_a_list, optz = Optz}, 1907 #prepared{qh = QLC, sort_info = SortInfo, 1908 is_unique_objects = Unique, 1909 is_cached = Cache =/= false}. 1910 1911%% 'sorted', 'sorted_info', and 'sorted_info2' are used to avoid 1912%% sorting on a key when there is no need to sort on the key. 'sorted' 1913%% is set by qlc:sort() only; its purpose is to assure that if columns 1914%% 1 to i are constant, then column i+1 is key-sorted (always true if 1915%% the tuples are sorted). Note: the implementation is (too?) simple. 1916%% For instance, each column is annotated with 'ascending' or 1917%% 'descending' (not yet). More exact would be, as examples, 'always 1918%% ascending' and 'ascending if all preceding columns are constant'. 1919%% 1920%% The 'size' of the template is not used (size_of_qualifier(QOpt, 0)). 1921 1922qlc_sort_info(Qdata0, QOpt) -> 1923 F = fun(?qual_data(_QNum, _GoI, _SI, fil)=Qd, Info) -> 1924 {Qd, Info}; 1925 (?qual_data(_QNum, _GoI, _SI, {gen, #join{}})=Qd, Info) -> 1926 {Qd, Info}; 1927 (?qual_data(QNum, GoI, SI, {gen, PrepLE0}), Info) -> 1928 PrepLE = sort_info(PrepLE0, QNum, QOpt), 1929 Qd = ?qual_data(QNum, GoI, SI, {gen, PrepLE}), 1930 I = [{{Column,Order}, [{traverse,QNum,C}]} || 1931 {{C,Order},What} <- PrepLE#prepared.sort_info2, 1932 What =:= [], % Something else later... 1933 Column <- equal_template_columns(QOpt, {QNum,C})], 1934 {Qd, [I | Info]} 1935 end, 1936 {Qdata, SortInfoL} = lists:mapfoldl(F, [], Qdata0), 1937 SortInfo0 = [{{Pos,Ord}, [template]} || 1938 Pos <- constant_columns(QOpt, 0), 1939 Ord <- orders(yes)] 1940 ++ lists:append(SortInfoL), 1941 SortInfo = family_union(SortInfo0), 1942 {Qdata, SortInfo}. 1943 1944sort_info(#prepared{sort_info = SI, sorted = S} = Prep, QNum, QOpt) -> 1945 SI1 = [{{C,Ord},[]} || 1946 S =/= no, 1947 is_integer(Sz = size_of_qualifier(QOpt, QNum)), 1948 Sz > 0, % the size of the pattern 1949 (NConstCols = size_of_constant_prefix(QOpt, QNum)) < Sz, 1950 C <- [NConstCols+1], 1951 Ord <- orders(S)] 1952 ++ [{{Pos,Ord},[]} || Pos <- constant_columns(QOpt, QNum), 1953 Ord <- orders(yes)] 1954 ++ [{PosOrd,[]} || {PosOrd,_} <- SI], 1955 SI2 = lists:usort(SI1), 1956 Prep#prepared{sort_info2 = SI2}. 1957 1958%orders(descending=O) -> 1959% [O]; 1960orders(ascending=O) -> 1961 [O]; 1962orders(yes) -> 1963 [ascending 1964% ,descending 1965 ]. 1966 1967sort_unique(true, #qlc_sort{fs_opts = SortOptions, keypos = sort}=Sort) -> 1968 Sort#qlc_sort{unique = false, 1969 fs_opts = 1970 lists:keydelete(unique, 1, 1971 lists:delete(unique, SortOptions))}; 1972sort_unique(_, Sort) -> 1973 Sort. 1974 1975sort_tmpdir(S, #qlc_opt{tmpdir = ""}) -> 1976 S; 1977sort_tmpdir(S, Opt) -> 1978 S#qlc_sort{tmpdir = Opt#qlc_opt.tmpdir}. 1979 1980short_list(L) -> 1981 %% length(L) < 2: all elements are known be equal 1982 #prepared{qh = #qlc_list{l = L}, sorted = yes, is_unique_objects = true, 1983 is_cached = true}. 1984 1985find_const_positions(IF, LU_fun, {KeyEquality, PosFun}, 1986 #qlc_opt{max_lookup = Max, lookup = Lookup}) 1987 when is_function(LU_fun), is_function(PosFun), is_function(IF), 1988 Lookup =/= false -> 1989 case call(IF, keypos, undefined, []) of 1990 undefined -> 1991 Indices = call(IF, indices, undefined, []), 1992 find_const_position_idx(Indices, KeyEquality, PosFun, Max, []); 1993 KeyPos -> 1994 case pos_vals(KeyPos, KeyEquality, PosFun(KeyPos), Max) of 1995 false -> 1996 find_const_position_idx(IF(indices), KeyEquality, 1997 PosFun, Max, []); 1998 PosValuesSkip -> 1999 PosValuesSkip 2000 end 2001 end; 2002find_const_positions(_IF, _LU_fun, _KE_PosFun, _Opt0) -> 2003 {false, {some,[]}}. 2004 2005find_const_position_idx([I | Is], KeyEquality, PosFun, Max, L0) -> 2006 case pos_vals(I, KeyEquality, PosFun(I), Max) of 2007 false -> 2008 find_const_position_idx(Is, KeyEquality, PosFun, Max, L0); 2009 {{_Pos, Values}, _SkipFils}=PosValuesFils -> 2010 L = [{length(Values), PosValuesFils} | L0], 2011 find_const_position_idx(Is, KeyEquality, PosFun, Max, L) 2012 end; 2013find_const_position_idx(_, _KeyEquality, _PosFun, _Max, []) -> 2014 {false, {some,[]}}; 2015find_const_position_idx(_, _KeyEquality, _PosFun, _Max, L) -> 2016 [{_,PVF} | _] = lists:sort(L), 2017 PVF. 2018 2019pos_vals(Pos, '==', {usort_needed, Values, SkipFils}, Max) -> 2020 pos_vals_max(Pos, lists:usort(Values), SkipFils, Max); 2021pos_vals(Pos, '=:=', {usort_needed, Values, SkipFils}, Max) -> 2022 pos_vals_max(Pos, lists:sort(nub(Values)), SkipFils, Max); 2023pos_vals(Pos, _KeyEquality, {values, Values, SkipFils}, Max) -> 2024 pos_vals_max(Pos, Values, SkipFils, Max); 2025pos_vals(_Pos, _KeyEquality, _T, _Max) -> 2026 false. 2027 2028nub([]) -> 2029 []; 2030nub([E | L]) -> 2031 case lists:member(E, Es=nub(L)) of 2032 true -> 2033 Es; 2034 false -> 2035 [E | Es] 2036 end. 2037 2038%% length(Values) >= 1 2039pos_vals_max(Pos, Values, Skip, Max) when Max =:= -1; Max >= length(Values) -> 2040 {{Pos, Values}, Skip}; 2041pos_vals_max(_Pos, _Value, _Skip, _Max) -> 2042 false. 2043 2044prep_join(Prep, QOpt, Opt) -> 2045 case join_opt(QOpt) of 2046 undefined -> 2047 check_join_option(Opt), 2048 Prep; 2049 EqualMatch -> 2050 {Ix, M} = case EqualMatch of 2051 {NEqual, NMatch} -> 2052 pref_join(NEqual, NMatch, Prep, QOpt, Opt); 2053 EM -> 2054 pref_join(EM, EM, Prep, QOpt, Opt) 2055 end, 2056 SI = family_union(Prep#prepared.sort_info ++ M), 2057 Prep#prepared{join = {Ix, M}, sort_info = SI} 2058 end. 2059 2060%% The parse transform ensures that only two tables are involved. 2061pref_join(Equal, Match, Prep, QOpt, #qlc_opt{join = JoinOpt}) -> 2062 JQs = [{KeyEquality, QCs} || 2063 {KeyEquality, QCsL} <- [{'==',Equal}, {'=:=',Match}], 2064 QCs <- QCsL], 2065 IxL = [pref_lookup_join(KE, QCs, Prep, QOpt) || 2066 JoinOpt =:= any orelse JoinOpt =:= lookup, 2067 {KE, QCs} <- JQs], 2068 ML = [pref_merge_join(KE, QCs, Prep, QOpt) || 2069 JoinOpt =:= any orelse JoinOpt =:= merge, 2070 {KE, QCs} <- JQs], 2071 {lists:usort(lists:append(IxL)), lists:usort(lists:append(ML))}. 2072 2073pref_lookup_join(KeyEquality, {[{Q1,C1},{Q2,C2}],Skip}, Prep, QOpt) 2074 when is_integer(C1), is_integer(C2) -> 2075 #prepared{qh = #qlc{qdata = QData}} = Prep, 2076 Is1 = lookup_qual_data(QData, Q1, KeyEquality), 2077 Lu2 = [pref_lookup_join2(Q2, C2, Q1, C1, Skip, QOpt, KeyEquality) || 2078 IC1 <- Is1, IC1 =:= C1], 2079 Is2 = lookup_qual_data(QData, Q2, KeyEquality), 2080 Lu1 = [pref_lookup_join2(Q1, C1, Q2, C2, Skip, QOpt, KeyEquality) || 2081 IC2 <- Is2, IC2 =:= C2], 2082 family(Lu1 ++ Lu2); 2083pref_lookup_join(KE, [{_,Cs1},{_,Cs2}]=L, Prep, QOpt) when is_list(Cs1), 2084 is_list(Cs2) -> 2085 %% --R12B-5 2086 lists:append([pref_lookup_join(KE, QC,Prep,QOpt) || 2087 QC <- selections_no_skip(L)]). 2088 2089lookup_qual_data(QData, QNum, KeyEquality) -> 2090 case lists:keysearch(QNum, 1, QData) of 2091 {value, ?qual_data(QNum, _, _, {gen, PrepLE})} -> 2092 join_indices(PrepLE, KeyEquality) 2093 end. 2094 2095%% If the table has a match specification (ms =/= no_match_spec) that 2096%% has _not_ been derived from a filter but from a query handle then 2097%% the lookup join cannot be done. This particular case has not been 2098%% excluded here but is taken care of in opt_join(). 2099join_indices(#prepared{qh = #qlc_table{info_fun = IF, 2100 lookup_fun = LU_fun, 2101 key_equality = KeyEquality, 2102 lu_vals = undefined}}, 2103 KE) when is_function(LU_fun), 2104 KE =:= KeyEquality orelse 2105 KE =:= '=:=' andalso 2106 KeyEquality =:= undefined -> % --R12B-5 2107 KpL = case call(IF, keypos, undefined, []) of 2108 undefined -> []; 2109 Kp -> [Kp] 2110 end, 2111 case call(IF, indices, undefined, []) of 2112 undefined -> KpL; 2113 Is0 -> lists:usort(KpL ++ Is0) 2114 end; 2115join_indices(_Prep, _KeyEquality) -> 2116 []. 2117 2118pref_lookup_join2(Q1, C1, Q2, C2, Skip, QOpt, KeyEquality) -> 2119 TemplCols = compared_template_columns(QOpt, {Q1,C1}, KeyEquality), 2120 {{Q1,C1,Q2,C2},{lookup_join,TemplCols,KeyEquality,Skip}}. 2121 2122pref_merge_join(KE, {[{Q1,C1},{Q2,C2}],Skip}, Prep, QOpt) 2123 when is_integer(C1), is_integer(C2) -> 2124 #prepared{qh = #qlc{qdata = QData}} = Prep, 2125 Sort1 = merge_qual_data(QData, Q1), 2126 Sort2 = merge_qual_data(QData, Q2), 2127 Merge = pref_merge(KE, Q1, C1, Q2, C2, Skip, Sort1, Sort2, QOpt), 2128 family_union(Merge); 2129pref_merge_join(KE, [{_,Cs1},{_,Cs2}]=L, Prep, QOpt) when is_list(Cs1), 2130 is_list(Cs2) -> 2131 %% --R12B-5 2132 lists:append([pref_merge_join(KE, QC, Prep, QOpt) || 2133 QC <- selections_no_skip(L)]). 2134 2135selections_no_skip(L) -> 2136 [{C,{some,[]}} || C <- all_selections(L)]. 2137 2138merge_qual_data(QData, QNum) -> 2139 case lists:keysearch(QNum, 1, QData) of 2140 {value, ?qual_data(QNum, _, _, {gen, PrepLE})} -> 2141 #prepared{sort_info2 = SortInfo} = PrepLE, 2142 SortInfo 2143 end. 2144 2145pref_merge(KE, Q1, C1, Q2, C2, Skip, Sort1, Sort2, QOpt) -> 2146 Col1 = {Q1,C1}, 2147 Col2 = {Q2,C2}, 2148 DoSort = [QC || {{_QNum,Col}=QC,SortL} <- [{Col1,Sort1}, {Col2,Sort2}], 2149 lists:keymember({Col, ascending}, 1, SortL) =:= false], 2150 J = [{{Q1,C1,Q2,C2}, {merge_join,DoSort,KE,Skip}}], 2151 %% true = (QOpt(template))(Col1, '==') =:= (QOpt(template))(Col2, '==') 2152 [{{Column, ascending}, J} || 2153 Column <- equal_template_columns(QOpt, Col1)] ++ [{other, J}]. 2154 2155table_sort_info(#qlc_table{info_fun = IF}) -> 2156 case call(IF, is_sorted_key, undefined, []) of 2157 undefined -> 2158 {[], no}; 2159 false -> 2160 {[], no}; 2161 true -> 2162 case call(IF, keypos, undefined, []) of 2163 undefined -> % strange 2164 {[], no}; 2165 KeyPos -> 2166 {[{{KeyPos,ascending},[]}], no} 2167 end 2168 end. 2169 2170sort_sort_info(#qlc_sort{keypos = sort, order = Ord0}) -> 2171 {[], sort_order(Ord0)}; 2172sort_sort_info(#qlc_sort{keypos = {keysort,Kp0}, order = Ord0}) -> 2173 Kp = case Kp0 of 2174 [Pos | _] -> Pos; 2175 _ -> Kp0 2176 end, 2177 {[{{Kp,sort_order(Ord0)},[]}], no}. 2178 2179sort_order(F) when is_function(F) -> 2180 no; 2181sort_order(Order) -> 2182 Order. 2183 2184check_join_option(#qlc_opt{join = any}) -> 2185 ok; 2186check_join_option(#qlc_opt{join = Join}) -> 2187 erlang:error(no_join_to_carry_out, [{join,Join}]). 2188 2189check_lookup_option(#qlc_opt{lookup = true}, false) -> 2190 erlang:error(no_lookup_to_carry_out, [{lookup,true}]); 2191check_lookup_option(_QOpt, _LuV) -> 2192 ok. 2193 2194compared_template_columns(QOpt, QNumColumn, KeyEquality) -> 2195 (QOpt(template))(QNumColumn, KeyEquality). 2196 2197equal_template_columns(QOpt, QNumColumn) -> 2198 (QOpt(template))(QNumColumn, '=='). 2199 2200%eq_template_columns(QOpt, QNumColumn) -> 2201% (QOpt(template))(QNumColumn, '=:='). 2202 2203size_of_constant_prefix(QOpt, QNum) -> 2204 (QOpt(n_leading_constant_columns))(QNum). 2205 2206constants(QOpt, QNum) -> 2207 (QOpt(constants))(QNum). 2208 2209equal_constants(QOpt, QNum) -> 2210 (QOpt(equal_constants))(QNum). 2211 2212join_opt(QOpt) -> 2213 QOpt(join). 2214 2215match_specs(QOpt, QNum) -> 2216 (QOpt(match_specs))(QNum). 2217 2218constant_columns(QOpt, QNum) -> 2219 (QOpt(constant_columns))(QNum). 2220 2221size_of_qualifier(QOpt, QNum) -> 2222 (QOpt(size))(QNum). 2223 2224%% Two optimizations are carried out: 2225%% 1. The first generator is never cached if the QLC itself is cached. 2226%% Since the answers do not need to be cached, the top-most QLC is 2227%% never cached either. Simple QLCs not holding any options are 2228%% removed. Simple QLCs are coalesced when possible. 2229%% 2. Merge join and lookup join is done if possible. 2230 2231opt_le(#prepared{qh = #simple_qlc{le = LE0, optz = Optz0}=QLC}=Prep0, 2232 GenNum) -> 2233 case LE0 of 2234 #prepared{qh = #simple_qlc{p = LE_Pvar, le = LE2, optz = Optz2}} -> 2235 %% Coalesce two simple QLCs. 2236 Cachez = case Optz2#optz.cache of 2237 false -> Optz0#optz.cache; 2238 Cache2 -> Cache2 2239 end, 2240 Optz = Optz0#optz{cache = Cachez, 2241 unique = Optz0#optz.unique or Optz2#optz.unique}, 2242 PVar = if 2243 LE_Pvar =:= ?SIMPLE_QVAR -> QLC#simple_qlc.p; 2244 true -> LE_Pvar 2245 end, 2246 Prep = Prep0#prepared{qh = QLC#simple_qlc{p = PVar, le = LE2, 2247 optz = Optz}}, 2248 opt_le(Prep, GenNum); 2249 _ -> 2250 Optz1 = no_cache_of_first_generator(Optz0, GenNum), 2251 case {opt_le(LE0, 1), Optz1} of 2252 {LE, #optz{unique = false, cache = false}} -> 2253 LE; 2254 {LE, _} -> 2255 Prep0#prepared{qh = QLC#simple_qlc{le = LE, optz = Optz1}} 2256 end 2257 end; 2258opt_le(#prepared{qh = #qlc{}, lu_skip_quals = LU_SkipQuals0}=Prep0, GenNum) -> 2259 #prepared{qh = #qlc{qdata = Qdata0, optz = Optz0}=QLC} = Prep0, 2260 #optz{join_option = JoinOption, opt = Opt} = Optz0, 2261 JoinOption = Optz0#optz.join_option, 2262 {LU_QNum, Join, JoinSkipFs, DoSort} = 2263 opt_join(Prep0#prepared.join, JoinOption, Qdata0, Opt, LU_SkipQuals0), 2264 {LU_Skip, LU_SkipQuals} = 2265 lists:partition(fun({QNum,_Fs}) -> QNum =:= LU_QNum end, 2266 LU_SkipQuals0), 2267 LU_SkipFs = lists:flatmap(fun({_QNum,Fs}) -> Fs end, LU_SkipQuals), 2268 %% If LU_QNum has a match spec it must be applied _after_ the 2269 %% lookup join (the filter must not be skipped!). 2270 Qdata1 = if 2271 LU_Skip =:= [] -> Qdata0; 2272 true -> activate_join_lookup_filter(LU_QNum, Qdata0) 2273 end, 2274 Qdata2 = skip_lookup_filters(Qdata1, LU_SkipFs ++ JoinSkipFs), 2275 F = fun(?qual_data(QNum, GoI, SI, {gen, #prepared{}=PrepLE}), GenNum1) -> 2276 NewPrepLE = maybe_sort(PrepLE, QNum, DoSort, Opt), 2277 {?qual_data(QNum, GoI, SI, {gen, opt_le(NewPrepLE, GenNum1)}), 2278 GenNum1 + 1}; 2279 (Qd, GenNum1) -> 2280 {Qd, GenNum1} 2281 end, 2282 {Qdata, _} = lists:mapfoldl(F, 1, Qdata2), 2283 Optz1 = no_cache_of_first_generator(Optz0, GenNum), 2284 Optz = Optz1#optz{fast_join = Join}, 2285 Prep0#prepared{qh = QLC#qlc{qdata = Qdata, optz = Optz}}; 2286opt_le(#prepared{qh = #qlc_append{hl = HL}}=Prep, GenNum) -> 2287 Hs = [opt_le(H, GenNum) || H <- HL], 2288 Prep#prepared{qh = #qlc_append{hl = Hs}}; 2289opt_le(#prepared{qh = #qlc_sort{h = H}=Sort}=Prep, GenNum) -> 2290 Prep#prepared{qh = Sort#qlc_sort{h = opt_le(H, GenNum)}}; 2291opt_le(Prep, _GenNum) -> 2292 Prep. 2293 2294no_cache_of_first_generator(Optz, GenNum) when GenNum > 1 -> 2295 Optz; 2296no_cache_of_first_generator(Optz, 1) -> 2297 Optz#optz{cache = false}. 2298 2299maybe_sort(LE, QNum, DoSort, Opt) -> 2300 case lists:keyfind(QNum, 1, DoSort) of 2301 {QNum, Col} -> 2302 #qlc_opt{tmpdir = TmpDir, tmpdir_usage = TmpUsage} = Opt, 2303 SortOpts = [{tmpdir,Dir} || Dir <- [TmpDir], Dir =/= ""], 2304 Sort = #qlc_sort{h = LE, keypos = {keysort, Col}, unique = false, 2305 compressed = [], order = ascending, 2306 fs_opts = SortOpts, tmpdir_usage = TmpUsage, 2307 tmpdir = TmpDir}, 2308 #prepared{qh = Sort, sorted = no, join = no}; 2309 false -> 2310 LE 2311 end. 2312 2313skip_lookup_filters(Qdata, []) -> 2314 Qdata; 2315skip_lookup_filters(Qdata0, LU_SkipFs) -> 2316 [case lists:member(QNum, LU_SkipFs) of 2317 true -> 2318 ?qual_data(QNum, GoI, ?SKIP, fil); 2319 false -> 2320 Qd 2321 end || ?qual_data(QNum, GoI, _, _)=Qd <- Qdata0]. 2322 2323%% If the qualifier used for lookup by the join (QNum) has a match 2324%% specification it must be applied _after_ the lookup join (the 2325%% filter must not be skipped!). 2326activate_join_lookup_filter(QNum, Qdata) -> 2327 {_,GoI2,SI2,{gen,Prep2}} = lists:keyfind(QNum, 1, Qdata), 2328 Table2 = Prep2#prepared.qh, 2329 NPrep2 = Prep2#prepared{qh = Table2#qlc_table{ms = no_match_spec}}, 2330 %% Table2#qlc_table.ms has been reset; the filter will be run. 2331 lists:keyreplace(QNum, 1, Qdata, ?qual_data(QNum,GoI2,SI2,{gen,NPrep2})). 2332 2333opt_join(Join, JoinOption, Qdata, Opt, LU_SkipQuals) -> 2334 %% prep_qlc_lc() assures that no unwanted join is carried out 2335 {Ix0, M0} = Join, 2336 Ix1 = opt_join_lu(Ix0, Qdata, LU_SkipQuals), 2337 Ix = lists:reverse(lists:keysort(2, Ix1)), % prefer to skip 2338 case Ix of 2339 [{{Q1,C1,Q2,C2},Skip,KE,LU_fun} | _] -> 2340 J = #qlc_join{kind = {lookup, KE, LU_fun}, q1 = Q1, 2341 c1 = C1, q2 = Q2, c2 = C2, opt = Opt}, 2342 {Q2, J, Skip, []}; 2343 [] -> 2344 M = opt_join_merge(M0), 2345 case M of 2346 [{{Q1,C1,Q2,C2},{merge_join,DoSort,KE,Skip}}|_] -> 2347 J = #qlc_join{kind = {merge, KE}, opt = Opt, 2348 q1 = Q1, c1 = C1, q2 = Q2, c2 = C2}, 2349 {not_a_qnum, J, Skip, DoSort}; 2350 [] when JoinOption =:= nested_loop -> 2351 {not_a_qnum, no, [], []}; 2352 _ when JoinOption =/= any -> 2353 erlang:error(cannot_carry_out_join, [JoinOption]); 2354 _ -> 2355 {not_a_qnum, no, [], []} 2356 end 2357 end. 2358 2359opt_join_lu([{{_Q1,_C1,Q2,_C2}=J,[{lookup_join,_KEols,JKE,Skip0} | _]} | LJ], 2360 Qdata, LU_SkipQuals) -> 2361 {Q2,_,_,{gen,Prep2}} = lists:keyfind(Q2, 1, Qdata), 2362 #qlc_table{ms = MS, key_equality = KE, 2363 lookup_fun = LU_fun} = Prep2#prepared.qh, 2364 %% If there is no filter to skip (the match spec was derived 2365 %% from a query handle) then the lookup join cannot be done. 2366 case 2367 MS =/= no_match_spec andalso 2368 lists:keymember(Q2, 1, LU_SkipQuals) =:= false 2369 of 2370 true -> 2371 opt_join_lu(LJ, Qdata, LU_SkipQuals); 2372 false -> 2373 %% The join is preferred before evaluating the match spec 2374 %% (if there is one). 2375 Skip = skip_if_possible(JKE, KE, Skip0), 2376 [{J,Skip,KE,LU_fun} | opt_join_lu(LJ, Qdata, LU_SkipQuals)] 2377 end; 2378opt_join_lu([], _Qdata, _LU_SkipQuals) -> 2379 []. 2380 2381opt_join_merge(M) -> 2382 %% Prefer not to sort arguments. Prefer to skip join filter. 2383 L = [{-length(DoSort),length(Skip), 2384 {QCs,{merge_join,DoSort,KE,Skip}}} || 2385 {_KpOrder_or_other,MJ} <- M, 2386 {QCs,{merge_join,DoSort,KE,Skip0}} <- MJ, 2387 Skip <- [skip_if_possible(KE, '==', Skip0)]], 2388 lists:reverse([J || {_,_,J} <- lists:sort(L)]). 2389 2390%% Cannot skip the join filter the join operator is '=:=' and the join 2391%% is performed using '=='. Note: the tag 'some'/'all' is not used. 2392skip_if_possible('=:=', '==', _) -> 2393 []; 2394skip_if_possible(_, _, {_SkipTag, Skip}) -> 2395 Skip. 2396 2397%% -> {Objects, Post, LocalPost} | throw() 2398%% Post is a list of funs (closures) to run afterwards. 2399%% LocalPost should be run when all objects have been found (optimization). 2400%% LocalPost will always be a subset of Post. 2401%% List expressions are evaluated, resulting in lists of objects kept in 2402%% RAM or on disk. 2403%% An error term is thrown as soon as cleanup according Post has been 2404%% done. (This is opposed to errors during evaluation; such errors are 2405%% returned as terms.) 2406setup_qlc(Prep, Setup) -> 2407 Post0 = [], 2408 setup_le(Prep, Post0, Setup). 2409 2410setup_le(#prepared{qh = #simple_qlc{le = LE, optz = Optz}}, Post0, Setup) -> 2411 {Objs, Post, LocalPost} = setup_le(LE, Post0, Setup), 2412 unique_cache(Objs, Post, LocalPost, Optz); 2413setup_le(#prepared{qh = #qlc{lcf = QFun, qdata = Qdata, init_value = V, 2414 optz = Optz}}, Post0, Setup) -> 2415 {GoTo, FirstState, Post, LocalPost} = 2416 setup_quals(Qdata, Post0, Setup, Optz), 2417 Objs = fun() -> QFun(FirstState, V, GoTo) end, 2418 unique_cache(Objs, Post, LocalPost, Optz); 2419setup_le(#prepared{qh = #qlc_table{post_fun = PostFun}=Table}, Post, Setup) -> 2420 H = table_handle(Table, Post, Setup), 2421 %% The pre fun has been called from table_handle(): 2422 {H, [PostFun | Post], []}; 2423setup_le(#prepared{qh = #qlc_append{hl = PrepL}}, Post0, Setup) -> 2424 F = fun(Prep, {Post1, LPost1}) -> 2425 {Objs, Post2, LPost2} = setup_le(Prep, Post1, Setup), 2426 {Objs, {Post2, LPost1++LPost2}} 2427 end, 2428 {ObjsL, {Post, LocalPost}} = lists:mapfoldl(F, {Post0,[]}, PrepL), 2429 {fun() -> append_loop(ObjsL, 0) end, Post, LocalPost}; 2430setup_le(#prepared{qh = #qlc_sort{h = Prep, keypos = Kp, 2431 unique = Unique, compressed = Compressed, 2432 order = Order, fs_opts = SortOptions0, 2433 tmpdir_usage = TmpUsage,tmpdir = TmpDir}}, 2434 Post0, Setup) -> 2435 SortOptions = sort_options_global_tmp(SortOptions0, TmpDir), 2436 LF = fun(Objs) -> 2437 sort_list(Objs, Order, Unique, Kp, SortOptions, Post0) 2438 end, 2439 case setup_le(Prep, Post0, Setup) of 2440 {L, Post, LocalPost} when is_list(L) -> 2441 {LF(L), Post, LocalPost}; 2442 {Objs, Post, LocalPost} -> 2443 FF = fun(Objs1) -> 2444 file_sort_handle(Objs1, Kp, SortOptions, TmpDir, 2445 Compressed, Post, LocalPost) 2446 end, 2447 sort_handle(Objs, LF, FF, SortOptions, Post, LocalPost, 2448 {TmpUsage, sorting}) 2449 end; 2450setup_le(#prepared{qh = #qlc_list{l = L, ms = MS}}, Post, _Setup) 2451 when (no_match_spec =:= MS); L =:= [] -> 2452 {L, Post, []}; 2453setup_le(#prepared{qh = #qlc_list{l = L, ms = MS}}, Post, _Setup) 2454 when is_list(L) -> 2455 {ets:match_spec_run(L, ets:match_spec_compile(MS)), Post, []}; 2456setup_le(#prepared{qh = #qlc_list{l = H0, ms = MS}}, Post0, Setup) -> 2457 {Objs0, Post, LocalPost} = setup_le(H0, Post0, Setup), 2458 Objs = ets:match_spec_run(Objs0, ets:match_spec_compile(MS)), 2459 {Objs, Post, LocalPost}. 2460 2461%% The goto table (a tuple) is created at runtime. It is accessed by 2462%% the generated code in order to find next clause to execute. For 2463%% generators there is also a fun; calling the fun runs the list 2464%% expression of the generator. There are two elements for a filter: 2465%% the first one is the state to go when the filter is false; the 2466%% other the state when the filter is true. There are three elements 2467%% for a generator G: the first one is the state of the generator 2468%% before G (or the stop state if there is no generator); the second 2469%% one is the state of the qualifier following the generator (or the 2470%% template if there is no next generator); the third one is the list 2471%% expression fun. 2472%% There are also join generators which are "activated" when it is 2473%% possbible to do a join. 2474 2475setup_quals(Qdata, Post0, Setup, Optz) -> 2476 {GoTo0, Post1, LocalPost0} = 2477 setup_quals(0, Qdata, [], Post0, [], Setup), 2478 GoTo1 = lists:keysort(1, GoTo0), 2479 FirstState0 = next_state(Qdata), 2480 {GoTo2, FirstState, Post, LocalPost1} = 2481 case Optz#optz.fast_join of 2482 #qlc_join{kind = {merge,_KE}, c1 = C1, c2 = C2, opt = Opt} = MJ -> 2483 MF = fun(_Rev, {H1, WH1}, {H2, WH2}) -> 2484 fun() -> 2485 merge_join(WH1(H1), C1, WH2(H2), C2, Opt) 2486 end 2487 end, 2488 setup_join(MJ, Qdata, GoTo1, FirstState0, MF, Post1); 2489 #qlc_join{kind = {lookup,_KE,LuF}, c1 = C1, c2 = C2} = LJ -> 2490 LF = fun(Rev, {H1, WH1}, {H2, WH2}) -> 2491 {H, W} = if 2492 Rev -> {H2, WH2}; 2493 true -> {H1, WH1} 2494 end, 2495 fun() -> 2496 lookup_join(W(H), C1, LuF, C2, Rev) 2497 end 2498 end, 2499 setup_join(LJ, Qdata, GoTo1, FirstState0, LF, Post1); 2500 no -> 2501 {flat_goto(GoTo1), FirstState0, Post1, []} 2502 end, 2503 GoTo = list_to_tuple(GoTo2), 2504 {GoTo, FirstState, Post, LocalPost0 ++ LocalPost1}. 2505 2506setup_quals(GenLoopS, [?qual_data(_QNum,GoI,?SKIP,fil) | Qdata], 2507 Gs, P, LP, Setup) -> 2508 %% ?SKIP causes runtime error. See also skip_lookup_filters(). 2509 setup_quals(GenLoopS, Qdata, [{GoI,[?SKIP,?SKIP]} | Gs], P, LP, Setup); 2510setup_quals(GenLoopS, [?qual_data(_QNum,GoI,_SI,fil) | Qdata], 2511 Gs, P, LP, Setup) -> 2512 setup_quals(GenLoopS, Qdata, [{GoI,[GenLoopS,next_state(Qdata)]} | Gs], 2513 P, LP, Setup); 2514setup_quals(GenLoopS, [?qual_data(_QNum,GoI,_SI, {gen,#join{}}) | Qdata], 2515 Gs, P, LP, Setup) -> 2516 setup_quals(GenLoopS, Qdata, [{GoI,[?SKIP,?SKIP,?SKIP]} | Gs],P,LP,Setup); 2517setup_quals(GenLoopS, [?qual_data(_QNum,GoI,SI,{gen,LE}) | Qdata], 2518 Gs, P, LP, Setup) -> 2519 {V, NP, LP1} = setup_le(LE, P, Setup), 2520 setup_quals(SI + 1, Qdata, [{GoI, [GenLoopS,next_state(Qdata),V]} | Gs], 2521 NP, LP ++ LP1, Setup); 2522setup_quals(GenLoopS, [], Gs, P, LP, _Setup) -> 2523 {[{1,[GenLoopS]} | Gs], P, LP}. 2524 2525%% Finds the qualifier in Qdata that performs the join between Q1 and 2526%% Q2, and sets it up using the handles already set up for Q1 and Q2. 2527%% Removes Q1 and Q2 from GoTo0 and updates the join qualifier in GoTo0. 2528%% Note: the parse transform has given each generator three slots 2529%% in the GoTo table. The position of these slots within the GoTo table 2530%% is fixed (at runtime). 2531%% (Assumes there is only one join-generator in Qdata.) 2532setup_join(J, Qdata, GoTo0, FirstState0, JoinFun, Post0) -> 2533 #qlc_join{q1 = QNum1a, q2 = QNum2a, opt = Opt} = J, 2534 {?qual_data(_QN,JGoI,JSI,_), Rev, QNum1, QNum2, WH1, WH2, _CsFun} = 2535 find_join_data(Qdata, QNum1a, QNum2a), 2536 [{GoI1,SI1}] = [{GoI,SI} || 2537 ?qual_data(QNum,GoI,SI,_) <- Qdata, QNum =:= QNum1], 2538 [{GoI2,SI2}] = [{GoI,SI} || 2539 ?qual_data(QNum,GoI,SI,_) <- Qdata, QNum =:= QNum2], 2540 2541 [H1] = [H || {GoI,[_Back,_Forth,H]} <- GoTo0, GoI =:= GoI1], 2542 [{BackH2,H2}] = 2543 [{Back,H} || {GoI,[Back,_Forth,H]} <- GoTo0, GoI =:= GoI2], 2544 H0 = JoinFun(Rev, {H1,WH1}, {H2,WH2}), 2545 %% The qlc expression options apply to the introduced qlc expr as well. 2546 {H, Post, LocalPost} = 2547 unique_cache(H0, Post0, [], join_unique_cache(Opt)), 2548 [JBack] = [Back || {GoI,[Back,_,_]} <- GoTo0, GoI =:= GoI1], 2549 JForth = next_after(Qdata, SI1, QNum2), 2550 GoTo1 = lists:map(fun({GoI,_}) when GoI =:= JGoI -> 2551 {JGoI, [JBack, JForth, H]}; 2552 ({GoI,_}) when GoI =:= GoI1; GoI =:= GoI2 -> 2553 {GoI, [?SKIP,?SKIP,?SKIP]}; % not necessary 2554 (Go) -> 2555 Go 2556 end, GoTo0), 2557 GoTo = lists:map(fun(S) when S =:= SI1 -> 2558 JSI; 2559 (S) when S =:= SI2 -> 2560 next_after(Qdata, S, QNum2); 2561 (S) when S =:= SI1+1 -> 2562 JSI+1; 2563 (S) when S =:= SI2+1, SI1 + 1 =:= BackH2 -> 2564 JSI+1; 2565 (S) when S =:= SI2+1 -> 2566 BackH2; 2567 (S) -> S 2568 end, flat_goto(GoTo1)), 2569 FirstState = if 2570 SI1 =:= FirstState0 -> JSI; 2571 true -> FirstState0 2572 end, 2573 {GoTo, FirstState, Post, LocalPost}. 2574 2575join_unique_cache(#qlc_opt{cache = Cache, unique = Unique}=Opt) -> 2576 #optz{cache = Cache, unique = Unique, opt = Opt}. 2577 2578flat_goto(GoTo) -> 2579 lists:flatmap(fun({_,L}) -> L end, GoTo). 2580 2581next_after([?qual_data(_, _, S, _) | Qdata], S, QNum2) -> 2582 case Qdata of 2583 [?qual_data(QNum2, _, _, _) | Qdata1] -> 2584 next_state(Qdata1); 2585 _ -> 2586 next_state(Qdata) 2587 end; 2588next_after([_ | Qdata], S, QNum2) -> 2589 next_after(Qdata, S, QNum2). 2590 2591next_state([?qual_data(_,_,_,{gen,#join{}}) | Qdata]) -> 2592 next_state(Qdata); 2593next_state([?qual_data(_,_,?SKIP,fil) | Qdata]) -> 2594 %% see skip_lookup_filters() 2595 next_state(Qdata); 2596next_state([?qual_data(_,_,S,_) | _]) -> 2597 S; 2598next_state([]) -> 2599 template_state(). 2600 2601find_join_data(Qdata, QNum1, QNum2) -> 2602 [QRev] = [{Q,Rev,QN1,QN2,H1,H2,CsF} || 2603 ?qual_data(_QN,_GoI,_SI, 2604 {gen,#join{q1 = QN1,q2 = QN2, 2605 wh1 = H1, wh2 = H2, 2606 cs_fun = CsF}})= Q <- Qdata, 2607 if 2608 QN1 =:= QNum1, QN2 =:= QNum2 -> 2609 not (Rev = false); 2610 QN1 =:= QNum2, QN2 =:= QNum1 -> 2611 Rev = true; 2612 true -> 2613 Rev = false 2614 end], 2615 QRev. 2616 2617table_handle(#qlc_table{trav_fun = TraverseFun, trav_MS = TravMS, 2618 pre_fun = PreFun, lookup_fun = LuF, 2619 parent_fun = ParentFun, lu_vals = LuVals, ms = MS}, 2620 Post, Setup) -> 2621 #setup{parent = Parent} = Setup, 2622 ParentValue = 2623 if 2624 ParentFun =:= undefined -> 2625 undefined; 2626 Parent =:= self() -> 2627 try 2628 ParentFun() 2629 catch Class:Reason:Stacktrace -> 2630 post_funs(Post), 2631 erlang:raise(Class, Reason, Stacktrace) 2632 end; 2633 true -> 2634 case monitor_request(Parent, {parent_fun, ParentFun}) of 2635 error -> % parent has died 2636 post_funs(Post), 2637 exit(normal); 2638 {value, Value} -> 2639 Value; 2640 {parent_fun_caught, Class, Reason, Stacktrace} -> 2641 %% No use augmenting Stacktrace here. 2642 post_funs(Post), 2643 erlang:raise(Class, Reason, Stacktrace) 2644 end 2645 end, 2646 StopFun = 2647 if 2648 Parent =:= self() -> 2649 undefined; 2650 true -> 2651 Cursor = #qlc_cursor{c = {self(), Parent}}, 2652 fun() -> delete_cursor(Cursor) end 2653 end, 2654 PreFunArgs = [{parent_value, ParentValue}, {stop_fun, StopFun}], 2655 _ = call(PreFun, PreFunArgs, ok, Post), 2656 case LuVals of 2657 {Pos, Vals} when MS =:= no_match_spec -> 2658 LuF(Pos, Vals); 2659 {Pos, Vals} -> 2660 case LuF(Pos, Vals) of 2661 [] -> 2662 []; 2663 Objs when is_list(Objs) -> 2664 ets:match_spec_run(Objs, 2665 ets:match_spec_compile(MS)); 2666 Error -> 2667 post_funs(Post), 2668 throw_error(Error) 2669 end; 2670 _ when not TravMS -> 2671 MS = no_match_spec, % assertion 2672 TraverseFun; 2673 _ when MS =:= no_match_spec -> 2674 fun() -> TraverseFun([{'$1',[],['$1']}]) end; 2675 _ -> 2676 fun() -> TraverseFun(MS) end 2677 end. 2678 2679-define(CHUNK_SIZE, 64*1024). 2680 2681open_file(FileName, Extra, Post) -> 2682 case file:open(FileName, [read, raw, binary | Extra]) of 2683 {ok, Fd} -> 2684 {fun() -> 2685 case file:position(Fd, bof) of 2686 {ok, 0} -> 2687 TF = fun([], _) -> 2688 []; 2689 (Ts, C) when is_list(Ts) -> 2690 lists:reverse(Ts, C) 2691 end, 2692 file_loop_read(<<>>, ?CHUNK_SIZE, {Fd,FileName}, TF); 2693 Error -> 2694 file_error(FileName, Error) 2695 end 2696 end, Fd}; 2697 Error -> 2698 post_funs(Post), 2699 throw_file_error(FileName, Error) 2700 end. 2701 2702file_loop(Bin0, Fd_FName, Ts0, TF) -> 2703 case 2704 try file_loop2(Bin0, Ts0) 2705 catch _:_ -> 2706 {_Fd, FileName} = Fd_FName, 2707 error({bad_object, FileName}) 2708 end 2709 of 2710 {terms, <<Size:4/unit:8, B/bytes>>=Bin, []} -> 2711 file_loop_read(Bin, Size - byte_size(B) + 4, Fd_FName, TF); 2712 {terms, <<Size:4/unit:8, _/bytes>>=Bin, Ts} -> 2713 C = fun() -> file_loop_read(Bin, Size+4, Fd_FName, TF) end, 2714 TF(Ts, C); 2715 {terms, B, Ts} -> 2716 C = fun() -> file_loop_read(B, ?CHUNK_SIZE, Fd_FName, TF) end, 2717 TF(Ts, C); 2718 Error -> 2719 Error 2720 end. 2721 2722file_loop2(<<Size:4/unit:8, B:Size/bytes, Bin/bytes>>, Ts) -> 2723 file_loop2(Bin, [binary_to_term(B) | Ts]); 2724file_loop2(Bin, Ts) -> 2725 {terms, Bin, Ts}. 2726 2727%% After power failures (and only then) files with corrupted Size 2728%% fields have been observed in a disk_log file. If file:read/2 is 2729%% asked to read a huge amount of data the emulator may crash. Nothing 2730%% has been done here to prevent such crashes (by inspecting 2731%% BytesToRead in some way) since temporary files will never be read 2732%% after a power failure. 2733file_loop_read(B, MinBytesToRead, {Fd, FileName}=Fd_FName, TF) -> 2734 BytesToRead = erlang:max(?CHUNK_SIZE, MinBytesToRead), 2735 case file:read(Fd, BytesToRead) of 2736 {ok, Bin} when byte_size(B) =:= 0 -> 2737 file_loop(Bin, Fd_FName, [], TF); 2738 {ok, Bin} -> 2739 case B of 2740 <<Size:4/unit:8, Tl/bytes>> 2741 when byte_size(Bin) + byte_size(Tl) >= Size -> 2742 {B1, B2} = split_binary(Bin, Size - byte_size(Tl)), 2743 Foo = fun([T], Fun) -> [T | Fun] end, 2744 %% TF should be applied exactly once. 2745 case 2746 file_loop(list_to_binary([B, B1]), Fd_FName, [], Foo) 2747 of 2748 [T | Fun] -> 2749 true = is_function(Fun), 2750 file_loop(B2, Fd_FName, [T], TF); 2751 Error -> 2752 Error 2753 end; 2754 _ -> 2755 file_loop(list_to_binary([B, Bin]), Fd_FName, [], TF) 2756 end; 2757 eof when byte_size(B) =:= 0 -> 2758 TF([], foo); 2759 eof -> 2760 error({bad_object, FileName}); 2761 Error -> 2762 file_error(FileName, Error) 2763 end. 2764 2765sort_cursor_input(H, NoObjects) -> 2766 fun(close) -> 2767 ok; 2768 (read) -> 2769 sort_cursor_input_read(H, NoObjects) 2770 end. 2771 2772sort_cursor_list_output(TmpDir, Z, Unique) -> 2773 fun(close) -> 2774 {terms, []}; 2775 ({value, NoObjects}) -> 2776 fun(BTerms) when Unique; length(BTerms) =:= NoObjects -> 2777 fun(close) -> 2778 {terms, BTerms}; 2779 (BTerms1) -> 2780 sort_cursor_file(BTerms ++ BTerms1, TmpDir, Z) 2781 end; 2782 (BTerms) -> 2783 sort_cursor_file(BTerms, TmpDir, Z) 2784 end 2785 end. 2786 2787sort_cursor_file(BTerms, TmpDir, Z) -> 2788 FName = tmp_filename(TmpDir), 2789 case file:open(FName, [write, raw, binary | Z]) of 2790 {ok, Fd} -> 2791 WFun = write_terms(FName, Fd), 2792 WFun(BTerms); 2793 Error -> 2794 throw_file_error(FName, Error) 2795 end. 2796 2797sort_options_global_tmp(S, "") -> 2798 S; 2799sort_options_global_tmp(S, TmpDir) -> 2800 [{tmpdir,TmpDir} | lists:keydelete(tmpdir, 1, S)]. 2801 2802tmp_filename(TmpDirOpt) -> 2803 U = "_", 2804 Node = node(), 2805 Pid = os:getpid(), 2806 Unique = erlang:unique_integer(), 2807 F = lists:concat([?MODULE,U,Node,U,Pid,U,Unique]), 2808 TmpDir = case TmpDirOpt of 2809 "" -> 2810 {ok, CurDir} = file:get_cwd(), 2811 CurDir; 2812 TDir -> 2813 TDir 2814 end, 2815 filename:join(filename:absname(TmpDir), F). 2816 2817write_terms(FileName, Fd) -> 2818 fun(close) -> 2819 _ = file:close(Fd), 2820 {file, FileName}; 2821 (BTerms) -> 2822 case file:write(Fd, size_bin(BTerms, [])) of 2823 ok -> 2824 write_terms(FileName, Fd); 2825 Error -> 2826 _ = file:close(Fd), 2827 throw_file_error(FileName, Error) 2828 end 2829 end. 2830 2831size_bin([], L) -> 2832 L; 2833size_bin([BinTerm | BinTerms], L) -> 2834 size_bin(BinTerms, [L, <<(byte_size(BinTerm)):4/unit:8>> | BinTerm]). 2835 2836sort_cursor_input_read([], NoObjects) -> 2837 {end_of_input, NoObjects}; 2838sort_cursor_input_read([Object | Cont], NoObjects) -> 2839 {[term_to_binary(Object)], sort_cursor_input(Cont, NoObjects + 1)}; 2840sort_cursor_input_read(F, NoObjects) -> 2841 case F() of 2842 Objects when is_list(Objects) -> 2843 sort_cursor_input_read(Objects, NoObjects); 2844 Term -> 2845 throw_error(Term) 2846 end. 2847 2848unique_cache(L, Post, LocalPost, Optz) when is_list(L) -> 2849 case Optz#optz.unique of 2850 true -> 2851 {unique_sort_list(L), Post, LocalPost}; 2852 false -> 2853 %% If Optz#optz.cache then an ETS table could be used. 2854 {L, Post, LocalPost} 2855 end; 2856unique_cache(H, Post, LocalPost, #optz{unique = false, cache = false}) -> 2857 {H, Post, LocalPost}; 2858unique_cache(H, Post, LocalPost, #optz{unique = true, cache = false}) -> 2859 E = ets:new(qlc, [set, private]), 2860 {fun() -> no_dups(H, E) end, [del_table(E) | Post], LocalPost}; 2861unique_cache(H, Post, LocalPost, #optz{unique = false, cache = true}) -> 2862 E = ets:new(qlc, [set, private]), 2863 {L, P} = unique_cache_post(E), 2864 {fun() -> cache(H, E, LocalPost) end, [P | Post], [L]}; 2865unique_cache(H, Post, LocalPost, #optz{unique = true, cache = true}) -> 2866 UT = ets:new(qlc, [bag, private]), 2867 MT = ets:new(qlc, [set, private]), 2868 {L1, P1} = unique_cache_post(UT), 2869 {L2, P2} = unique_cache_post(MT), 2870 {fun() -> ucache(H, UT, MT, LocalPost) end, [P1, P2 | Post], [L1, L2]}; 2871unique_cache(H, Post, LocalPost, #optz{unique = false, cache = list}=Optz) -> 2872 Ref = make_ref(), 2873 F = del_lcache(Ref), 2874 #qlc_opt{tmpdir = TmpDir, max_list = MaxList, tmpdir_usage = TmpUsage} = 2875 Optz#optz.opt, 2876 {fun() -> lcache(H, Ref, LocalPost, TmpDir, MaxList, TmpUsage) end, 2877 [F | Post], [F]}; 2878unique_cache(H, Post0, LocalPost0, #optz{unique = true, cache = list}=Optz) -> 2879 #qlc_opt{tmpdir = TmpDir, max_list = MaxList, tmpdir_usage = TmpUsage} = 2880 Optz#optz.opt, 2881 Size = if 2882 MaxList >= 1 bsl 31 -> (1 bsl 31) - 1; 2883 MaxList =:= 0 -> 1; 2884 true -> MaxList 2885 end, 2886 SortOptions = [{size, Size}, {tmpdir, TmpDir}], 2887 USortOptions = [{unique, true} | SortOptions], 2888 TmpUsageM = {TmpUsage, caching}, 2889 LF1 = fun(Objs) -> lists:ukeysort(1, Objs) end, 2890 FF1 = fun(Objs) -> 2891 file_sort_handle(Objs, {keysort, 1}, USortOptions, 2892 TmpDir, [], Post0, LocalPost0) 2893 end, 2894 {UH, Post1, LocalPost1} = sort_handle(tag_objects(H, 1), LF1, FF1, 2895 USortOptions, Post0, LocalPost0, 2896 TmpUsageM), 2897 LF2 = fun(Objs) -> lists:keysort(2, Objs) end, 2898 FF2 = fun(Objs) -> 2899 file_sort_handle(Objs, {keysort, 2}, SortOptions, TmpDir, 2900 [], Post1, LocalPost1) 2901 end, 2902 {SH, Post, LocalPost} = 2903 sort_handle(UH, LF2, FF2, SortOptions, Post1, LocalPost1, TmpUsageM), 2904 if 2905 is_list(SH) -> 2906 %% Remove the tag once and for all. 2907 {untag_objects2(SH), Post, LocalPost}; 2908 true -> 2909 %% Every traversal untags the objects... 2910 {fun() -> untag_objects(SH) end, Post, LocalPost} 2911 end. 2912 2913unique_cache_post(E) -> 2914 {empty_table(E), del_table(E)}. 2915 2916unique_sort_list(L) -> 2917 E = ets:new(qlc, [set, private]), 2918 unique_list(L, E). 2919 2920unique_list([], E) -> 2921 true = ets:delete(E), 2922 []; 2923unique_list([Object | Objects], E) -> 2924 case ets:member(E, Object) of 2925 false -> 2926 true = ets:insert(E, {Object}), 2927 [Object | unique_list(Objects, E)]; 2928 true -> 2929 unique_list(Objects, E) 2930 end. 2931 2932sort_list(L, CFun, true, sort, _SortOptions, _Post) when is_function(CFun) -> 2933 lists:usort(CFun, L); 2934sort_list(L, CFun, false, sort, _SortOptions, _Post) when is_function(CFun) -> 2935 lists:sort(CFun, L); 2936sort_list(L, ascending, true, sort, _SortOptions, _Post) -> 2937 lists:usort(L); 2938sort_list(L, descending, true, sort, _SortOptions, _Post) -> 2939 lists:reverse(lists:usort(L)); 2940sort_list(L, ascending, false, sort, _SortOptions, _Post) -> 2941 lists:sort(L); 2942sort_list(L, descending, false, sort, _SortOptions, _Post) -> 2943 lists:reverse(lists:sort(L)); 2944sort_list(L, Order, Unique, {keysort, Kp}, _SortOptions, _Post) 2945 when is_integer(Kp), is_atom(Order) -> 2946 case {Order, Unique} of 2947 {ascending, true} -> 2948 lists:ukeysort(Kp, L); 2949 {ascending, false} -> 2950 lists:keysort(Kp, L); 2951 {descending, true} -> 2952 lists:reverse(lists:ukeysort(Kp, L)); 2953 {descending, false} -> 2954 lists:reverse(lists:keysort(Kp, L)) 2955 end; 2956sort_list(L, _Order, _Unique, Sort, SortOptions, Post) -> 2957 In = fun(_) -> {L, fun(_) -> end_of_input end} end, 2958 Out = sort_list_output([]), 2959 TSortOptions = [{format,term} | SortOptions], 2960 do_sort(In, Out, Sort, TSortOptions, Post). 2961 2962sort_list_output(L) -> 2963 fun(close) -> 2964 lists:append(lists:reverse(L)); 2965 (Terms) when is_list(Terms) -> 2966 sort_list_output([Terms | L]) 2967 end. 2968 2969%% Don't use the file_sorter unless it is known that objects will be 2970%% put on a temporary file (optimization). 2971sort_handle(H, ListFun, FileFun, SortOptions, Post, LocalPost, TmpUsageM) -> 2972 Size = case lists:keyfind(size, 1, SortOptions) of 2973 {size, Size0} -> Size0; 2974 false -> default_option(size) 2975 end, 2976 sort_cache(H, [], Size, {ListFun, FileFun, Post, LocalPost, TmpUsageM}). 2977 2978sort_cache([], CL, _Sz, {LF, _FF, Post, LocalPost, _TmpUsageM}) -> 2979 {LF(lists:reverse(CL)), Post, LocalPost}; 2980sort_cache(Objs, CL, Sz, C) when Sz < 0 -> 2981 sort_cache2(Objs, CL, false, C); 2982sort_cache([Object | Cont], CL, Sz0, C) -> 2983 Sz = decr_list_size(Sz0, Object), 2984 sort_cache(Cont, [Object | CL], Sz, C); 2985sort_cache(F, CL, Sz, C) -> 2986 case F() of 2987 Objects when is_list(Objects) -> 2988 sort_cache(Objects, CL, Sz, C); 2989 Term -> 2990 {_LF, _FF, Post, _LocalPost, _TmpUsageM} = C, 2991 post_funs(Post), 2992 throw_error(Term) 2993 end. 2994 2995sort_cache2([], CL, _X, {LF, _FF, Post, LocalPost, _TmpUsageM}) -> 2996 {LF(lists:reverse(CL)), Post, LocalPost}; 2997sort_cache2([Object | Cont], CL, _, C) -> 2998 sort_cache2(Cont, [Object | CL], true, C); 2999sort_cache2(F, CL, false, C) -> 3000 %% Find one extra object to be sure that temporary file(s) will be 3001 %% used when calling the file_sorter. This works even if 3002 %% duplicates are removed. 3003 case F() of 3004 Objects when is_list(Objects) -> 3005 sort_cache2(Objects, CL, true, C); 3006 Term -> 3007 {_LF, _FF, Post, _LocalPost, _TmpUsageM} = C, 3008 post_funs(Post), 3009 throw_error(Term) 3010 end; 3011sort_cache2(_Cont, _CL, true, {_LF,_FF,Post,_LocalPost, {not_allowed,M}}) -> 3012 post_funs(Post), 3013 throw_reason({tmpdir_usage, M}); 3014sort_cache2(Cont, CL, true, {_LF, FF, _Post, _LocalPost, {TmpUsage, M}}) -> 3015 maybe_error_logger(TmpUsage, M), 3016 FF(lists:reverse(CL, Cont)). 3017 3018file_sort_handle(H, Kp, SortOptions, TmpDir, Compressed, Post, LocalPost) -> 3019 In = sort_cursor_input(H, 0), 3020 Unique = lists:member(unique, SortOptions) 3021 orelse 3022 lists:keymember(unique, 1, SortOptions), 3023 Out = sort_cursor_list_output(TmpDir, Compressed, Unique), 3024 Reply = do_sort(In, Out, Kp, SortOptions, Post), 3025 case Reply of 3026 {file, FileName} -> 3027 {F, Fd} = open_file(FileName, Compressed, Post), 3028 P = fun() -> _ = file:close(Fd), 3029 _ = file:delete(FileName) 3030 end, 3031 {F, [P | Post], LocalPost}; 3032 {terms, BTerms} -> 3033 try 3034 {[binary_to_term(B) || B <- BTerms], Post, LocalPost} 3035 catch Class:Reason:Stacktrace -> 3036 post_funs(Post), 3037 erlang:raise(Class, Reason, Stacktrace) 3038 end 3039 end. 3040 3041do_sort(In, Out, Sort, SortOptions, Post) -> 3042 try 3043 case do_sort(In, Out, Sort, SortOptions) of 3044 {error, Reason} -> throw_reason(Reason); 3045 Reply -> Reply 3046 end 3047 catch Class:Term:Stacktrace -> 3048 post_funs(Post), 3049 erlang:raise(Class, Term, Stacktrace) 3050 end. 3051 3052do_sort(In, Out, sort, SortOptions) -> 3053 file_sorter:sort(In, Out, SortOptions); 3054do_sort(In, Out, {keysort, KeyPos}, SortOptions) -> 3055 file_sorter:keysort(KeyPos, In, Out, SortOptions). 3056 3057del_table(Ets) -> 3058 fun() -> true = ets:delete(Ets) end. 3059 3060empty_table(Ets) -> 3061 fun() -> true = ets:delete_all_objects(Ets) end. 3062 3063append_loop([[_ | _]=L], _N) -> 3064 L; 3065append_loop([F], _N) -> 3066 F(); 3067append_loop([L | Hs], N) -> 3068 append_loop(L, N, Hs). 3069 3070append_loop([], N, Hs) -> 3071 append_loop(Hs, N); 3072append_loop([Object | Cont], N, Hs) -> 3073 [Object | append_loop(Cont, N + 1, Hs)]; 3074append_loop(F, 0, Hs) -> 3075 case F() of 3076 [] -> 3077 append_loop(Hs, 0); 3078 [Object | Cont] -> 3079 [Object | append_loop(Cont, 1, Hs)]; 3080 Term -> 3081 Term 3082 end; 3083append_loop(F, _N, Hs) -> % when _N > 0 3084 fun() -> append_loop(F, 0, Hs) end. 3085 3086no_dups([]=Cont, UTab) -> 3087 true = ets:delete_all_objects(UTab), 3088 Cont; 3089no_dups([Object | Cont], UTab) -> 3090 case ets:member(UTab, Object) of 3091 false -> 3092 true = ets:insert(UTab, {Object}), 3093 %% A fun is created here, even if Cont is a list; objects 3094 %% will not be copied to the ETS table unless requested. 3095 [Object | fun() -> no_dups(Cont, UTab) end]; 3096 true -> 3097 no_dups(Cont, UTab) 3098 end; 3099no_dups(F, UTab) -> 3100 case F() of 3101 Objects when is_list(Objects) -> 3102 no_dups(Objects, UTab); 3103 Term -> 3104 Term 3105 end. 3106 3107%% When all objects have been returned from a cached QLC, the 3108%% generators of the expression will never be called again, and so the 3109%% tables used by the generators (LocalPost) can be emptied. 3110 3111cache(H, MTab, LocalPost) -> 3112 case ets:member(MTab, 0) of 3113 false -> 3114 true = ets:insert(MTab, {0}), 3115 cache(H, MTab, 1, LocalPost); 3116 true -> 3117 cache_recall(MTab, 1) 3118 end. 3119 3120cache([]=Cont, _MTab, _SeqNo, LocalPost) -> 3121 local_post(LocalPost), 3122 Cont; 3123cache([Object | Cont], MTab, SeqNo, LocalPost) -> 3124 true = ets:insert(MTab, {SeqNo, Object}), 3125 %% A fun is created here, even if Cont is a list; objects 3126 %% will not be copied to the ETS table unless requested. 3127 [Object | fun() -> cache(Cont, MTab, SeqNo + 1, LocalPost) end]; 3128cache(F, MTab, SeqNo, LocalPost) -> 3129 case F() of 3130 Objects when is_list(Objects) -> 3131 cache(Objects, MTab, SeqNo, LocalPost); 3132 Term -> 3133 Term 3134 end. 3135 3136cache_recall(MTab, SeqNo) -> 3137 case ets:lookup(MTab, SeqNo) of 3138 []=Cont -> 3139 Cont; 3140 [{SeqNo, Object}] -> 3141 [Object | fun() -> cache_recall(MTab, SeqNo + 1) end] 3142 end. 3143 3144ucache(H, UTab, MTab, LocalPost) -> 3145 case ets:member(MTab, 0) of 3146 false -> 3147 true = ets:insert(MTab, {0}), 3148 ucache(H, UTab, MTab, 1, LocalPost); 3149 true -> 3150 ucache_recall(UTab, MTab, 1) 3151 end. 3152 3153ucache([]=Cont, _UTab, _MTab, _SeqNo, LocalPost) -> 3154 local_post(LocalPost), 3155 Cont; 3156ucache([Object | Cont], UTab, MTab, SeqNo, LocalPost) -> 3157 %% Always using 28 bits hash value... 3158 Hash = erlang:phash2(Object), 3159 case ets:lookup(UTab, Hash) of 3160 [] -> 3161 ucache3(Object, Cont, Hash, UTab, MTab, SeqNo, LocalPost); 3162 HashSeqObjects -> 3163 case lists:keymember(Object, 3, HashSeqObjects) of 3164 true -> 3165 ucache(Cont, UTab, MTab, SeqNo, LocalPost); 3166 false -> 3167 ucache3(Object, Cont, Hash, UTab, MTab, SeqNo, LocalPost) 3168 end 3169 end; 3170ucache(F, UTab, MTab, SeqNo, LocalPost) -> 3171 case F() of 3172 Objects when is_list(Objects) -> 3173 ucache(Objects, UTab, MTab, SeqNo, LocalPost); 3174 Term -> 3175 Term 3176 end. 3177 3178ucache3(Object, Cont, Hash, UTab, MTab, SeqNo, LocalPost) -> 3179 true = ets:insert(UTab, {Hash, SeqNo, Object}), 3180 true = ets:insert(MTab, {SeqNo, Hash}), 3181 %% A fun is created here, even if Cont is a list; objects 3182 %% will not be copied to the ETS table unless requested. 3183 [Object | fun() -> ucache(Cont, UTab, MTab, SeqNo+1, LocalPost) end]. 3184 3185ucache_recall(UTab, MTab, SeqNo) -> 3186 case ets:lookup(MTab, SeqNo) of 3187 []=Cont -> 3188 Cont; 3189 [{SeqNo, Hash}] -> 3190 Object = case ets:lookup(UTab, Hash) of 3191 [{Hash, SeqNo, Object0}] -> Object0; 3192 HashSeqObjects -> 3193 {Hash, SeqNo, Object0} = 3194 lists:keyfind(SeqNo, 2, HashSeqObjects), 3195 Object0 3196 end, 3197 [Object | fun() -> ucache_recall(UTab, MTab, SeqNo + 1) end] 3198 end. 3199 3200-define(LCACHE_FILE(Ref), {Ref, '$_qlc_cache_tmpfiles_'}). 3201 3202lcache(H, Ref, LocalPost, TmpDir, MaxList, TmpUsage) -> 3203 Key = ?LCACHE_FILE(Ref), 3204 case get(Key) of 3205 undefined -> 3206 lcache1(H, {Key, LocalPost, TmpDir, MaxList, TmpUsage}, 3207 MaxList, []); 3208 {file, _Fd, _TmpFile, F} -> 3209 F(); 3210 L when is_list(L) -> 3211 L 3212 end. 3213 3214lcache1([]=Cont, {Key, LocalPost, _TmpDir, _MaxList, _TmpUsage}, _Sz, Acc) -> 3215 local_post(LocalPost), 3216 case get(Key) of 3217 undefined -> 3218 put(Key, lists:reverse(Acc)), 3219 Cont; 3220 {file, Fd, TmpFile, _F} -> 3221 case lcache_write(Fd, TmpFile, Acc) of 3222 ok -> 3223 Cont; 3224 Error -> 3225 Error 3226 end 3227 end; 3228lcache1(H, State, Sz, Acc) when Sz < 0 -> 3229 {Key, LocalPost, TmpDir, MaxList, TmpUsage} = State, 3230 GetFile = 3231 case get(Key) of 3232 {file, Fd0, TmpFile, _F} -> 3233 {TmpFile, Fd0}; 3234 undefined when TmpUsage =:= not_allowed -> 3235 error({tmpdir_usage, caching}); 3236 undefined -> 3237 maybe_error_logger(TmpUsage, caching), 3238 FName = tmp_filename(TmpDir), 3239 {F, Fd0} = open_file(FName, [write], LocalPost), 3240 put(Key, {file, Fd0, FName, F}), 3241 {FName, Fd0} 3242 end, 3243 case GetFile of 3244 {FileName, Fd} -> 3245 case lcache_write(Fd, FileName, Acc) of 3246 ok -> 3247 lcache1(H, State, MaxList, []); 3248 Error -> 3249 Error 3250 end; 3251 Error -> 3252 Error 3253 end; 3254lcache1([Object | Cont], State, Sz0, Acc) -> 3255 Sz = decr_list_size(Sz0, Object), 3256 [Object | lcache2(Cont, State, Sz, [Object | Acc])]; 3257lcache1(F, State, Sz, Acc) -> 3258 case F() of 3259 Objects when is_list(Objects) -> 3260 lcache1(Objects, State, Sz, Acc); 3261 Term -> 3262 Term 3263 end. 3264 3265lcache2([Object | Cont], State, Sz0, Acc) when Sz0 >= 0 -> 3266 Sz = decr_list_size(Sz0, Object), 3267 [Object | lcache2(Cont, State, Sz, [Object | Acc])]; 3268lcache2(Cont, State, Sz, Acc) -> 3269 fun() -> lcache1(Cont, State, Sz, Acc) end. 3270 3271lcache_write(Fd, FileName, L) -> 3272 write_binary_terms(t2b(L, []), Fd, FileName). 3273 3274t2b([], Bs) -> 3275 Bs; 3276t2b([T | Ts], Bs) -> 3277 t2b(Ts, [term_to_binary(T) | Bs]). 3278 3279del_lcache(Ref) -> 3280 fun() -> 3281 Key = ?LCACHE_FILE(Ref), 3282 case get(Key) of 3283 undefined -> 3284 ok; 3285 {file, Fd, TmpFile, _F} -> 3286 _ = file:close(Fd), 3287 _ = file:delete(TmpFile), 3288 erase(Key); 3289 _L -> 3290 erase(Key) 3291 end 3292 end. 3293 3294tag_objects([Object | Cont], T) -> 3295 [{Object, T} | tag_objects2(Cont, T + 1)]; 3296tag_objects([]=Cont, _T) -> 3297 Cont; 3298tag_objects(F, T) -> 3299 case F() of 3300 Objects when is_list(Objects) -> 3301 tag_objects(Objects, T); 3302 Term -> 3303 Term 3304 end. 3305 3306tag_objects2([Object | Cont], T) -> 3307 [{Object, T} | tag_objects2(Cont, T + 1)]; 3308tag_objects2(Objects, T) -> 3309 fun() -> tag_objects(Objects, T) end. 3310 3311untag_objects([]=Objs) -> 3312 Objs; 3313untag_objects([{Object, _N} | Cont]) -> 3314 [Object | untag_objects2(Cont)]; 3315untag_objects(F) -> 3316 case F() of 3317 Objects when is_list(Objects) -> 3318 untag_objects(Objects); 3319 Term -> % Cannot happen 3320 Term 3321 end. 3322 3323untag_objects2([{Object, _N} | Cont]) -> 3324 [Object | untag_objects2(Cont)]; 3325untag_objects2([]=Cont) -> 3326 Cont; 3327untag_objects2(Objects) -> 3328 fun() -> untag_objects(Objects) end. 3329 3330%%% Merge join. 3331%%% Temporary files are used when many objects have the same key. 3332 3333-define(JWRAP(E1, E2), [E1 | E2]). 3334 3335-record(m, {id, tmpdir, max_list, tmp_usage}). 3336 3337merge_join([]=Cont, _C1, _T2, _C2, _Opt) -> 3338 Cont; 3339merge_join([E1 | L1], C1, L2, C2, Opt) -> 3340 #qlc_opt{tmpdir = TmpDir, max_list = MaxList, 3341 tmpdir_usage = TmpUsage} = Opt, 3342 M = #m{id = merge_join_id(), tmpdir = TmpDir, max_list = MaxList, 3343 tmp_usage = TmpUsage}, 3344 merge_join2(E1, element(C1, E1), L1, C1, L2, C2, M); 3345merge_join(F1, C1, L2, C2, Opt) -> 3346 case F1() of 3347 L1 when is_list(L1) -> 3348 merge_join(L1, C1, L2, C2, Opt); 3349 T1 -> 3350 T1 3351 end. 3352 3353merge_join1(_E2, _K2, []=Cont, _C1, _L2, _C2, M) -> 3354 end_merge_join(Cont, M); 3355merge_join1(E2, K2, [E1 | L1], C1, L2, C2, M) -> 3356 K1 = element(C1, E1), 3357 if 3358 K1 == K2 -> 3359 same_keys2(E1, K1, L1, C1, L2, C2, E2, M); 3360 K1 > K2 -> 3361 merge_join2(E1, K1, L1, C1, L2, C2, M); 3362 true -> % K1 < K2 3363 merge_join1(E2, K2, L1, C1, L2, C2, M) 3364 end; 3365merge_join1(E2, K2, F1, C1, L2, C2, M) -> 3366 case F1() of 3367 L1 when is_list(L1) -> 3368 merge_join1(E2, K2, L1, C1, L2, C2, M); 3369 T1 -> 3370 T1 3371 end. 3372 3373merge_join2(_E1, _K1, _L1, _C1, []=Cont, _C2, M) -> 3374 end_merge_join(Cont, M); 3375merge_join2(E1, K1, L1, C1, [E2 | L2], C2, M) -> 3376 K2 = element(C2, E2), 3377 if 3378 K1 == K2 -> 3379 same_keys2(E1, K1, L1, C1, L2, C2, E2, M); 3380 K1 > K2 -> 3381 merge_join2(E1, K1, L1, C1, L2, C2, M); 3382 true -> % K1 < K2 3383 merge_join1(E2, K2, L1, C1, L2, C2, M) 3384 end; 3385merge_join2(E1, K1, L1, C1, F2, C2, M) -> 3386 case F2() of 3387 L2 when is_list(L2) -> 3388 merge_join2(E1, K1, L1, C1, L2, C2, M); 3389 T2 -> 3390 T2 3391 end. 3392 3393%% element(C2, E2_0) == K1 3394same_keys2(E1, K1, L1, C1, [], _C2, E2_0, M) -> 3395 Cont = fun(_L1b) -> end_merge_join([], M) end, 3396 loop_same_keys(E1, K1, L1, C1, [E2_0], Cont, M); 3397same_keys2(E1, K1, L1, C1, [E2 | L2]=L2_0, C2, E2_0, M) -> 3398 K2 = element(C2, E2), 3399 if 3400 K1 == K2 -> 3401 same_keys1(E1, K1, L1, C1, E2, C2, E2_0, L2, M); 3402 K1 < K2 -> 3403 [?JWRAP(E1, E2_0) | 3404 fun() -> same_loop1(L1, K1, C1, E2_0, L2_0, C2, M) end] 3405 end; 3406same_keys2(E1, K1, L1, C1, F2, C2, E2_0, M) -> 3407 case F2() of 3408 L2 when is_list(L2) -> 3409 same_keys2(E1, K1, L1, C1, L2, C2, E2_0, M); 3410 T2 -> 3411 Cont = fun(_L1b) -> T2 end, 3412 loop_same_keys(E1, K1, L1, C1, [E2_0], Cont, M) 3413 end. 3414 3415same_loop1([], _K1_0, _C1, _E2_0, _L2, _C2, M) -> 3416 end_merge_join([], M); 3417same_loop1([E1 | L1], K1_0, C1, E2_0, L2, C2, M) -> 3418 K1 = element(C1, E1), 3419 if 3420 K1 == K1_0 -> 3421 [?JWRAP(E1, E2_0) | 3422 fun() -> same_loop1(L1, K1_0, C1, E2_0, L2, C2, M) end]; 3423 K1_0 < K1 -> 3424 merge_join2(E1, K1, L1, C1, L2, C2, M) 3425 end; 3426same_loop1(F1, K1_0, C1, E2_0, L2, C2, M) -> 3427 case F1() of 3428 L1 when is_list(L1) -> 3429 same_loop1(L1, K1_0, C1, E2_0, L2, C2, M); 3430 T1 -> 3431 T1 3432 end. 3433 3434%% element(C2, E2_0) == K1, element(C2, E2) == K1_0 3435same_keys1(E1_0, K1_0, []=L1, C1, E2, C2, E2_0, L2, M) -> 3436 [?JWRAP(E1_0, E2_0), ?JWRAP(E1_0, E2) | 3437 fun() -> same_keys(K1_0, E1_0, L1, C1, L2, C2, M) end]; 3438same_keys1(E1_0, K1_0, [E1 | _]=L1, C1, E2, C2, E2_0, L2, M) -> 3439 K1 = element(C1, E1), 3440 if 3441 K1_0 == K1 -> 3442 E2s = [E2, E2_0], 3443 Sz0 = decr_list_size(M#m.max_list, E2s), 3444 same_keys_cache(E1_0, K1_0, L1, C1, L2, C2, E2s, Sz0, M); 3445 K1_0 < K1 -> 3446 [?JWRAP(E1_0, E2_0), ?JWRAP(E1_0, E2) | 3447 fun() -> same_keys(K1_0, E1_0, L1, C1, L2, C2, M) end] 3448 end; 3449same_keys1(E1_0, K1_0, F1, C1, E2, C2, E2_0, L2, M) -> 3450 case F1() of 3451 L1 when is_list(L1) -> 3452 same_keys1(E1_0, K1_0, L1, C1, E2, C2, E2_0, L2, M); 3453 T1 -> 3454 Cont = fun() -> T1 end, 3455 loop_same(E1_0, [E2, E2_0], Cont) 3456 end. 3457 3458%% There is no such element E in L1 such that element(C1, E) == K1. 3459same_keys(_K1, _E1, _L1, _C1, []=Cont, _C2, M) -> 3460 end_merge_join(Cont, M); 3461same_keys(K1, E1, L1, C1, [E2 | L2], C2, M) -> 3462 K2 = element(C2, E2), 3463 if 3464 K1 == K2 -> 3465 [?JWRAP(E1, E2) | 3466 fun() -> same_keys(K1, E1, L1, C1, L2, C2, M) end]; 3467 K1 < K2 -> 3468 merge_join1(E2, K2, L1, C1, L2, C2, M) 3469 end; 3470same_keys(K1, E1, L1, C1, F2, C2, M) -> 3471 case F2() of 3472 L2 when is_list(L2) -> 3473 same_keys(K1, E1, L1, C1, L2, C2, M); 3474 T2 -> 3475 T2 3476 end. 3477 3478%% There are at least two elements in [E1 | L1] that are to be combined 3479%% with the elements in E2s (length(E2s) > 1). This loop covers the case 3480%% when all elements in E2 with key K1 can be kept in RAM. 3481same_keys_cache(E1, K1, L1, C1, [], _C2, E2s, _Sz, M) -> 3482 Cont = fun(_L1b) -> end_merge_join([], M) end, 3483 loop_same_keys(E1, K1, L1, C1, E2s, Cont, M); 3484same_keys_cache(E1, K1, L1, C1, L2, C2, E2s, Sz0, M) when Sz0 < 0 -> 3485 case init_merge_join(M) of 3486 ok -> 3487 Sz = M#m.max_list, 3488 C = fun() -> 3489 same_keys_file(E1, K1, L1, C1, L2, C2, [], Sz, M) 3490 end, 3491 write_same_keys(E1, E2s, M, C); 3492 Error -> 3493 Error 3494 end; 3495same_keys_cache(E1, K1, L1, C1, [E2 | L2], C2, E2s, Sz0, M) -> 3496 K2 = element(C2, E2), 3497 if 3498 K1 == K2 -> 3499 Sz = decr_list_size(Sz0, E2), 3500 same_keys_cache(E1, K1, L1, C1, L2, C2, [E2 | E2s], Sz, M); 3501 K1 < K2 -> 3502 Cont = fun(L1b) -> merge_join1(E2, K2, L1b, C1, L2, C2, M) end, 3503 loop_same_keys(E1, K1, L1, C1, E2s, Cont, M) 3504 end; 3505same_keys_cache(E1, K1, L1, C1, F2, C2, E2s, Sz, M) -> 3506 case F2() of 3507 L2 when is_list(L2) -> 3508 same_keys_cache(E1, K1, L1, C1, L2, C2, E2s, Sz, M); 3509 T2 -> 3510 Cont = fun(_L1b) -> T2 end, 3511 loop_same_keys(E1, K1, L1, C1, E2s, Cont, M) 3512 end. 3513 3514%% E2s holds all elements E2 in L2 such that element(E2, C2) == K1. 3515loop_same_keys(E1, _K1, [], _C1, E2s, _Cont, M) -> 3516 end_merge_join(loop_same(E1, E2s, []), M); 3517loop_same_keys(E1, K1, L1, C1, E2s, Cont, M) -> 3518 loop_same(E1, E2s, fun() -> loop_keys(K1, L1, C1, E2s, Cont, M) end). 3519 3520loop_same(_E1, [], L) -> 3521 L; 3522loop_same(E1, [E2 | E2s], L) -> 3523 loop_same(E1, E2s, [?JWRAP(E1, E2) | L]). 3524 3525loop_keys(K, [E1 | L1]=L1_0, C1, E2s, Cont, M) -> 3526 K1 = element(C1, E1), 3527 if 3528 K1 == K -> 3529 loop_same_keys(E1, K1, L1, C1, E2s, Cont, M); 3530 K1 > K -> 3531 Cont(L1_0) 3532 end; 3533loop_keys(_K, []=L1, _C1, _Es2, Cont, _M) -> 3534 Cont(L1); 3535loop_keys(K, F1, C1, E2s, Cont, M) -> 3536 case F1() of 3537 L1 when is_list(L1) -> 3538 loop_keys(K, L1, C1, E2s, Cont, M); 3539 T1 -> 3540 T1 3541 end. 3542 3543%% This is for the case when a temporary file has to be used. 3544same_keys_file(E1, K1, L1, C1, [], _C2, E2s, _Sz, M) -> 3545 Cont = fun(_L1b) -> end_merge_join([], M) end, 3546 same_keys_file_write(E1, K1, L1, C1, E2s, M, Cont); 3547same_keys_file(E1, K1, L1, C1, L2, C2, E2s, Sz0, M) when Sz0 < 0 -> 3548 Sz = M#m.max_list, 3549 C = fun() -> same_keys_file(E1, K1, L1, C1, L2, C2, [], Sz, M) end, 3550 write_same_keys(E1, E2s, M, C); 3551same_keys_file(E1, K1, L1, C1, [E2 | L2], C2, E2s, Sz0, M) -> 3552 K2 = element(C2, E2), 3553 if 3554 K1 == K2 -> 3555 Sz = decr_list_size(Sz0, E2), 3556 same_keys_file(E1, K1, L1, C1, L2, C2, [E2 | E2s], Sz, M); 3557 K1 < K2 -> 3558 Cont = fun(L1b) -> 3559 %% The temporary file could be truncated here. 3560 merge_join1(E2, K2, L1b, C1, L2, C2, M) 3561 end, 3562 same_keys_file_write(E1, K1, L1, C1, E2s, M, Cont) 3563 end; 3564same_keys_file(E1, K1, L1, C1, F2, C2, E2s, Sz, M) -> 3565 case F2() of 3566 L2 when is_list(L2) -> 3567 same_keys_file(E1, K1, L1, C1, L2, C2, E2s, Sz, M); 3568 T2 -> 3569 Cont = fun(_L1b) -> T2 end, 3570 same_keys_file_write(E1, K1, L1, C1, E2s, M, Cont) 3571 end. 3572 3573same_keys_file_write(E1, K1, L1, C1, E2s, M, Cont) -> 3574 C = fun() -> loop_keys_file(K1, L1, C1, Cont, M) end, 3575 write_same_keys(E1, E2s, M, C). 3576 3577write_same_keys(_E1, [], _M, Cont) -> 3578 Cont(); 3579write_same_keys(E1, Es2, M, Cont) -> 3580 write_same_keys(E1, Es2, M, [], Cont). 3581 3582%% Avoids one (the first) traversal of the temporary file. 3583write_same_keys(_E1, [], M, E2s, Objs) -> 3584 case write_merge_join(M, E2s) of 3585 ok -> Objs; 3586 Error -> Error 3587 end; 3588write_same_keys(E1, [E2 | E2s0], M, E2s, Objs) -> 3589 BE2 = term_to_binary(E2), 3590 write_same_keys(E1, E2s0, M, [BE2 | E2s], [?JWRAP(E1, E2) | Objs]). 3591 3592loop_keys_file(K, [E1 | L1]=L1_0, C1, Cont, M) -> 3593 K1 = element(C1, E1), 3594 if 3595 K1 == K -> 3596 C = fun() -> loop_keys_file(K1, L1, C1, Cont, M) end, 3597 read_merge_join(M, E1, C); 3598 K1 > K -> 3599 Cont(L1_0) 3600 end; 3601loop_keys_file(_K, []=L1, _C1, Cont, _M) -> 3602 Cont(L1); 3603loop_keys_file(K, F1, C1, Cont, M) -> 3604 case F1() of 3605 L1 when is_list(L1) -> 3606 loop_keys_file(K, L1, C1, Cont, M); 3607 T1 -> 3608 T1 3609 end. 3610 3611end_merge_join(Reply, M) -> 3612 end_merge_join(M), 3613 Reply. 3614 3615%% Normally post_funs() cleans up temporary files by calling funs in 3616%% Post. It seems impossible to do that with the temporary file(s) 3617%% used when many objects have the same key--such a file is created 3618%% after the setup when Post is prepared. There seems to be no real 3619%% alternative to using the process dictionary, at least as things 3620%% have been implemented so far. Probably all of Post could have been 3621%% put in the process dictionary... 3622 3623-define(MERGE_JOIN_FILE, '$_qlc_merge_join_tmpfiles_'). 3624 3625init_merge_join(#m{id = MergeId, tmpdir = TmpDir, tmp_usage = TmpUsage}) -> 3626 case tmp_merge_file(MergeId) of 3627 {Fd, FileName} -> 3628 case file:position(Fd, bof) of 3629 {ok, 0} -> 3630 case file:truncate(Fd) of 3631 ok -> 3632 ok; 3633 Error -> 3634 file_error(FileName, Error) 3635 end; 3636 Error -> 3637 file_error(FileName, Error) 3638 end; 3639 none when TmpUsage =:= not_allowed -> 3640 error({tmpdir_usage, joining}); 3641 none -> 3642 maybe_error_logger(TmpUsage, joining), 3643 FName = tmp_filename(TmpDir), 3644 case file:open(FName, [raw, binary, read, write]) of 3645 {ok, Fd} -> 3646 TmpFiles = get(?MERGE_JOIN_FILE), 3647 put(?MERGE_JOIN_FILE, [{MergeId, Fd, FName} | TmpFiles]), 3648 ok; 3649 Error -> 3650 file_error(FName, Error) 3651 end 3652 end. 3653 3654write_merge_join(#m{id = MergeId}, BTerms) -> 3655 {Fd, FileName} = tmp_merge_file(MergeId), 3656 write_binary_terms(BTerms, Fd, FileName). 3657 3658read_merge_join(#m{id = MergeId}, E1, Cont) -> 3659 {Fd, FileName} = tmp_merge_file(MergeId), 3660 case file:position(Fd, bof) of 3661 {ok, 0} -> 3662 Fun = fun([], _) -> 3663 Cont(); 3664 (Ts, C) when is_list(Ts) -> 3665 join_read_terms(E1, Ts, C) 3666 end, 3667 file_loop_read(<<>>, ?CHUNK_SIZE, {Fd, FileName}, Fun); 3668 Error -> 3669 file_error(FileName, Error) 3670 end. 3671 3672join_read_terms(_E1, [], Objs) -> 3673 Objs; 3674join_read_terms(E1, [E2 | E2s], Objs) -> 3675 join_read_terms(E1, E2s, [?JWRAP(E1, E2) | Objs]). 3676 3677end_merge_join(#m{id = MergeId}) -> 3678 case tmp_merge_file(MergeId) of 3679 none -> 3680 ok; 3681 {Fd, FileName} -> 3682 _ = file:close(Fd), 3683 _ = file:delete(FileName), 3684 put(?MERGE_JOIN_FILE, 3685 lists:keydelete(MergeId, 1, get(?MERGE_JOIN_FILE))) 3686 end. 3687 3688end_all_merge_joins() -> 3689 lists:foreach( 3690 fun(Id) -> end_merge_join(#m{id = Id}) end, 3691 [Id || {Id, _Fd, _FileName} <- lists:flatten([get(?MERGE_JOIN_FILE)])]), 3692 erase(?MERGE_JOIN_FILE). 3693 3694merge_join_id() -> 3695 case get(?MERGE_JOIN_FILE) of 3696 undefined -> 3697 put(?MERGE_JOIN_FILE, []); 3698 _ -> 3699 ok 3700 end, 3701 make_ref(). 3702 3703tmp_merge_file(MergeId) -> 3704 TmpFiles = get(?MERGE_JOIN_FILE), 3705 case lists:keyfind(MergeId, 1, TmpFiles) of 3706 {MergeId, Fd, FileName} -> 3707 {Fd, FileName}; 3708 false -> 3709 none 3710 end. 3711 3712decr_list_size(Sz0, E) when is_integer(Sz0) -> 3713 Sz0 - erlang:external_size(E). 3714 3715%%% End of merge join. 3716 3717lookup_join([E1 | L1], C1, LuF, C2, Rev) -> 3718 K1 = element(C1, E1), 3719 case LuF(C2, [K1]) of 3720 [] -> 3721 lookup_join(L1, C1, LuF, C2, Rev); 3722 [E2] when Rev -> 3723 [?JWRAP(E2, E1) | fun() -> lookup_join(L1, C1, LuF, C2, Rev) end]; 3724 [E2] -> 3725 [?JWRAP(E1, E2) | fun() -> lookup_join(L1, C1, LuF, C2, Rev) end]; 3726 E2s when is_list(E2s), Rev -> 3727 [?JWRAP(E2, E1) || E2 <- E2s] ++ 3728 fun() -> lookup_join(L1, C1, LuF, C2, Rev) end; 3729 E2s when is_list(E2s) -> 3730 [?JWRAP(E1, E2) || E2 <- E2s] ++ 3731 fun() -> lookup_join(L1, C1, LuF, C2, Rev) end; 3732 Term -> 3733 Term 3734 end; 3735lookup_join([]=Cont, _C1, _LuF, _C2, _Rev) -> 3736 Cont; 3737lookup_join(F1, C1, LuF, C2, Rev) -> 3738 case F1() of 3739 L1 when is_list(L1) -> 3740 lookup_join(L1, C1, LuF, C2, Rev); 3741 T1 -> 3742 T1 3743 end. 3744 3745maybe_error_logger(allowed, _) -> 3746 ok; 3747maybe_error_logger(Name, Why) -> 3748 [_, _, {?MODULE,maybe_error_logger,_,_} | Stacktrace] = 3749 expand_stacktrace(), 3750 Trimmer = fun(M, _F, _A) -> M =:= erl_eval end, 3751 Formater = fun(Term, I) -> io_lib:print(Term, I, 80, -1) end, 3752 X = erl_error:format_stacktrace(1, Stacktrace, Trimmer, Formater), 3753 error_logger:Name("qlc: temporary file was needed for ~w\n~ts\n", 3754 [Why, lists:flatten(X)]). 3755 3756expand_stacktrace() -> 3757 D = erlang:system_flag(backtrace_depth, 8), 3758 try 3759 %% Compensate for a bug in erlang:system_flag/2: 3760 expand_stacktrace(erlang:max(1, D)) 3761 after 3762 erlang:system_flag(backtrace_depth, D) 3763 end. 3764 3765expand_stacktrace(D) -> 3766 _ = erlang:system_flag(backtrace_depth, D), 3767 {'EXIT', {foo, Stacktrace}} = (catch erlang:error(foo)), 3768 L = lists:takewhile(fun({M,_,_,_}) -> M =/= ?MODULE 3769 end, lists:reverse(Stacktrace)), 3770 if 3771 length(L) < 3 andalso length(Stacktrace) =:= D -> 3772 expand_stacktrace(D + 5); 3773 true -> 3774 Stacktrace 3775 end. 3776 3777write_binary_terms(BTerms, Fd, FileName) -> 3778 case file:write(Fd, size_bin(BTerms, [])) of 3779 ok -> 3780 ok; 3781 Error -> 3782 file_error(FileName, Error) 3783 end. 3784 3785post_funs(L) -> 3786 end_all_merge_joins(), 3787 local_post(L). 3788 3789local_post(L) -> 3790 lists:foreach(fun(undefined) -> ok; 3791 (F) -> catch (F)() 3792 end, L). 3793 3794call(undefined, _Arg, Default, _Post) -> 3795 Default; 3796call(Fun, Arg, _Default, Post) -> 3797 try 3798 Fun(Arg) 3799 catch Class:Reason:Stacktrace -> 3800 post_funs(Post), 3801 erlang:raise(Class, Reason, Stacktrace) 3802 end. 3803 3804grd(undefined, _Arg) -> 3805 false; 3806grd(Fun, Arg) -> 3807 case Fun(Arg) of 3808 true -> 3809 true; 3810 _ -> 3811 false 3812 end. 3813 3814anno0() -> 3815 anno(0). 3816 3817anno1() -> 3818 anno(1). 3819 3820anno(L) -> 3821 erl_anno:new(L). 3822 3823family(L) -> 3824 sofs:to_external(sofs:relation_to_family(sofs:relation(L))). 3825 3826family_union(L) -> 3827 R = sofs:relation(L,[{atom,[atom]}]), 3828 sofs:to_external(sofs:family_union(sofs:relation_to_family(R))). 3829 3830file_error(File, {error, Reason}) -> 3831 error({file_error, File, Reason}). 3832 3833-spec throw_file_error(string(), {'error',atom()}) -> no_return(). 3834 3835throw_file_error(File, {error, Reason}) -> 3836 throw_reason({file_error, File, Reason}). 3837 3838-spec throw_reason(term()) -> no_return(). 3839 3840throw_reason(Reason) -> 3841 throw_error(error(Reason)). 3842 3843-spec throw_error(term()) -> no_return(). 3844 3845throw_error(Error) -> 3846 throw(Error). 3847 3848error(Reason) -> 3849 {error, ?MODULE, Reason}. 3850