1%% 2%% %CopyrightBegin% 3%% 4%% Copyright Ericsson AB 1996-2018. All Rights Reserved. 5%% 6%% Licensed under the Apache License, Version 2.0 (the "License"); 7%% you may not use this file except in compliance with the License. 8%% You may obtain a copy of the License at 9%% 10%% http://www.apache.org/licenses/LICENSE-2.0 11%% 12%% Unless required by applicable law or agreed to in writing, software 13%% distributed under the License is distributed on an "AS IS" BASIS, 14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15%% See the License for the specific language governing permissions and 16%% limitations under the License. 17%% 18%% %CopyrightEnd% 19%% 20-module(dets_SUITE). 21 22%%-define(debug, true). 23 24-ifdef(debug). 25-define(format(S, A), io:format(S, A)). 26-define(config(X,Y), foo). 27-define(t, test_server). 28-define(privdir(_), "./dets_SUITE_priv"). 29-define(datadir(_), "./dets_SUITE_data"). 30-else. 31-include_lib("common_test/include/ct.hrl"). 32-define(format(S, A), ok). 33-define(privdir(Conf), proplists:get_value(priv_dir, Conf)). 34-define(datadir(Conf), proplists:get_value(data_dir, Conf)). 35-endif. 36 37-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 38 init_per_group/2,end_per_group/2, newly_started/1, basic/1, 39 open/1, sets/1, bags/1, duplicate_bags/1, access/1, dirty_mark/1, 40 dirty_mark2/1, bag_next/1, oldbugs/1, 41 truncated_segment_array/1, open_file/1, init_table/1, repair/1, 42 phash/1, fold/1, fixtable/1, match/1, select/1, update_counter/1, 43 badarg/1, cache_sets/1, cache_bags/1, cache_duplicate_bags/1, 44 otp_4208/1, otp_4989/1, many_clients/1, otp_4906/1, otp_5402/1, 45 simultaneous_open/1, insert_new/1, repair_continuation/1, 46 otp_5487/1, otp_6206/1, otp_6359/1, otp_4738/1, otp_7146/1, 47 otp_8070/1, otp_8856/1, otp_8898/1, otp_8899/1, otp_8903/1, 48 otp_8923/1, otp_9282/1, otp_11245/1, otp_11709/1, otp_13229/1, 49 otp_13260/1, otp_13830/1]). 50 51-export([dets_dirty_loop/0]). 52 53-export([histogram/1, sum_histogram/1, ave_histogram/1]). 54 55-export([init_per_testcase/2, end_per_testcase/2]). 56 57%% Internal export. 58-export([client/2]). 59 60-import(lists, 61 [append/1, delete/2, duplicate/2, filter/2, foreach/2, keysearch/3, 62 last/1, map/2, member/2, reverse/1, seq/2, sort/1, usort/1]). 63 64-include_lib("kernel/include/file.hrl"). 65 66-define(DETS_SERVER, dets). 67 68%% HEADSZ taken from dets_v9.erl. 69-define(HEADSZ_v9, (56+28*4+16)). 70-define(NO_KEYS_POS_v9, 36). 71-define(CLOSED_PROPERLY_POS, 8). 72 73-define(NOT_PROPERLY_CLOSED,0). 74-define(CLOSED_PROPERLY,1). 75 76init_per_testcase(_Case, Config) -> 77 Config. 78 79end_per_testcase(_Case, _Config) -> 80 ok. 81 82suite() -> 83 [{ct_hooks,[ts_install_cth]}, 84 {timetrap,{minutes,15}}]. 85 86all() -> 87 [ 88 basic, open, sets, bags, duplicate_bags, newly_started, open_file, 89 init_table, repair, access, oldbugs, 90 truncated_segment_array, dirty_mark, dirty_mark2, bag_next, 91 phash, fold, fixtable, match, select, update_counter, badarg, 92 cache_sets, cache_bags, cache_duplicate_bags, otp_4208, otp_4989, 93 many_clients, otp_4906, otp_5402, simultaneous_open, 94 insert_new, repair_continuation, otp_5487, otp_6206, 95 otp_6359, otp_4738, otp_7146, otp_8070, otp_8856, otp_8898, 96 otp_8899, otp_8903, otp_8923, otp_9282, otp_11245, otp_11709, 97 otp_13229, otp_13260, otp_13830 98 ]. 99 100groups() -> 101 []. 102 103init_per_suite(Config) -> 104 Config. 105 106end_per_suite(_Config) -> 107 ok. 108 109init_per_group(_GroupName, Config) -> 110 Config. 111 112end_per_group(_GroupName, Config) -> 113 Config. 114 115%% OTP-3621 116newly_started(Config) when is_list(Config) -> 117 true = is_alive(), 118 {ok, Node} = test_server:start_node(slave1, slave, []), 119 [] = rpc:call(Node, dets, all, []), 120 test_server:stop_node(Node), 121 ok. 122 123basic(Config) when is_list(Config) -> 124 Tab = dets_basic_test, 125 FName = filename(Tab, Config), 126 127 P0 = pps(), 128 {ok, _} = dets:open_file(Tab,[{file, FName}]), 129 ok = dets:insert(Tab,{mazda,japan}), 130 ok = dets:insert(Tab,{toyota,japan}), 131 ok = dets:insert(Tab,{suzuki,japan}), 132 ok = dets:insert(Tab,{honda,japan}), 133 ok = dets:insert(Tab,{renault,france}), 134 ok = dets:insert(Tab,{citroen,france}), 135 ok = dets:insert(Tab,{opel,germany}), 136 ok = dets:insert(Tab,{saab,sweden}), 137 ok = dets:insert(Tab,{volvo,sweden}), 138 [{opel,germany}] = dets:lookup(Tab,opel), 139 Japs = dets:traverse(Tab, fun(Obj) -> 140 case Obj of 141 {_, japan} -> {continue, Obj}; 142 _ -> continue 143 end 144 end), 145 4 = length(Japs), 146 ok = dets:close(Tab), 147 file:delete(FName), 148 check_pps(P0), 149 ok. 150 151 152open(Config) when is_list(Config) -> 153 %% Running this test twice means that the Dets server is restarted 154 %% twice. dets_sup specifies a maximum of 4 restarts in an hour. 155 %% If this becomes a problem, one should consider running this 156 %% test on a slave node. 157 158 {Sets, Bags, Dups} = args(Config), 159 160 All = Sets ++ Bags ++ Dups, 161 delete_files(All), 162 163 Data = make_data(1), 164 165 P0 = pps(), 166 Tabs = open_files(1, All), 167 initialize(Tabs, Data), 168 check(Tabs, Data), 169 170 foreach(fun(Tab) -> ok = dets:close(Tab) end, Tabs), 171 %% Now reopen the files 172 ?format("Reopening closed files \n", []), 173 Tabs = open_files(1, All), 174 ?format("Checking contents of reopened files \n", []), 175 check(Tabs, Data), 176 %% crash the dets server 177 178 ?format("Crashing dets server \n", []), 179 process_flag(trap_exit, true), 180 Procs = [whereis(?DETS_SERVER) | [dets:info(Tab, pid) || Tab <- Tabs]], 181 foreach(fun(Pid) -> exit(Pid, kill) end, Procs), 182 timer:sleep(100), 183 c:flush(), %% flush all the EXIT sigs 184 timer:sleep(200), 185 186 %% Now reopen the files again 187 ?format("Reopening crashed files \n", []), 188 open_files(1, All), 189 ?format("Checking contents of repaired files \n", []), 190 check(Tabs, Data), 191 192 close_all(Tabs), 193 delete_files(All), 194 195 {Ports0, Procs0} = P0, 196 Test = fun() -> 197 P1 = pps(), 198 {Ports1, Procs1} = P1, 199 show("Old port", Ports0 -- Ports1), 200 show("New port", Ports1 -- Ports0), 201 show("Old procs", Procs0 -- Procs1), 202 show("New procs", Procs1 -- Procs0), 203 io:format("Remaining Dets-pids (should be nil): ~p~n", 204 [find_dets_pids()]), 205 true = Ports1 =:= Ports0, 206 %% The dets_server process has been restarted: 207 [_] = Procs0 -- Procs1, 208 [_] = Procs1 -- Procs0, 209 ok 210 end, 211 case catch Test() of 212 ok -> ok; 213 _ -> 214 timer:sleep(500), 215 ok = Test() 216 end. 217 218check(Tabs, Data) -> 219 foreach(fun(Tab) -> 220 Kp = dets:info(Tab, keypos), 221 ?format("checking ~p~n", [Tab]), 222 foreach(fun(Item) -> 223 case dets:lookup(Tab, k(Kp,Item)) of 224 [Item] -> ok; 225 _Other -> bad(Tab,Item) 226 end 227 end, Data) 228 end, Tabs), 229 ok. 230 231k(Kp, Obj) -> element(Kp, Obj). 232 233bad(_Tab, _Item) -> 234 ?format("Can't find item ~p in ~p ~n", [_Item, _Tab]), 235 exit(badtab). 236 237%% Perform traversal and match testing on set type dets tables. 238sets(Config) when is_list(Config) -> 239 {Sets, _, _} = args(Config), 240 241 Data = make_data(1), 242 delete_files(Sets), 243 P0 = pps(), 244 Tabs = open_files(1, Sets), 245 Bigger = [{17,q,w,w}, {48,q,w,w,w,w,w,w}], % 48 requires a bigger buddy 246 initialize(Tabs, Data++Bigger++Data), % overwrite 247 Len = length(Data), 248 foreach(fun(Tab) -> trav_test(Data, Len, Tab) end, Tabs), 249 size_test(Len, Tabs), 250 no_keys_test(Tabs), 251 foreach(fun(Tab) -> del_test(Tab) end, Tabs), 252 initialize(Tabs, Data), 253 foreach(fun(Tab) -> del_obj_test(Tab) end, Tabs), 254 initialize(Tabs, Data), 255 foreach(fun(Tab) -> 256 Len = dets:info(Tab, size) end, 257 Tabs), 258 foreach(fun(Tab) -> match_test(Data, Tab) end, Tabs), 259 foreach(fun(Tab) -> match_del_test(Tab) end, Tabs), 260 261 close_all(Tabs), 262 delete_files(Sets), 263 check_pps(P0), 264 ok. 265 266%% Perform traversal and match testing on bag type dets tables. 267bags(Config) when is_list(Config) -> 268 {_, Bags, _} = args(Config), 269 Data = make_data(1, bag), %% gives twice as many objects 270 delete_files(Bags), 271 P0 = pps(), 272 Tabs = open_files(1, Bags), 273 initialize(Tabs, Data++Data), 274 Len = length(Data), 275 foreach(fun(Tab) -> trav_test(Data, Len, Tab) end, Tabs), 276 size_test(Len, Tabs), 277 no_keys_test(Tabs), 278 foreach(fun(Tab) -> del_test(Tab) end, Tabs), 279 initialize(Tabs, Data), 280 foreach(fun(Tab) -> del_obj_test(Tab) end, Tabs), 281 initialize(Tabs, Data), 282 foreach(fun(Tab) -> 283 Len = dets:info(Tab, size) end, 284 Tabs), 285 foreach(fun(Tab) -> match_test(Data, Tab) end, Tabs), 286 foreach(fun(Tab) -> match_del_test(Tab) end, Tabs), 287 close_all(Tabs), 288 delete_files(Bags), 289 check_pps(P0), 290 ok. 291 292 293%% Perform traversal and match testing on duplicate_bag type dets tables. 294duplicate_bags(Config) when is_list(Config) -> 295 {_, _, Dups} = args(Config), 296 Data = make_data(1, duplicate_bag), %% gives twice as many objects 297 delete_files(Dups), 298 P0 = pps(), 299 Tabs = open_files(1, Dups), 300 initialize(Tabs, Data), 301 Len = length(Data), 302 foreach(fun(Tab) -> trav_test(Data, Len, Tab) end, Tabs), 303 size_test(Len, Tabs), 304 no_keys_test(Tabs), 305 foreach(fun(Tab) -> del_test(Tab) end, Tabs), 306 initialize(Tabs, Data), 307 foreach(fun(Tab) -> del_obj_test(Tab) end, Tabs), 308 initialize(Tabs, Data), 309 foreach(fun(Tab) -> 310 Len = dets:info(Tab, size) end, 311 Tabs), 312 foreach(fun(Tab) -> match_test(Data, Tab) end, Tabs), 313 foreach(fun(Tab) -> match_del_test(Tab) end, Tabs), 314 close_all(Tabs), 315 delete_files(Dups), 316 check_pps(P0), 317 ok. 318 319 320access(Config) when is_list(Config) -> 321 Args_acc = [[{ram_file, true}, {access, read}], 322 [{access, read}]], 323 Args = [[{ram_file, true}], 324 []], 325 326 {Args_acc_1, _, _} = zip_filename(Args_acc, [], [], Config), 327 delete_files(Args_acc_1), 328 {Args_1, _, _} = zip_filename(Args, [], [], Config), 329 330 P0 = pps(), 331 {error, {file_error,_,enoent}} = dets:open_file('1', hd(Args_acc_1)), 332 333 Tabs = open_files(1, Args_1), 334 close_all(Tabs), 335 Tabs = open_files(1, Args_acc_1), 336 337 foreach(fun(Tab) -> 338 {error, {access_mode,_}} = dets:insert(Tab, {1,2}), 339 [] = dets:lookup(Tab, 11), 340 '$end_of_table' = dets:first(Tab), 341 {error, {access_mode,_}} = dets:delete(Tab, 22) 342 end, Tabs), 343 close_all(Tabs), 344 delete_files(Args_acc_1), 345 check_pps(P0), 346 ok. 347 348 349%% Test that the table is not marked dirty if not written. 350dirty_mark(Config) when is_list(Config) -> 351 true = is_alive(), 352 Tab = dets_dirty_mark_test, 353 FName = filename(Tab, Config), 354 P0 = pps(), 355 dets:open_file(Tab,[{file, FName}]), 356 dets:insert(Tab,{mazda,japan}), 357 dets:insert(Tab,{toyota,japan}), 358 dets:insert(Tab,{suzuki,japan}), 359 dets:insert(Tab,{honda,japan}), 360 dets:insert(Tab,{renault,france}), 361 dets:insert(Tab,{citroen,france}), 362 dets:insert(Tab,{opel,germany}), 363 dets:insert(Tab,{saab,sweden}), 364 dets:insert(Tab,{volvo,sweden}), 365 [{opel,germany}] = dets:lookup(Tab,opel), 366 ok = dets:close(Tab), 367 Call = fun(P,A) -> 368 P ! {self(), A}, 369 receive 370 {P, Ans} -> 371 Ans 372 after 5000 -> 373 exit(other_process_dead) 374 end 375 end, 376 {ok, Node} = test_server:start_node(dets_dirty_mark, 377 slave, 378 [{linked, false}, 379 {args, "-pa " ++ 380 filename:dirname 381 (code:which(?MODULE))}]), 382 ok = ensure_node(20, Node), 383 %% io:format("~p~n",[rpc:call(Node, code, get_path, [])]), 384 %% io:format("~p~n",[rpc:call(Node, file, get_cwd, [])]), 385 %% io:format("~p~n",[Config]), 386 Pid = rpc:call(Node,erlang, spawn, 387 [?MODULE, dets_dirty_loop, []]), 388 {ok, Tab} = Call(Pid, [open, Tab, [{file, FName}]]), 389 [{opel,germany}] = Call(Pid, [read,Tab,opel]), 390 test_server:stop_node(Node), 391 {ok, Tab} = dets:open_file(Tab,[{file, FName}, 392 {repair,false}]), 393 ok = dets:close(Tab), 394 file:delete(FName), 395 check_pps(P0), 396 ok. 397 398%% Test that the table is flushed when auto_save is in effect. 399dirty_mark2(Config) when is_list(Config) -> 400 true = is_alive(), 401 Tab = dets_dirty_mark2_test, 402 FName = filename(Tab, Config), 403 P0 = pps(), 404 dets:open_file(Tab,[{file, FName}]), 405 dets:insert(Tab,{toyota,japan}), 406 dets:insert(Tab,{suzuki,japan}), 407 dets:insert(Tab,{honda,japan}), 408 dets:insert(Tab,{renault,france}), 409 dets:insert(Tab,{citroen,france}), 410 dets:insert(Tab,{opel,germany}), 411 dets:insert(Tab,{saab,sweden}), 412 dets:insert(Tab,{volvo,sweden}), 413 [{opel,germany}] = dets:lookup(Tab,opel), 414 ok = dets:close(Tab), 415 Call = fun(P,A) -> 416 P ! {self(), A}, 417 receive 418 {P, Ans} -> 419 Ans 420 after 5000 -> 421 exit(other_process_dead) 422 end 423 end, 424 {ok, Node} = test_server:start_node(dets_dirty_mark2, 425 slave, 426 [{linked, false}, 427 {args, "-pa " ++ 428 filename:dirname 429 (code:which(?MODULE))}]), 430 ok = ensure_node(20, Node), 431 Pid = rpc:call(Node,erlang, spawn, 432 [?MODULE, dets_dirty_loop, []]), 433 {ok, Tab} = Call(Pid, [open, Tab, [{file, FName},{auto_save,1000}]]), 434 ok = Call(Pid, [write,Tab,{mazda,japan}]), 435 timer:sleep(2100), 436 %% Read something, just to give auto save time to finish. 437 [{opel,germany}] = Call(Pid, [read,Tab,opel]), 438 test_server:stop_node(Node), 439 {ok, Tab} = dets:open_file(Tab, [{file, FName}, {repair,false}]), 440 ok = dets:close(Tab), 441 file:delete(FName), 442 check_pps(P0), 443 ok. 444 445dets_dirty_loop() -> 446 receive 447 {From, [open, Name, Args]} -> 448 Ret = dets:open_file(Name, Args), 449 From ! {self(), Ret}, 450 dets_dirty_loop(); 451 {From, [read, Name, Key]} -> 452 Ret = dets:lookup(Name, Key), 453 From ! {self(), Ret}, 454 dets_dirty_loop(); 455 {From, [write, Name, Value]} -> 456 Ret = dets:insert(Name, Value), 457 From ! {self(), Ret}, 458 dets_dirty_loop(); 459 {From, [close, Name]} -> 460 Ret = dets:close(Name), 461 From ! {self(), Ret}, 462 dets_dirty_loop() 463 end. 464 465 466%% Check that bags and next work as expected. 467bag_next(Config) when is_list(Config) -> 468 Tab = dets_bag_next_test, 469 FName = filename(Tab, Config), 470 471 %% first and next crash upon error 472 dets:open_file(Tab,[{file, FName}, {type, bag}]), 473 ok = dets:insert(Tab, [{1,1},{2,2},{3,3},{4,4}]), 474 FirstKey = dets:first(Tab), 475 NextKey = dets:next(Tab, FirstKey), 476 [FirstObj | _] = dets:lookup(Tab, FirstKey), 477 [NextObj | _] = dets:lookup(Tab, NextKey), 478 {ok, FirstPos} = dets:where(Tab, FirstObj), 479 {ok, NextPos} = dets:where(Tab, NextObj), 480 crash(FName, NextPos+12), 481 {'EXIT',BadObject1} = (catch dets:next(Tab, FirstKey)), 482 bad_object(BadObject1, FName), 483 crash(FName, FirstPos+12), 484 {'EXIT',BadObject2} = (catch dets:first(Tab)), 485 bad_object(BadObject2, FName), 486 dets:close(Tab), 487 file:delete(FName), 488 489 P0 = pps(), 490 dets:open_file(Tab,[{file, FName}, {type, bag}]), 491 dets:insert(Tab,{698,hopp}), 492 dets:insert(Tab,{186,hopp}), 493 dets:insert(Tab,{hej,hopp}), 494 dets:insert(Tab,{186,plopp}), 495 Loop = fun(N, Last, Self) -> 496 case N of 497 0 -> 498 exit({unterminated_first_next_sequence, N, Last}); 499 _ -> 500 case Last of 501 '$end_of_table' -> 502 ok; 503 _ -> 504 Self(N-1, dets:next(Tab,Last), Self) 505 end 506 end 507 end, 508 ok = Loop(4,dets:first(Tab),Loop), 509 dets:close(Tab), 510 file:delete(FName), 511 check_pps(P0), 512 ok. 513 514oldbugs(Config) when is_list(Config) -> 515 FName = filename(dets_suite_oldbugs_test, Config), 516 P0 = pps(), 517 {ok, ob} = dets:open_file(ob, [{type, bag}, {file, FName}]), 518 ok = dets:insert(ob, {1, 2}), 519 ok = dets:insert(ob, {1,3}), 520 ok = dets:insert(ob, {1, 2}), 521 2 = dets:info(ob, size), %% assertion 522 ok = dets:close(ob), 523 file:delete(FName), 524 check_pps(P0), 525 ok. 526 527%% Test that a file where the segment array has been truncated 528%% is possible to repair. 529truncated_segment_array(Config) when is_list(Config) -> 530 TabRef = dets_suite_truncated_segment_array_test, 531 Fname = filename(TabRef, Config), 532 %% Create file that needs to be repaired 533 file:delete(Fname), 534 P0 = pps(), 535 {ok, TabRef} = dets:open_file(TabRef, [{file, Fname}]), 536 ok = dets:close(TabRef), 537 538 %% Truncate the file 539 HeadSize = headsz(), 540 truncate(Fname, HeadSize + 10), 541 542 %% Open the truncated file 543 io:format("Expect repair:~n"), 544 {ok, TabRef} = dets:open_file(TabRef, 545 [{file, Fname}, {repair, true}]), 546 ok = dets:close(TabRef), 547 file:delete(Fname), 548 check_pps(P0), 549 ok. 550 551%% Test open_file/1. 552open_file(Config) when is_list(Config) -> 553 T = open_v9, 554 Fname = filename(T, Config), 555 {ok, _} = dets:open_file(T, [{file,Fname}]), 556 9 = dets:info(T, version), % Backwards compatibility. 557 true = [self()] =:= dets:info(T, users), 558 {ok, _} = dets:open_file(T, [{file,Fname}]), 559 true = [self(),self()] =:= dets:info(T, users), 560 ok = dets:close(T), 561 true = [self()] =:= dets:info(T, users), 562 ok = dets:close(T), 563 undefined = ets:info(T, users), 564 file:delete(Fname), 565 566 open_1(Config). 567 568open_1(Config) -> 569 TabRef = open_file_1_test, 570 Fname = filename(TabRef, Config), 571 file:delete(Fname), 572 573 P0 = pps(), 574 {error,{file_error,Fname,enoent}} = dets:open_file(Fname), 575 576 ok = file:write_file(Fname, duplicate(100,65)), 577 {error,{not_a_dets_file,Fname}} = dets:open_file(Fname), 578 file:delete(Fname), 579 580 HeadSize = headsz(), 581 {ok, TabRef} = dets:open_file(TabRef, [{file, Fname}]), 582 ok = dets:close(TabRef), 583 truncate(Fname, HeadSize + 10), 584 true = dets:is_dets_file(Fname), 585 io:format("Expect repair:~n"), 586 {ok, Ref} = dets:open_file(Fname), % repairing 587 ok = dets:close(Ref), 588 file:delete(Fname), 589 590 %% truncated file header, invalid type 591 {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]), 592 ok = ins(TabRef, 3000), 593 ok = dets:close(TabRef), 594 TypePos = 12, 595 crash(Fname, TypePos), 596 {error, {invalid_type_code,Fname}} = dets:open_file(Fname), 597 truncate(Fname, HeadSize - 10), 598 {error,{not_a_dets_file,Fname}} = dets:open_file(Fname), 599 {error,{not_a_dets_file,Fname}} = 600 dets:open_file(TabRef, [{file,Fname}]), 601 file:delete(Fname), 602 603 {error,{file_error,{foo,bar},_}} = dets:is_dets_file({foo,bar}), 604 check_pps(P0), 605 ok. 606 607%% Test initialize_table/2 and from_ets/2. 608init_table(Config) when is_list(Config) -> 609 %% Objects are returned in "time order". 610 T = init_table_v9, 611 Fname = filename(T, Config), 612 file:delete(Fname), 613 L = [{1,a},{2,b},{1,c},{2,c},{1,c},{2,a},{1,b}], 614 Input = init([L]), 615 {ok, _} = dets:open_file(T, [{file,Fname},{type,duplicate_bag}]), 616 ok = dets:init_table(T, Input), 617 [{1,a},{1,c},{1,c},{1,b}] = dets:lookup(T, 1), 618 [{2,b},{2,c},{2,a}] = dets:lookup(T, 2), 619 ok = dets:close(T), 620 file:delete(Fname), 621 622 init_table_1(Config), 623 fast_init_table(Config). 624 625init_table_1(Config) -> 626 TabRef = init_table_test, 627 Fname = filename(TabRef, Config), 628 file:delete(Fname), 629 P0 = pps(), 630 631 Args = [{file,Fname},{auto_save,120000}], 632 {ok, _} = dets:open_file(TabRef, Args), 633 {'EXIT', _} = 634 (catch dets:init_table(TabRef, fun(foo) -> bar end)), 635 dets:close(TabRef), 636 {ok, _} = dets:open_file(TabRef, Args), 637 {'EXIT', _} = (catch dets:init_table(TabRef, fun() -> foo end)), 638 dets:close(TabRef), 639 {ok, _} = dets:open_file(TabRef, Args), 640 {'EXIT', {badarg, _}} = (catch dets:init_table(TabRef, nofun)), 641 {'EXIT', {badarg, _}} = 642 (catch dets:init_table(TabRef, fun(_X) -> end_of_input end, 643 [{foo,bar}])), 644 dets:close(TabRef), 645 {ok, _} = dets:open_file(TabRef, Args), 646 away = (catch dets:init_table(TabRef, fun(_) -> throw(away) end)), 647 dets:close(TabRef), 648 {ok, _} = dets:open_file(TabRef, Args), 649 {error, {init_fun, fopp}} = 650 dets:init_table(TabRef, fun(read) -> fopp end), 651 dets:close(TabRef), 652 653 {ok, _} = dets:open_file(TabRef, Args), 654 dets:safe_fixtable(TabRef, true), 655 {error, {fixed_table, TabRef}} = dets:init_table(TabRef, init([])), 656 dets:safe_fixtable(TabRef, false), 657 ET = ets:new(foo,[]), 658 ok = dets:from_ets(TabRef, ET), 659 [] = get_all_objects(TabRef), 660 [] = get_all_objects_fast(TabRef), 661 true = ets:insert(ET, {1,a}), 662 true = ets:insert(ET, {2,b}), 663 ok = dets:from_ets(TabRef, ET), 664 [{1,a},{2,b}] = sort(get_all_objects(TabRef)), 665 [{1,a},{2,b}] = sort(get_all_objects_fast(TabRef)), 666 true = ets:delete(ET), 667 120000 = dets:info(TabRef, auto_save), 668 ok = dets:close(TabRef), 669 670 {ok, _} = dets:open_file(TabRef, [{access,read} | Args]), 671 {error, {access_mode, Fname}} = dets:init_table(TabRef, init([])), 672 ok = dets:close(TabRef), 673 674 {ok, _} = dets:open_file(TabRef, Args), 675 {error, invalid_objects_list} = 676 (catch dets:init_table(TabRef, init([[{1,2},bad,{3,4}]]))), 677 _ = dets:close(TabRef), 678 file:delete(Fname), 679 680 L1 = [[{1,a},{2,b}],[],[{3,c}],[{4,d}],[]], 681 bulk_init(L1, set, 4, Config), 682 L2 = [[{1,a},{2,b}],[],[{2,q},{3,c}],[{4,d}],[{4,e},{2,q}]], 683 bulk_init(L2, set, 4, Config), 684 bulk_init(L2, bag, 6, Config), 685 bulk_init(L2, duplicate_bag, 7, Config), 686 bulk_init(L1, set, 4, 512, Config), 687 bulk_init([], set, 0, 10000, Config), 688 file:delete(Fname), 689 690 %% Initiate a file that contains a lot of objects. 691 {ok, _} = dets:open_file(TabRef, [{min_no_slots,10000} | Args]), 692 ok = ins(TabRef, 6000), 693 Fun = init_fun(0, 10000), 694 ok = dets:init_table(TabRef, Fun,{format,term}), 695 All = sort(get_all_objects(TabRef)), 696 FAll = get_all_objects_fast(TabRef), 697 true = All =:= sort(FAll), 698 true = length(All) =:= 10000, 699 ok = dets:close(TabRef), 700 file:delete(Fname), 701 702 {ok, _} = dets:open_file(TabRef, [{min_no_slots,4000} | Args]), 703 ok = ins(TabRef, 6000), 704 FileSize1 = dets:info(TabRef, file_size), 705 Fun2 = init_fun(0, 4000), 706 ok = dets:init_table(TabRef, Fun2), 707 FileSize2 = dets:info(TabRef, file_size), 708 ok = dets:close(TabRef), 709 true = FileSize1 > FileSize2, 710 file:delete(Fname), 711 712 check_pps(P0), 713 ok. 714 715bulk_init(Ls, Type, N, Config) -> 716 bulk_init(Ls, Type, N, 256, Config). 717 718bulk_init(Ls, Type, N, Est, Config) -> 719 T = init_table_test, 720 Fname = filename(T, Config), 721 file:delete(Fname), 722 Input = init(Ls), 723 Args = [{ram_file,false}, {type,Type},{keypos,1},{file,Fname}, 724 {estimated_no_objects, Est}], 725 {ok, T} = dets:open_file(T, Args), 726 ok = dets:init_table(T, Input), 727 All = sort(get_all_objects(T)), 728 FAll = get_all_objects_fast(T), 729 true = All =:= sort(FAll), 730 true = length(All) =:= N, 731 true = dets:info(T, size) =:= N, 732 ok = dets:close(T), 733 734 {ok, T} = dets:open_file(T, Args), 735 All2 = sort(get_all_objects(T)), 736 FAll2 = get_all_objects_fast(T), 737 true = All =:= All2, 738 true = All =:= sort(FAll2), 739 ok = dets:close(T), 740 file:delete(Fname). 741 742init(L) -> 743 fun(close) -> 744 ok; 745 (read) when [] =:= L -> 746 end_of_input; 747 (read) -> 748 [E | Es] = L, 749 {E, init(Es)} 750 end. 751 752init_fun(I, N) -> 753 fun(read) when I =:= N -> 754 end_of_input; 755 (read) -> 756 {NewN, Items} = items(I, N, 1000, []), 757 {Items, init_fun(NewN, N)}; 758 (close) -> 759 ignored 760 end. 761 762fast_init_table(Config) -> 763 TabRef = init_table_test, 764 Fname = filename(TabRef, Config), 765 file:delete(Fname), 766 P0 = pps(), 767 768 Args = [{file,Fname},{auto_save,120000}], 769 770 Source = init_table_test_source, 771 SourceFname = filename(Source, Config), 772 file:delete(SourceFname), 773 SourceArgs = [{file,SourceFname},{auto_save,120000}], 774 775 {ok, Source} = dets:open_file(Source, SourceArgs), 776 777 {ok, _} = dets:open_file(TabRef, Args), 778 {'EXIT', _} = 779 (catch dets:init_table(TabRef, fun(foo) -> bar end, {format,bchunk})), 780 dets:close(TabRef), 781 file:delete(Fname), 782 {ok, _} = dets:open_file(TabRef, Args), 783 {'EXIT', _} = (catch dets:init_table(TabRef, fun() -> foo end, 784 {format,bchunk})), 785 dets:close(TabRef), 786 file:delete(Fname), 787 {ok, _} = dets:open_file(TabRef, Args), 788 {'EXIT', {badarg, _}} = 789 (catch dets:init_table(TabRef, nofun, {format,bchunk})), 790 dets:close(TabRef), 791 {ok, _} = dets:open_file(TabRef, Args), 792 away = (catch dets:init_table(TabRef, fun(_) -> throw(away) end, 793 {format,bchunk})), 794 dets:close(TabRef), 795 file:delete(Fname), 796 {ok, _} = dets:open_file(TabRef, Args), 797 {error, {init_fun, fopp}} = 798 dets:init_table(TabRef, fun(read) -> fopp end, {format,bchunk}), 799 dets:close(TabRef), 800 file:delete(Fname), 801 {ok, _} = dets:open_file(TabRef, Args), 802 dets:safe_fixtable(TabRef, true), 803 {error, {fixed_table, TabRef}} = 804 dets:init_table(TabRef, init([]), {format,bchunk}), 805 dets:safe_fixtable(TabRef, false), 806 ok = dets:close(TabRef), 807 808 {ok, _} = dets:open_file(TabRef, [{access,read} | Args]), 809 {error, {access_mode, Fname}} = 810 dets:init_table(TabRef, init([]), {format,bchunk}), 811 ok = dets:close(TabRef), 812 813 {ok, _} = dets:open_file(TabRef, Args), 814 {error, {init_fun,{1,2}}} = 815 dets:init_table(TabRef, init([[{1,2},bad,{3,4}]]), {format,bchunk}), 816 _ = dets:close(TabRef), 817 file:delete(Fname), 818 819 {ok, _} = dets:open_file(TabRef, Args), 820 {error, {init_fun, end_of_input}} = 821 dets:init_table(TabRef, init([]),{format,bchunk}), 822 _ = dets:close(TabRef), 823 file:delete(Fname), 824 825 {ok, _} = dets:open_file(TabRef, Args), 826 {'EXIT', {badarg, _}} = 827 (catch dets:init_table(TabRef, init([]),{format,foppla})), 828 _ = dets:close(TabRef), 829 file:delete(Fname), 830 831 {ok, _} = dets:open_file(TabRef, Args), 832 ok = ins(TabRef, 100), 833 834 [BParms | Objs] = collect_bchunk(TabRef, init_bchunk(TabRef)), 835 Parms = binary_to_term(BParms), 836 {error, {init_fun, <<"foobar">>}} = 837 dets:init_table(TabRef, init([[<<"foobar">>]]),{format,bchunk}), 838 _ = dets:close(TabRef), 839 file:delete(Fname), 840 841 {ok, _} = dets:open_file(TabRef, Args), 842 Parms1 = setelement(1, Parms, foobar), 843 BParms1 = term_to_binary(Parms1), 844 {error, {init_fun, BParms1}} = 845 dets:init_table(TabRef, init([[BParms1 | Objs]]),{format,bchunk}), 846 _ = dets:close(TabRef), 847 file:delete(Fname), 848 849 {ok, _} = dets:open_file(TabRef, Args), 850 [{Sz1,No1} | NoColls17] = element(tuple_size(Parms), Parms), 851 Parms2 = setelement(tuple_size(Parms), Parms, [{Sz1,No1+1} | NoColls17]), 852 BParms2 = term_to_binary(Parms2), 853 {error, invalid_objects_list} = 854 dets:init_table(TabRef, init([[BParms2 | Objs]]),{format,bchunk}), 855 _ = dets:close(TabRef), 856 file:delete(Fname), 857 858 {ok, _} = dets:open_file(TabRef, Args), 859 [{LSize1,Slot1,Obj1} | ObjsRest] = Objs, 860 861 BadSize = byte_size(Obj1)-1, 862 <<BadSizeObj:BadSize/binary,_:1/binary>> = Obj1, 863 BadObjs = [{LSize1,Slot1,BadSizeObj} | ObjsRest], 864 {error, invalid_objects_list} = 865 dets:init_table(TabRef, init([[BParms | BadObjs]]),{format,bchunk}), 866 _ = dets:close(TabRef), 867 file:delete(Fname), 868 869 {ok, _} = dets:open_file(TabRef, Args), 870 <<Size:32,BigObj0/binary>> = list_to_binary(lists:duplicate(16,Obj1)), 871 BigObj = <<(Size*16):32,BigObj0/binary>>, 872 BadColl = [BParms, {LSize1+4,Slot1,BigObj} | ObjsRest], 873 {error, invalid_objects_list} = 874 dets:init_table(TabRef, init([BadColl]),{format,bchunk}), 875 _ = dets:close(TabRef), 876 file:delete(Fname), 877 878 {ok, _} = dets:open_file(TabRef, Args), 879 BadObj = <<"foobar">>, 880 {error, invalid_objects_list} = 881 dets:init_table(TabRef, init([[BParms, BadObj]]),{format,bchunk}), 882 _ = dets:close(TabRef), 883 file:delete(Fname), 884 885 {ok, _} = dets:open_file(TabRef, [{type,bag} | Args]), 886 {error, {init_fun, _}} = 887 dets:init_table(TabRef, init([[BParms]]),{format,bchunk}), 888 _ = dets:close(TabRef), 889 file:delete(Fname), 890 891 ok = dets:close(Source), 892 file:delete(SourceFname), 893 894 L1 = [{1,a},{2,b},{3,c},{4,d}], 895 fast_bulk_init(L1, set, 4, 4, Config), 896 L2 = [{1,a},{2,b},{2,q},{3,c},{4,d},{4,e},{2,q}], 897 fast_bulk_init(L2, set, 4, 4, Config), 898 fast_bulk_init(L2, bag, 6, 4, Config), 899 fast_bulk_init(L2, duplicate_bag, 7, 4, Config), 900 fast_bulk_init(L1, set, 4, 4, 512, Config), 901 fast_bulk_init([], set, 0, 0, 10000, Config), 902 file:delete(Fname), 903 904 %% Initiate a file that contains a lot of objects. 905 {ok, _} = dets:open_file(Source, [{min_no_slots,10000} | SourceArgs]), 906 Fun1 = init_fun(0, 10000), 907 ok = dets:init_table(Source, Fun1, {format,term}), 908 909 {ok, _} = dets:open_file(TabRef, [{min_no_slots,10000} | Args]), 910 ok = ins(TabRef, 6000), 911 Fun2 = init_bchunk(Source), 912 true = 913 dets:is_compatible_bchunk_format(TabRef, 914 dets:info(Source, bchunk_format)), 915 false = dets:is_compatible_bchunk_format(TabRef, <<"foobar">>), 916 ok = dets:init_table(TabRef, Fun2, {format, bchunk}), 917 ok = dets:close(Source), 918 file:delete(SourceFname), 919 All = sort(get_all_objects(TabRef)), 920 FAll = get_all_objects_fast(TabRef), 921 true = All =:= sort(FAll), 922 true = length(All) =:= 10000, 923 ok = dets:close(TabRef), 924 file:delete(Fname), 925 926 %% Initiate inserts fewer objects than the table contains. 927 {ok, _} = dets:open_file(Source, [{min_no_slots,1000} | SourceArgs]), 928 ok = ins(Source, 4000), 929 930 {ok, _} = dets:open_file(TabRef, [{min_no_slots,1000} | Args]), 931 ok = ins(TabRef, 6000), 932 FileSize1 = dets:info(TabRef, file_size), 933 Fun4 = init_bchunk(Source), 934 ok = dets:init_table(TabRef, Fun4, {format, bchunk}), 935 ok = dets:close(Source), 936 file:delete(SourceFname), 937 FileSize2 = dets:info(TabRef, file_size), 938 All_2 = sort(get_all_objects(TabRef)), 939 FAll_2 = get_all_objects_fast(TabRef), 940 true = All_2 =:= sort(FAll_2), 941 true = length(All_2) =:= 4000, 942 ok = dets:close(TabRef), 943 true = FileSize1 > FileSize2, 944 945 %% Bchunk and fixed table. 946 {ok, _} = dets:open_file(TabRef, Args), 947 NoItems = dets:info(TabRef, no_objects), 948 AllObjects1 = sort(get_all_objects_fast(TabRef)), 949 dets:safe_fixtable(TabRef, true), 950 true = dets:info(TabRef, fixed), 951 Cont1 = init_bchunk(TabRef), 952 NoDel = 953 dets:select_delete(TabRef, [{{'_',{item,'_','_'}},[],[true]}]), 954 true = (NoDel > 0), 955 AllObjects2 = sort(get_all_objects_fast(TabRef)), 956 true = dets:info(TabRef, fixed), 957 Cont2 = init_bchunk(TabRef), 958 NoItems2 = dets:info(TabRef, no_objects), 959 true = (NoItems =:= NoItems2 + NoDel), 960 NoDel2 = dets:select_delete(TabRef, [{'_',[],[true]}]), 961 true = (NoDel2 > 0), 962 AllObjects3 = sort(get_all_objects_fast(TabRef)), 963 NoItems3 = dets:info(TabRef, no_objects), 964 true = (NoItems3 =:= 0), 965 true = dets:info(TabRef, fixed), 966 true = (NoItems2 =:= NoItems3 + NoDel2), 967 Cont3 = init_bchunk(TabRef), 968 969 BinColl1 = collect_bchunk(TabRef, Cont1), 970 BinColl2 = collect_bchunk(TabRef, Cont2), 971 BinColl3 = collect_bchunk(TabRef, Cont3), 972 dets:safe_fixtable(TabRef, false), 973 ok = dets:close(TabRef), 974 file:delete(Fname), 975 976 %% Now check that the above collected binaries are correct. 977 {ok, _} = dets:open_file(TabRef, Args), 978 ok = dets:init_table(TabRef, init([BinColl1]),{format,bchunk}), 979 true = (AllObjects1 =:= sort(get_all_objects_fast(TabRef))), 980 true = (length(AllObjects1) =:= dets:info(TabRef, no_objects)), 981 ok = dets:init_table(TabRef, init([BinColl2]),{format,bchunk}), 982 true = (AllObjects2 =:= sort(get_all_objects_fast(TabRef))), 983 true = (length(AllObjects2) =:= dets:info(TabRef, no_objects)), 984 ok = dets:init_table(TabRef, init([BinColl3]),{format,bchunk}), 985 true = (AllObjects3 =:= sort(get_all_objects_fast(TabRef))), 986 true = (length(AllObjects3) =:= dets:info(TabRef, no_objects)), 987 ok = dets:close(TabRef), 988 file:delete(Fname), 989 check_pps(P0), 990 ok. 991 992fast_bulk_init(L, Type, N, NoKeys, Config) -> 993 fast_bulk_init(L, Type, N, NoKeys, 256, Config). 994 995fast_bulk_init(L, Type, N, NoKeys, Est, Config) -> 996 T = init_table_test, 997 Fname = filename(T, Config), 998 file:delete(Fname), 999 1000 Args0 = [{ram_file,false}, {type,Type},{keypos,1}, 1001 {estimated_no_objects, Est}], 1002 Args = [{file,Fname} | Args0], 1003 S = init_table_test_source, 1004 SFname = filename(S, Config), 1005 file:delete(SFname), 1006 SArgs = [{file,SFname} | Args0], 1007 1008 {ok, S} = dets:open_file(S, SArgs), 1009 ok = dets:insert(S, L), 1010 1011 Input = init_bchunk(S), 1012 {ok, T} = dets:open_file(T, Args), 1013 ok = dets:init_table(T, Input, [{format,bchunk}]), 1014 All = sort(get_all_objects(T)), 1015 FAll = get_all_objects_fast(T), 1016 true = All =:= sort(FAll), 1017 true = length(All) =:= N, 1018 true = dets:info(T, size) =:= N, 1019 true = dets:info(T, no_keys) =:= NoKeys, 1020 ok = dets:close(T), 1021 1022 {ok, T} = dets:open_file(T, Args), 1023 All2 = sort(get_all_objects(T)), 1024 FAll2 = get_all_objects_fast(T), 1025 true = All =:= All2, 1026 true = All =:= sort(FAll2), 1027 ok = dets:close(T), 1028 file:delete(Fname), 1029 1030 ok = dets:close(S), 1031 file:delete(SFname), 1032 ok. 1033 1034init_bchunk(T) -> 1035 Start = dets:bchunk(T, start), 1036 init_bchunk(T, Start). 1037 1038init_bchunk(Tab, State) -> 1039 fun(read) when State =:= '$end_of_table' -> 1040 end_of_input; 1041 (read) when element(1, State) =:= error -> 1042 State; 1043 (read) -> 1044 {Cont, Objs} = State, 1045 {Objs, init_bchunk(Tab, dets:bchunk(Tab, Cont))}; 1046 (close) -> 1047 ok 1048 end. 1049 1050collect_bchunk(Tab, Fun) -> 1051 collect_bchunk(Tab, Fun, []). 1052 1053collect_bchunk(Tab, Fun, L) -> 1054 case Fun(read) of 1055 end_of_input -> 1056 lists:append(lists:reverse(L)); 1057 {Objs, Fun2} when is_list(Objs) -> 1058 collect_bchunk(Tab, Fun2, [Objs | L]); 1059 Error -> 1060 Error 1061 end. 1062 1063items(I, N, C, L) when I =:= N; C =:= 0 -> 1064 {I, L}; 1065items(I, N, C, L) -> 1066 items(I+1, N, C-1, [{I, item(I)} | L]). 1067 1068%% Test open_file and repair. 1069repair(Config) when is_list(Config) -> 1070 %% The short lived format 9(a). 1071 %% Not very throughly tested here. 1072 A9 = a9, 1073 Version9aS = filename:join(?datadir(Config), "version_9a.dets"), 1074 Version9aT = filename('v9a.dets', Config), 1075 {ok, _} = file:copy(Version9aS, Version9aT), 1076 {ok, A9} = dets:open_file(A9, [{file,Version9aT}]), 1077 undefined = dets:info(A9, bchunk_format), 1078 [{1,a},{2,b},{3,c}] = sort(dets:match_object(A9, '_')), 1079 ok = dets:insert(A9, {4,d}), 1080 ok = dets:close(A9), 1081 {ok, A9} = dets:open_file(A9, [{file,Version9aT}]), 1082 {error, old_version} = dets:bchunk(A9, start), 1083 ok = dets:close(A9), 1084 io:format("Expect forced repair:~n"), 1085 {ok, A9} = dets:open_file(A9, [{file,Version9aT},{repair,force}]), 1086 {_, _} = dets:bchunk(A9, start), 1087 ok = dets:close(A9), 1088 file:delete(Version9aT), 1089 1090 repair_1(Config). 1091 1092repair_1(Config) -> 1093 TabRef = repair_test, 1094 Fname = filename(TabRef, Config), 1095 file:delete(Fname), 1096 HeadSize = headsz(), 1097 1098 P0 = pps(), 1099 {'EXIT', {badarg, _}} = 1100 (catch dets:open_file(TabRef, [{min_no_slots,1000}, 1101 {max_no_slots,500}])), 1102 {error,{file_error,hoppla,enoent}} = dets:file_info(hoppla), 1103 {error,{file_error,Fname,enoent}} = 1104 dets:open_file(TabRef, [{file, Fname}, {access, read}]), 1105 1106 %% compacting, and some kind of test that free lists are saved OK on file 1107 {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]), 1108 0 = dets:info(TabRef, size), 1109 ok = ins(TabRef, 30000), 1110 ok = del(TabRef, 30000, 3), 1111 ok = dets:close(TabRef), 1112 {error, {access_mode,Fname}} = 1113 dets:open_file(foo, [{file,Fname},{repair,force},{access,read}]), 1114 {ok, Ref3} = dets:open_file(Fname), % no repair! 1115 20000 = dets:info(Ref3, size), 1116 20000 = dets:foldl(fun(_, N) -> N+1 end, 0, Ref3), 1117 20000 = count_objects_quite_fast(Ref3), % actually a test of match 1118 no_keys_test(Ref3), 1119 ok = dets:close(Ref3), 1120 {error,{keypos_mismatch,Fname}} = 1121 dets:open_file(TabRef, [{file, Fname},{keypos,17}]), 1122 {error,{type_mismatch,Fname}} = 1123 dets:open_file(TabRef, [{file, Fname},{type,duplicate_bag}]), 1124 1125 %% make one of the temporary files unwritable 1126 TmpFile = Fname ++ ".TMP.1", 1127 file:delete(TmpFile), 1128 {ok, TmpFd} = file:open(TmpFile, [read,write]), 1129 ok = file:close(TmpFd), 1130 unwritable(TmpFile), 1131 {error,{file_error,TmpFile,eacces}} = dets:fsck(Fname), 1132 {ok, _} = dets:open_file(TabRef, 1133 [{repair,false},{file, Fname}]), 1134 20000 = length(get_all_objects(TabRef)), 1135 _ = histogram(TabRef, silent), 1136 20000 = length(get_all_objects_fast(TabRef)), 1137 ok = dets:close(TabRef), 1138 writable(TmpFile), 1139 file:delete(TmpFile), 1140 1141 truncate(Fname, HeadSize + 10), 1142 {error,{not_closed, Fname}} = 1143 dets:open_file(TabRef, [{file, Fname}, {access, read}]), 1144 {error,{not_closed, Fname}} = 1145 dets:open_file(TabRef, [{file, Fname}, {access, read}, 1146 {repair,force}]), 1147 {error,{needs_repair, Fname}} = 1148 dets:open_file(TabRef, [{file, Fname}, {repair, false}]), 1149 file:delete(Fname), 1150 1151 %% truncated file header 1152 {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]), 1153 ok = ins(TabRef, 100), 1154 ok = dets:close(TabRef), 1155 file:delete(Fname), 1156 1157 %% FIXME. 1158 %% will fail unless the slots are properly sorted when repairing (v8) 1159 BArgs = [{file, Fname},{type,duplicate_bag}, 1160 {delayed_write,{3000,10000}}], 1161 {ok, TabRef} = dets:open_file(TabRef, BArgs), 1162 Seq = seq(1, 500), 1163 Small = map(fun(X) -> {X,X} end, Seq), 1164 Big = map(fun(X) -> erlang:make_tuple(20, X) end, Seq), 1165 ok = dets:insert(TabRef, Small), 1166 ok = dets:insert(TabRef, Big), 1167 ok = dets:insert(TabRef, Small), 1168 ok = dets:insert(TabRef, Big), 1169 All = sort(safe_get_all_objects(TabRef)), 1170 ok = dets:close(TabRef), 1171 io:format("Expect forced repair:~n"), 1172 {ok, _} = 1173 dets:open_file(TabRef, [{repair,force},{min_no_slots,2000} | BArgs]), 1174 1175 {MinNoSlots,_,MaxNoSlots} = dets:info(TabRef, no_slots), 1176 ok = dets:close(TabRef), 1177 io:format("Expect compaction:~n"), 1178 {ok, _} = 1179 dets:open_file(TabRef, [{repair,force}, 1180 {min_no_slots,MinNoSlots}, 1181 {max_no_slots,MaxNoSlots} | BArgs]), 1182 All2 = get_all_objects(TabRef), 1183 true = All =:= sort(All2), 1184 FAll2 = get_all_objects_fast(TabRef), 1185 true = All =:= sort(FAll2), 1186 true = length(All) =:= dets:info(TabRef, size), 1187 no_keys_test(TabRef), 1188 Fun = fun(X) -> 4 = length(dets:lookup(TabRef, X)) end, 1189 foreach(Fun, Seq), 1190 _ = histogram(TabRef, silent), 1191 ok = dets:close(TabRef), 1192 file:delete(Fname), 1193 1194 %% object bigger than segments, the "hole" is taken care of 1195 {ok, TabRef} = dets:open_file(TabRef, [{file, Fname}]), 1196 Tuple = erlang:make_tuple(1000, foobar), % > 2 kB 1197 ok = dets:insert(TabRef, Tuple), 1198 %% at least one full segment (objects smaller than 2 kB): 1199 ins(TabRef, 2000), 1200 ok = dets:close(TabRef), 1201 1202 {ok, _} = 1203 dets:open_file(TabRef, [{repair,false},{file, Fname}]), 1204 {ok, ObjPos} = dets:where(TabRef, {66,{item,number,66}}), 1205 ok = dets:close(TabRef), 1206 %% Damaged object. 1207 Pos = 12, % v9: compaction fails, proper repair follows 1208 crash(Fname, ObjPos+Pos), 1209 io:format( 1210 "Expect forced repair (possibly after attempted compaction):~n"), 1211 {ok, _} = dets:open_file(TabRef, [{repair,force},{file, Fname}]), 1212 true = dets:info(TabRef, size) < 2001, 1213 ok = dets:close(TabRef), 1214 file:delete(Fname), 1215 1216 %% The file is smaller than the padded object. 1217 {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]), 1218 ok = dets:insert(TabRef, Tuple), 1219 ok = dets:close(TabRef), 1220 io:format("Expect forced repair or compaction:~n"), 1221 {ok, _} = 1222 dets:open_file(TabRef, [{repair,force},{file, Fname}]), 1223 true = 1 =:= dets:info(TabRef, size), 1224 ok = dets:close(TabRef), 1225 file:delete(Fname), 1226 1227 %% Damaged free lists. 1228 {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]), 1229 ok = ins(TabRef, 300), 1230 ok = dets:sync(TabRef), 1231 ok = del(TabRef, 300, 3), 1232 %% FileSize is approximately where the free lists will be written. 1233 FileSize = dets:info(TabRef, memory), 1234 ok = dets:close(TabRef), 1235 crash(Fname, FileSize+20), 1236 %% Used to return bad_freelists, but that changed in OTP-9622 1237 {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]), 1238 ok = dets:close(TabRef), 1239 file:delete(Fname), 1240 1241 %% File not closed, opening with read and read_write access tried. 1242 {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]), 1243 ok = ins(TabRef, 300), 1244 ok = dets:close(TabRef), 1245 crash(Fname, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED), 1246 {error, {not_closed, Fname}} = 1247 dets:open_file(foo, [{file,Fname},{repair,force}, 1248 {access,read}]), 1249 {error, {not_closed, Fname}} = 1250 dets:open_file(foo, [{file,Fname},{repair,true}, 1251 {access,read}]), 1252 io:format("Expect repair:~n"), 1253 {ok, TabRef} = 1254 dets:open_file(TabRef, [{file,Fname},{repair,true}, 1255 {access,read_write}]), 1256 ok = dets:close(TabRef), 1257 crash(Fname, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED), 1258 io:format("Expect forced repair:~n"), 1259 {ok, TabRef} = 1260 dets:open_file(TabRef, [{file,Fname},{repair,force}, 1261 {access,read_write}]), 1262 ok = dets:close(TabRef), 1263 file:delete(Fname), 1264 1265 %% The size of an object is huge. 1266 {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]), 1267 ok = dets:insert(TabRef, [{1,2,3},{2,3,4}]), 1268 {ok, ObjPos2} = dets:where(TabRef, {1,2,3}), 1269 ok = dets:close(TabRef), 1270 crash(Fname, ObjPos2, 255), 1271 io:format("Expect forced repair:~n"), 1272 {ok, TabRef} = dets:open_file(TabRef, [{file,Fname},{repair,force}]), 1273 ok = dets:close(TabRef), 1274 file:delete(Fname), 1275 1276 check_pps(P0), 1277 ok. 1278 1279 1280%% Test version 9(b) with erlang:phash/2 as hash function. 1281phash(Config) when is_list(Config) -> 1282 T = phash, 1283 Phash_v9bS = filename:join(?datadir(Config), "version_9b_phash.dat"), 1284 Fname = filename('v9b.dets', Config), 1285 {ok, _} = file:copy(Phash_v9bS, Fname), 1286 1287 %% Deleting all objects changes the hash function. 1288 %% A feature... (it's for free) 1289 {ok, T} = dets:open_file(T, [{file, Fname}]), 1290 phash = dets:info(T, hash), 1291 dets:delete_all_objects(T), 1292 phash2 = dets:info(T, hash), 1293 [] = get_all_objects(T), 1294 [] = get_all_objects_fast(T), 1295 ok = dets:close(T), 1296 1297 %% The hash function is kept when compacting a table. 1298 {ok, _} = file:copy(Phash_v9bS, Fname), 1299 io:format("Expect compaction:~n"), 1300 {ok, T} = dets:open_file(T, [{file, Fname},{repair,force}]), 1301 phash = dets:info(T, hash), 1302 [{1,a},{2,b},{3,c},{4,d},{5,e}] = 1303 lists:sort(dets:lookup_keys(T, [1,2,3,4,5])), 1304 ok = dets:close(T), 1305 1306 %% The hash function is updated when repairing a table (no cost). 1307 {ok, _} = file:copy(Phash_v9bS, Fname), 1308 crash(Fname, ?CLOSED_PROPERLY_POS+3, 0), 1309 io:format("Expect repair:~n"), 1310 {ok, T} = dets:open_file(T, [{file, Fname}]), 1311 phash2 = dets:info(T, hash), 1312 [{1,a},{2,b},{3,c},{4,d},{5,e}] = 1313 lists:sort(dets:lookup_keys(T, [1,2,3,4,5])), 1314 ok = dets:close(T), 1315 1316 %% One cannot use the bchunk format when copying between a phash 1317 %% table and a phash2 table. (There is no test for the case an 1318 %% Erlang/OTP R9 (or later) node (using phash2) copies a table to 1319 %% an Erlang/OTP R8 node (using phash).) See also the comment on 1320 %% HASH_PARMS in dets_v9.erl. 1321 {ok, _} = file:copy(Phash_v9bS, Fname), 1322 {ok, T} = dets:open_file(T, [{file, Fname}]), 1323 Type = dets:info(T, type), 1324 KeyPos = dets:info(T, keypos), 1325 Input = init_bchunk(T), 1326 T2 = phash_table, 1327 Fname2 = filename(T2, Config), 1328 Args = [{type,Type},{keypos,KeyPos},{file,Fname2}], 1329 {ok, T2} = dets:open_file(T2, Args), 1330 {error, {init_fun, _}} = 1331 dets:init_table(T2, Input, {format,bchunk}), 1332 _ = dets:close(T2), 1333 ok = dets:close(T), 1334 file:delete(Fname2), 1335 1336 file:delete(Fname), 1337 ok. 1338 1339%% Test foldl, foldr, to_ets. 1340fold(Config) when is_list(Config) -> 1341 T = test_table, 1342 N = 100, 1343 Fname = filename(T, Config), 1344 file:delete(Fname), 1345 P0 = pps(), 1346 1347 Args = [{file,Fname}, {estimated_no_objects, N}], 1348 {ok, _} = dets:open_file(T, Args), 1349 1350 ok = ins(T, N), 1351 1352 Ets = ets:new(to_ets, [public]), 1353 dets:to_ets(T, Ets), 1354 true = N =:= ets:info(Ets, size), 1355 ets:delete(Ets), 1356 1357 Ets2 = ets:new(to_ets, [private]), 1358 dets:to_ets(T, Ets2), 1359 true = N =:= ets:info(Ets2, size), 1360 ets:delete(Ets2), 1361 1362 {'EXIT', {badarg, _}} = (catch dets:to_ets(T, not_an_ets_table)), 1363 1364 F0 = fun(X, A) -> [X | A] end, 1365 true = N =:= length(dets:foldl(F0, [], T)), 1366 true = N =:= length(dets:foldr(F0, [], T)), 1367 1368 F1 = fun(_X, _A) -> throw(away) end, 1369 away = (catch dets:foldl(F1, [], T)), 1370 away = (catch dets:foldr(F1, [], T)), 1371 1372 F2 = fun(X, A) -> X + A end, 1373 {'EXIT', _} = (catch dets:foldl(F2, [], T)), 1374 {'EXIT', _} = (catch dets:foldr(F2, [], T)), 1375 1376 F3 = fun(_X) -> throw(away) end, 1377 away = (catch dets:traverse(T, F3)), 1378 1379 F4 = fun(X) -> X + 17 end, 1380 {'EXIT', _} = (catch dets:traverse(T, F4)), 1381 1382 F5 = fun(_X) -> done end, 1383 done = dets:traverse(T, F5), 1384 1385 {ok, ObjPos} = dets:where(T, {66,{item,number,66}}), 1386 ok = dets:close(T), 1387 1388 %% Damaged object. 1389 Pos = 8, 1390 crash(Fname, ObjPos+Pos), 1391 {ok, _} = dets:open_file(T, Args), 1392 io:format("Expect corrupt table:~n"), 1393 BadObject1 = dets:foldl(F0, [], T), 1394 bad_object(BadObject1, Fname), 1395 BadObject2 = dets:close(T), 1396 bad_object(BadObject2, Fname), 1397 1398 file:delete(Fname), 1399 check_pps(P0), 1400 ok. 1401 1402%% Add objects to a fixed table. 1403fixtable(Config) when is_list(Config) -> 1404 T = fixtable, 1405 Fname = filename(fixtable, Config), 1406 file:delete(Fname), 1407 Args = [{file,Fname}], 1408 P0 = pps(), 1409 {ok, _} = dets:open_file(T, Args), 1410 1411 %% badarg 1412 check_badarg(catch dets:safe_fixtable(no_table,true), 1413 dets, safe_fixtable, [no_table,true]), 1414 check_badarg(catch dets:safe_fixtable(T,undefined), 1415 dets, safe_fixtable, [T,undefined]), 1416 1417 %% The table is not allowed to grow while the elements are inserted: 1418 1419 ok = ins(T, 500), 1420 dets:safe_fixtable(T, false), 1421 %% Now the table can grow. At the same time as elements are inserted, 1422 %% the table tries to catch up with the previously inserted elements. 1423 ok = ins(T, 1000), 1424 1000 = dets:info(T, size), 1425 ok = dets:close(T), 1426 file:delete(Fname), 1427 1428 {ok, _} = dets:open_file(T, [{type, duplicate_bag} | Args]), 1429 %% In a fixed table, delete and re-insert an object. 1430 ok = dets:insert(T, {1, a, b}), 1431 SysBefore = erlang:timestamp(), 1432 MonBefore = erlang:monotonic_time(), 1433 dets:safe_fixtable(T, true), 1434 MonAfter = erlang:monotonic_time(), 1435 SysAfter = erlang:timestamp(), 1436 Self = self(), 1437 {FixMonTime,[{Self,1}]} = dets:info(T,safe_fixed_monotonic_time), 1438 {FixSysTime,[{Self,1}]} = dets:info(T,safe_fixed), 1439 true = is_integer(FixMonTime), 1440 true = MonBefore =< FixMonTime, 1441 true = FixMonTime =< MonAfter, 1442 {FstMs,FstS,FstUs} = FixSysTime, 1443 true = is_integer(FstMs), 1444 true = is_integer(FstS), 1445 true = is_integer(FstUs), 1446 case erlang:system_info(time_warp_mode) of 1447 no_time_warp -> 1448 true = timer:now_diff(FixSysTime, SysBefore) >= 0, 1449 true = timer:now_diff(SysAfter, FixSysTime) >= 0; 1450 _ -> 1451 %% ets:info(Tab,safe_fixed) not timewarp safe... 1452 ignore 1453 end, 1454 ok = dets:match_delete(T, {1, a, b}), 1455 ok = dets:insert(T, {1, a, b}), 1456 {FixMonTime,[{Self,1}]} = dets:info(T,safe_fixed_monotonic_time), 1457 {FixSysTime,[{Self,1}]} = dets:info(T,safe_fixed), 1458 dets:safe_fixtable(T, false), 1459 1 = length(dets:match_object(T, '_')), 1460 1461 ok = dets:match_delete(T, '_'), 1462 %% In a fixed table, delete and insert a smaller object. 1463 ok = dets:insert(T, {1, duplicate(100, e)}), 1464 dets:safe_fixtable(T, true), 1465 ok = dets:match_delete(T, {1, '_'}), 1466 ok = dets:insert(T, {1, a, b}), 1467 dets:safe_fixtable(T, false), 1468 1 = length(dets:match_object(T, '_')), 1469 1470 ok = dets:delete_all_objects(T), 1471 %% Like the last one, but one extra object. 1472 ok = dets:insert(T, {1, duplicate(100, e)}), 1473 ok = dets:insert(T, {2, duplicate(100, e)}), 1474 dets:safe_fixtable(T, true), 1475 ok = dets:match_delete(T, {1, '_'}), 1476 ok = dets:insert(T, {1, a, b}), 1477 dets:safe_fixtable(T, false), 1478 2 = length(dets:match_object(T, '_')), 1479 dets:safe_fixtable(T, true), 1480 ok = dets:delete_all_objects(T), 1481 true = dets:info(T, fixed), 1482 0 = length(dets:match_object(T, '_')), 1483 1484 ok = dets:close(T), 1485 file:delete(Fname), 1486 check_pps(P0), 1487 ok. 1488 1489%% Matching objects of a fixed table. 1490match(Config) when is_list(Config) -> 1491 T = match, 1492 Fname = filename(match, Config), 1493 file:delete(Fname), 1494 P0 = pps(), 1495 1496 Args = [{file,Fname}, {type, duplicate_bag}, {estimated_no_objects,550}], 1497 {ok, _} = dets:open_file(T, Args), 1498 ok = dets:insert(T, {1, a, b}), 1499 ok = dets:insert(T, {1, b, a}), 1500 ok = dets:insert(T, {2, a, b}), 1501 ok = dets:insert(T, {2, b, a}), 1502 1503 %% match, badarg 1504 MSpec = [{'_',[],['$_']}], 1505 check_badarg(catch dets:match(no_table, '_'), 1506 dets, match, [no_table,'_']), 1507 check_badarg(catch dets:match(T, '_', not_a_number), 1508 dets, match, [T,'_',not_a_number]), 1509 {EC1, _} = dets:select(T, MSpec, 1), 1510 check_badarg(catch dets:match(EC1), 1511 dets, match, [EC1]), 1512 1513 %% match_object, badarg 1514 check_badarg(catch dets:match_object(no_table, '_'), 1515 dets, match_object, [no_table,'_']), 1516 check_badarg(catch dets:match_object(T, '_', not_a_number), 1517 dets, match_object, [T,'_',not_a_number]), 1518 {EC2, _} = dets:select(T, MSpec, 1), 1519 check_badarg(catch dets:match_object(EC2), 1520 dets, match_object, [EC2]), 1521 1522 dets:safe_fixtable(T, true), 1523 {[_, _], C1} = dets:match_object(T, '_', 2), 1524 {[_, _], C2} = dets:match_object(C1), 1525 '$end_of_table' = dets:match_object(C2), 1526 {[_, _], C3} = dets:match_object(T, {1, '_', '_'}, 100), 1527 '$end_of_table' = dets:match_object(C3), 1528 '$end_of_table' = dets:match_object(T, {'_'}, default), 1529 dets:safe_fixtable(T, false), 1530 1531 dets:safe_fixtable(T, true), 1532 {[_, _], C30} = dets:match(T, '$1', 2), 1533 {[_, _], C31} = dets:match(C30), 1534 '$end_of_table' = dets:match(C31), 1535 {[_, _], C32} = dets:match(T, {1, '$1', '_'}, 100), 1536 '$end_of_table' = dets:match(C32), 1537 '$end_of_table' = dets:match(T, {'_'}, default), 1538 dets:safe_fixtable(T, false), 1539 [[1],[1],[2],[2]] = sort(dets:match(T, {'$1','_','_'})), 1540 1541 %% delete and insert while chunking 1542 %% (this case almost worthless after changes in OTP-5232) 1543 ok = dets:match_delete(T, '_'), 1544 L500 = seq(1, 500), 1545 Fun = fun(X) -> ok = dets:insert(T, {X, a, b, c, d}) end, 1546 foreach(Fun, L500), 1547 %% Select one object DI in L3 below to be deleted. 1548 {_, TmpCont} = dets:match_object(T, '_', 200), 1549 {_, TmpCont1} = dets:match_object(TmpCont), 1550 {TTL, _} = dets:match_object(TmpCont1), 1551 DI = hd(TTL), 1552 dets:safe_fixtable(T, true), 1553 {L1, C20} = dets:match_object(T, '_', 200), 1554 true = 200 =< length(L1), 1555 ok = dets:match_delete(T, {'2','_','_'}), % no match 1556 ok = dets:match_delete(T, DI), % last object 1557 Tiny = {1050}, 1558 ok = dets:insert(T, Tiny), 1559 true = member(Tiny, dets:match_object(T, '_')), 1560 {_L2, C21} = dets:match_object(C20), 1561 {_L3, _C22} = dets:match_object(C21), 1562 %% It used to be that Tiny was not visible here, but since the 1563 %% scanning of files was changed to inspect the free lists every 1564 %% now and then it may very well be visible here. 1565 %% false = member(Tiny, _L3), 1566 %% DI used to visible here, but the above mentioned modification 1567 %% has changed that; it may or may not be visible. 1568 %% true = member(DI, _L3), 1569 dets:safe_fixtable(T, false), 1570 true = dets:member(T, 1050), 1571 true = member(Tiny, dets:match_object(T, '_')), 1572 false = member(DI, dets:match_object(T, '_')), 1573 1574 ok = dets:close(T), 1575 file:delete(Fname), 1576 1577 N = 100, 1578 {ok, _} = dets:open_file(T, [{estimated_no_objects,N} | Args]), 1579 ok = ins(T, N), 1580 Obj = {66,{item,number,66}}, 1581 Spec = {'_','_'}, 1582 {ok, ObjPos} = dets:where(T, Obj), 1583 ok = dets:close(T), 1584 %% Damaged object. 1585 crash(Fname, ObjPos+12), 1586 {ok, _} = dets:open_file(T, Args), 1587 io:format("Expect corrupt table:~n"), 1588 case ins(T, N) of 1589 ok -> 1590 bad_object(dets:sync(T), Fname); 1591 Else -> 1592 bad_object(Else, Fname) 1593 end, 1594 io:format("Expect corrupt table:~n"), 1595 bad_object(dets:match(T, Spec), Fname), 1596 io:format("Expect corrupt table:~n"), 1597 bad_object(dets:match_delete(T, Spec), Fname), 1598 bad_object(dets:close(T), Fname), 1599 file:delete(Fname), 1600 1601 {ok, _} = dets:open_file(T, [{estimated_no_objects,N} | Args]), 1602 ok = ins(T, N), 1603 {ok, ObjPos2} = dets:where(T, Obj), 1604 ok = dets:close(T), 1605 1606 %% Damaged size of object. 1607 CrashPos = 1, 1608 crash(Fname, ObjPos2+CrashPos), 1609 {ok, _} = dets:open_file(T, Args), 1610 case dets:insert_new(T, Obj) of % OTP-12024 1611 ok -> 1612 bad_object(dets:sync(T), Fname); 1613 Else3 -> 1614 bad_object(Else3, Fname) 1615 end, 1616 io:format("Expect corrupt table:~n"), 1617 case ins(T, N) of 1618 ok -> 1619 bad_object(dets:sync(T), Fname); 1620 Else2 -> 1621 bad_object(Else2, Fname) 1622 end, 1623 %% Just echoes... 1624 bad_object(dets:match(T, Spec), Fname), 1625 bad_object(dets:match_delete(T, Spec), Fname), 1626 bad_object(dets:close(T), Fname), 1627 file:delete(Fname), 1628 1629 {ok, _} = dets:open_file(T, [{estimated_no_objects,N} | Args]), 1630 ok = ins(T, N), 1631 {ok, ObjPos3} = dets:where(T, Obj), 1632 ok = dets:close(T), 1633 1634 %% match_delete finds an error 1635 CrashPos3 = 16, 1636 crash(Fname, ObjPos3+CrashPos3), 1637 {ok, _} = dets:open_file(T, Args), 1638 bad_object(dets:match_delete(T, Spec), Fname), 1639 bad_object(dets:close(T), Fname), 1640 file:delete(Fname), 1641 1642 %% The key is not fixed, but not all objects with the key are removed. 1643 {ok, _} = dets:open_file(T, Args), 1644 ok = dets:insert(T, [{1,a},{1,b},{1,c},{1,a},{1,b},{1,c}]), 1645 6 = dets:info(T, size), 1646 ok = dets:match_delete(T, {'_',a}), 1647 4 = dets:info(T, size), 1648 [{1,b},{1,b},{1,c},{1,c}] = 1649 sort(dets:match_object(T,{'_','_'})), 1650 ok = dets:close(T), 1651 file:delete(Fname), 1652 1653 check_pps(P0), 1654 ok. 1655 1656%% Selecting objects of a fixed table. 1657select(Config) when is_list(Config) -> 1658 T = select, 1659 Fname = filename(select, Config), 1660 file:delete(Fname), 1661 P0 = pps(), 1662 1663 Args = [{file,Fname}, {type, duplicate_bag},{estimated_no_objects,550}], 1664 {ok, _} = dets:open_file(T, Args), 1665 ok = dets:insert(T, {1, a, b}), 1666 ok = dets:insert(T, {1, b, a}), 1667 ok = dets:insert(T, {2, a, b}), 1668 ok = dets:insert(T, {2, b, a}), 1669 ok = dets:insert(T, {3, a, b}), 1670 ok = dets:insert(T, {3, b, a}), 1671 1672 %% badarg 1673 MSpec = [{'_',[],['$_']}], 1674 check_badarg(catch dets:select(no_table, MSpec), 1675 dets, select, [no_table,MSpec]), 1676 check_badarg(catch dets:select(T, <<17>>), 1677 dets, select, [T,<<17>>]), 1678 check_badarg(catch dets:select(T, []), 1679 dets, select, [T,[]]), 1680 check_badarg(catch dets:select(T, MSpec, not_a_number), 1681 dets, select, [T,MSpec,not_a_number]), 1682 {EC, _} = dets:match(T, '_', 1), 1683 check_badarg(catch dets:select(EC), 1684 dets, select, [EC]), 1685 1686 AllSpec = [{'_',[],['$_']}], 1687 1688 dets:safe_fixtable(T, true), 1689 {[_, _], C1} = dets:select(T, AllSpec, 2), 1690 {[_, _], C2} = dets:select(C1), 1691 {[_, _], C2a} = dets:select(C2), 1692 '$end_of_table' = dets:select(C2a), 1693 {[_, _], C3} = dets:select(T, [{{1,'_','_'},[],['$_']}], 100), 1694 '$end_of_table' = dets:select(C3), 1695 '$end_of_table' = dets:select(T, [{{'_'},[],['$_']}], default), 1696 dets:safe_fixtable(T, false), 1697 Sp1 = [{{1,'_','_'},[],['$_']},{{1,'_','_'},[],['$_']}, 1698 {{2,'_','_'},[],['$_']}], 1699 [_,_,_,_] = dets:select(T, Sp1), 1700 Sp2 = [{{1,'_','_'},[],['$_']},{{1,'_','_'},[],['$_']}, 1701 {{'_','_','_'},[],['$_']}], 1702 [_,_,_,_,_,_] = dets:select(T, Sp2), 1703 1704 AllDeleteSpec = [{'_',[],[true]}], 1705 %% delete and insert while chunking 1706 %% (this case almost worthless after changes in OTP-5232) 1707 6 = dets:select_delete(T, AllDeleteSpec), 1708 L500 = seq(1, 500), 1709 Fun = fun(X) -> ok = dets:insert(T, {X, a, b, c, d}) end, 1710 foreach(Fun, L500), 1711 %% Select one object DI in L3 below to be deleted. 1712 {_, TmpCont} = dets:match_object(T, '_', 200), 1713 {_, TmpCont1} = dets:match_object(TmpCont), 1714 {TTL, _} = dets:match_object(TmpCont1), 1715 DI = hd(TTL), 1716 dets:safe_fixtable(T, true), 1717 {L1, C20} = dets:select(T, AllSpec, 200), 1718 true = 200 =< length(L1), 1719 0 = dets:select_delete(T, [{{2,'_','_'},[],[true]}]), 1720 1 = dets:select_delete(T, [{DI,[],[true]}]), % last object 1721 Tiny = {1050}, 1722 ok = dets:insert(T, Tiny), 1723 true = member(Tiny, dets:select(T, AllSpec)), 1724 {_L2, C21} = dets:select(C20), 1725 {_L3, _C22} = dets:select(C21), 1726 %% It used to be that Tiny was not visible here, but since the 1727 %% scanning of files was changed to inspect the free lists every 1728 %% now and then it may very well be visible here. 1729 %% false = member(Tiny, _L3), 1730 %% DI used to visible here, but the above mentioned modification 1731 %% has changed that; it may or may not be visible. 1732 %% true = member(DI, _L3), 1733 true = dets:member(T, 1050), 1734 true = member(Tiny, dets:select(T, AllSpec)), 1735 false = member(DI, dets:select(T, AllSpec)), 1736 dets:safe_fixtable(T, false), 1737 true = dets:member(T, 1050), 1738 true = member(Tiny, dets:select(T, AllSpec)), 1739 false = member(DI, dets:select(T, AllSpec)), 1740 ok = dets:close(T), 1741 file:delete(Fname), 1742 1743 %% The key is not fixed, but not all objects with the key are removed. 1744 {ok, _} = dets:open_file(T, Args), 1745 ok = dets:insert(T, [{1,a},{1,b},{1,c},{1,a},{1,b},{1,c}]), 1746 6 = dets:info(T, size), 1747 2 = dets:select_delete(T, [{{'_',a},[],[true]}]), 1748 4 = dets:info(T, size), 1749 [{1,b},{1,b},{1,c},{1,c}] = sort(dets:select(T, AllSpec)), 1750 ok = dets:close(T), 1751 file:delete(Fname), 1752 1753 check_pps(P0), 1754 ok. 1755 1756%% Test update_counter/1. 1757update_counter(Config) when is_list(Config) -> 1758 T = update_counter, 1759 Fname = filename(select, Config), 1760 file:delete(Fname), 1761 P0 = pps(), 1762 1763 check_badarg(catch dets:update_counter(no_table, 1, 1), 1764 dets, update_counter, [no_table,1,1]), 1765 1766 Args = [{file,Fname},{keypos,2}], 1767 {ok, _} = dets:open_file(T, [{type,set} | Args]), 1768 {'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)), 1769 ok = dets:insert(T, {1,a}), 1770 {'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)), 1771 ok = dets:insert(T, {0,1}), 1772 {'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)), 1773 ok = dets:insert(T, {0,1,0}), 1774 1 = dets:update_counter(T, 1, 1), 1775 2 = dets:update_counter(T, 1, 1), 1776 6 = dets:update_counter(T, 1, {3,4}), 1777 {'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, {0,3})), 1778 ok = dets:close(T), 1779 file:delete(Fname), 1780 1781 {ok, _} = dets:open_file(T, [{type,bag} | Args]), 1782 ok = dets:insert(T, {0,1,0}), 1783 {'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)), 1784 ok = dets:close(T), 1785 file:delete(Fname), 1786 check_pps(P0), 1787 1788 ok. 1789 1790%% Call some functions with bad arguments. 1791badarg(Config) when is_list(Config) -> 1792 T = badarg, 1793 Fname = filename(select, Config), 1794 file:delete(Fname), 1795 P0 = pps(), 1796 1797 Args = [{file,Fname},{keypos,3}], 1798 {ok, _} = dets:open_file(T, [{type,set} | Args]), 1799 1800 %% badargs are tested in match, select and fixtable too. 1801 1802 %% open 1803 check_badarg(catch dets:open_file({a,tuple},[]), 1804 dets, open_file, [{a,tuple},[]]), 1805 check_badarg(catch dets:open_file({a,tuple}), 1806 dets, open_file,[{a,tuple}]), 1807 check_badarg(catch dets:open_file(file,[foo]), 1808 dets, open_file, [file,[foo]]), 1809 check_badarg(catch dets:open_file({hej,san},[{type,set}|3]), 1810 dets, open_file, [{hej,san},[{type,set}|3]]), 1811 1812 %% insert 1813 check_badarg(catch dets:insert(no_table, {1,2}), 1814 dets, insert, [no_table,{1,2}]), 1815 check_badarg(catch dets:insert(no_table, [{1,2}]), 1816 dets, insert, [no_table,[{1,2}]]), 1817 check_badarg(catch dets:insert(T, {1,2}), 1818 dets, insert, [T,{1,2}]), 1819 check_badarg(catch dets:insert(T, [{1,2}]), 1820 dets, insert, [T,[{1,2}]]), 1821 check_badarg(catch dets:insert(T, [{1,2,3} | 3]), 1822 dets, insert, [T,[{1,2,3}|3]]), 1823 1824 %% lookup{_keys} 1825 check_badarg(catch dets:lookup_keys(T, []), 1826 dets, lookup_keys, [badarg,[]]), 1827 check_badarg(catch dets:lookup(no_table, 1), 1828 dets, lookup, [no_table,1]), 1829 check_badarg(catch dets:lookup_keys(T, [1 | 2]), 1830 dets, lookup_keys, [T,[1|2]]), 1831 1832 %% member 1833 check_badarg(catch dets:member(no_table, 1), 1834 dets, member, [no_table,1]), 1835 1836 %% sync 1837 check_badarg(catch dets:sync(no_table), 1838 dets, sync, [no_table]), 1839 1840 %% delete{_keys} 1841 check_badarg(catch dets:delete(no_table, 1), 1842 dets, delete, [no_table,1]), 1843 1844 %% delete_object 1845 check_badarg(catch dets:delete_object(no_table, {1,2,3}), 1846 dets, delete_object, [no_table,{1,2,3}]), 1847 check_badarg(catch dets:delete_object(T, {1,2}), 1848 dets, delete_object, [T,{1,2}]), 1849 check_badarg(catch dets:delete_object(no_table, [{1,2,3}]), 1850 dets, delete_object, [no_table,[{1,2,3}]]), 1851 check_badarg(catch dets:delete_object(T, [{1,2}]), 1852 dets, delete_object, [T,[{1,2}]]), 1853 check_badarg(catch dets:delete_object(T, [{1,2,3} | 3]), 1854 dets, delete_object, [T,[{1,2,3}|3]]), 1855 1856 %% first,next,slot 1857 check_badarg(catch dets:first(no_table), 1858 dets, first, [no_table]), 1859 check_badarg(catch dets:next(no_table, 1), 1860 dets, next, [no_table,1]), 1861 check_badarg(catch dets:slot(no_table, 0), 1862 dets, slot, [no_table,0]), 1863 1864 %% info 1865 undefined = dets:info(no_table), 1866 undefined = dets:info(no_table, foo), 1867 undefined = dets:info(T, foo), 1868 1869 %% match_delete 1870 check_badarg(catch dets:match_delete(no_table, '_'), 1871 dets, match_delete, [no_table,'_']), 1872 1873 %% delete_all_objects 1874 check_badarg(catch dets:delete_all_objects(no_table), 1875 dets, delete_all_objects, [no_table]), 1876 1877 %% select_delete 1878 MSpec = [{'_',[],['$_']}], 1879 check_badarg(catch dets:select_delete(no_table, MSpec), 1880 dets, select_delete, [no_table,MSpec]), 1881 check_badarg(catch dets:select_delete(T, <<17>>), 1882 dets, select_delete, [T, <<17>>]), 1883 1884 %% traverse, fold 1885 TF = fun(_) -> continue end, 1886 check_badarg(catch dets:traverse(no_table, TF), 1887 dets, traverse, [no_table,TF]), 1888 FF = fun(_, A) -> A end, 1889 check_badarg(catch dets:foldl(FF, [], no_table), 1890 dets, foldl, [FF,[],no_table]), 1891 check_badarg(catch dets:foldr(FF, [], no_table), 1892 dets, foldl, [FF,[],no_table]), 1893 1894 %% close 1895 ok = dets:close(T), 1896 {error, not_owner} = dets:close(T), 1897 {error, not_owner} = dets:close(T), 1898 1899 %% init_table 1900 IF = fun(X) -> X end, 1901 check_badarg(catch dets:init_table(no_table, IF), 1902 dets, init_table, [no_table,IF,[]]), 1903 check_badarg(catch dets:init_table(no_table, IF, []), 1904 dets, init_table, [no_table,IF,[]]), 1905 1906 %% from_ets 1907 Ets = ets:new(ets,[]), 1908 check_badarg(catch dets:from_ets(no_table, Ets), 1909 dets, from_ets, [no_table,Ets]), 1910 ets:delete(Ets), 1911 1912 {ok, T} = dets:open_file(T, Args), 1913 {error,incompatible_arguments} = 1914 dets:open_file(T, [{type,bag} | Args]), 1915 ok = dets:close(T), 1916 1917 file:delete(Fname), 1918 check_pps(P0), 1919 ok. 1920 1921%% Test the write cache for sets. 1922cache_sets(Config) when is_list(Config) -> 1923 Small = 2, 1924 cache_sets(Config, {0,0}, false, Small), 1925 cache_sets(Config, {0,0}, true, Small), 1926 cache_sets(Config, {5000,5000}, false, Small), 1927 cache_sets(Config, {5000,5000}, true, Small), 1928 %% Objects of size greater than 2 kB. 1929 Big = 1200, 1930 cache_sets(Config, {0,0}, false, Big), 1931 cache_sets(Config, {0,0}, true, Big), 1932 cache_sets(Config, {5000,5000}, false, Big), 1933 cache_sets(Config, {5000,5000}, true, Big), 1934 ok. 1935 1936cache_sets(Config, DelayedWrite, Extra, Sz) -> 1937 %% Extra = bool(). Insert tuples until the tested key is not alone. 1938 %% Sz = integer(). Size of the inserted tuples. 1939 1940 T = cache, 1941 Fname = filename(cache, Config), 1942 file:delete(Fname), 1943 P0 = pps(), 1944 1945 {ok, _} = dets:open_file(T,[{file,Fname}, {type,set}, 1946 {delayed_write, DelayedWrite}]), 1947 1948 Dups = 1, 1949 {Key, OtherKeys} = 1950 if 1951 Extra -> 1952 %% Insert enough to get three keys in some slot. 1953 dets:safe_fixtable(T, true), 1954 insert_objs(T, 1, Sz, Dups); 1955 true -> 1956 {1,[]} 1957 end, 1958 Tuple = erlang:make_tuple(Sz, Key), 1959 ok = dets:delete(T, Key), 1960 ok = dets:sync(T), 1961 1962 %% The values of keys in the same slot as Key are checked. 1963 OtherValues = sort(lookup_keys(T, OtherKeys)), 1964 1965 ok = dets:insert(T, Tuple), 1966 [Tuple] = dets:lookup(T, Key), 1967 true = dets:member(T, Key), 1968 ok = dets:insert(T, [Tuple,Tuple]), 1969 %% If no delay, the cache gets filled immediately, and written. 1970 [Tuple] = dets:lookup_keys(T, [Key,a,b,c,d,e,f]), 1971 true = dets:member(T, Key), 1972 1973 %% If delay, this happens without file access. 1974 ok = dets:delete(T,Key), 1975 ok = dets:insert(T,Tuple), 1976 ok = dets:insert(T,Tuple), 1977 [Tuple] = dets:lookup(T, Key), 1978 true = dets:member(T, Key), 1979 ok = dets:sync(T), 1980 [Tuple] = dets:lookup(T, Key), 1981 true = dets:member(T, Key), 1982 1983 %% Key's objects are is on file only, 1984 %% key 'toto' in the cache (if there is one). 1985 ok = dets:delete(T,toto), 1986 ok = dets:insert(T,[{toto,b},{toto,b}]), 1987 true = sort([Tuple,{toto,b}]) =:= 1988 sort(dets:lookup_keys(T, [Key,toto])), 1989 true = dets:member(T, toto), 1990 1991 ok = dets:delete(T, Key), 1992 ok = dets:sync(T), 1993 false = dets:member(T, Key), 1994 Size = dets:info(T, size), 1995 1996 %% No object with the key on the file. 1997 %% Delete, add one object. 1998 Size1 = Size + 2, 1999 del_and_ins(key, T, Size1, Tuple, Key, 1), 2000 del_and_ins(object, T, Size1, Tuple, Key, 1), 2001 del_and_ins(both, T, Size1, Tuple, Key, 1), 2002 2003 %% One object with the key on the file. 2004 %% Delete it, add one object. 2005 Size2 = Size + 2, 2006 del_and_ins(key, T, Size2, Tuple, Key, 1), 2007 del_and_ins(object, T, Size2, Tuple, Key, 1), 2008 del_and_ins(both, T, Size2, Tuple, Key, 1), 2009 2010 %% Overwrite an old objekt with exactly the same size. 2011 Element = case element(2, Tuple) of 2012 255 -> 254; 2013 E -> E + 1 2014 end, 2015 Tuple2 = setelement(2, Tuple, Element), 2016 ok = dets:sync(T), 2017 ok = dets:insert(T, Tuple2), 2018 [Tuple2] = dets:lookup(T, Key), 2019 true = dets:member(T, Key), 2020 ok = dets:sync(T), 2021 [Tuple2] = dets:lookup(T, Key), 2022 true = dets:member(T, Key), 2023 2024 ok = dets:insert(T, {3,a}), 2025 ok = dets:insert(T, {3,b}), 2026 ok = dets:delete_object(T, {3,c}), 2027 ok = dets:delete_object(T, {3,d}), 2028 [{3,b}] = dets:lookup(T, 3), 2029 2030 ok = dets:delete(T, 3), 2031 ok = dets:delete_object(T, {3,c}), 2032 ok = dets:delete_object(T, {3,d}), 2033 [] = dets:lookup(T, 3), 2034 2035 OtherValues = sort(lookup_keys(T, OtherKeys)), 2036 if 2037 Extra -> 2038 %% Let the table grow a while, if it needs to. 2039 All1 = get_all_objects(T), 2040 dets:safe_fixtable(T, false), 2041 timer:sleep(1000), 2042 OtherValues = sort(lookup_keys(T, OtherKeys)), 2043 dets:safe_fixtable(T, true), 2044 All2 = get_all_objects(T), 2045 FAll2 = get_all_objects_fast(T), 2046 true = sort(All2) =:= sort(FAll2), 2047 case symdiff(All1, All2) of 2048 {[],[]} -> ok; 2049 {X,Y} -> 2050 NoBad = length(X) + length(Y), 2051 ct:fail({sets,DelayedWrite,Extra,Sz,NoBad}) 2052 end; 2053 true -> 2054 ok 2055 end, 2056 ok = dets:close(T), 2057 2058 file:delete(Fname), 2059 check_pps(P0), 2060 ok. 2061 2062%% Test the write cache for bags. 2063cache_bags(Config) when is_list(Config) -> 2064 Small = 2, 2065 cache_bags(Config, {0,0}, false, Small), 2066 cache_bags(Config, {0,0}, true, Small), 2067 cache_bags(Config, {5000,5000}, false, Small), 2068 cache_bags(Config, {5000,5000}, true, Small), 2069 %% Objects of size greater than 2 kB. 2070 Big = 1200, 2071 cache_bags(Config, {0,0}, false, Big), 2072 cache_bags(Config, {0,0}, true, Big), 2073 cache_bags(Config, {5000,5000}, false, Big), 2074 cache_bags(Config, {5000,5000}, true, Big), 2075 ok. 2076 2077cache_bags(Config, DelayedWrite, Extra, Sz) -> 2078 %% Extra = bool(). Insert tuples until the tested key is not alone. 2079 %% Sz = integer(). Size of the inserted tuples. 2080 2081 T = cache, 2082 Fname = filename(cache, Config), 2083 file:delete(Fname), 2084 P0 = pps(), 2085 2086 {ok, _} = dets:open_file(T,[{file,Fname}, {type,bag}, 2087 {delayed_write, DelayedWrite}]), 2088 2089 Dups = 1, 2090 {Key, OtherKeys} = 2091 if 2092 Extra -> 2093 %% Insert enough to get three keys in some slot. 2094 dets:safe_fixtable(T, true), 2095 insert_objs(T, 1, Sz, Dups); 2096 true -> 2097 {1,[]} 2098 end, 2099 Tuple = erlang:make_tuple(Sz, Key), 2100 ok = dets:delete(T, Key), 2101 ok = dets:sync(T), 2102 2103 %% The values of keys in the same slot as Key are checked. 2104 OtherValues = sort(lookup_keys(T, OtherKeys)), 2105 2106 ok = dets:insert(T, Tuple), 2107 [Tuple] = dets:lookup(T, Key), 2108 true = dets:member(T, Key), 2109 ok = dets:insert(T, [Tuple,Tuple]), 2110 %% If no delay, the cache gets filled immediately, and written. 2111 [Tuple] = dets:lookup_keys(T, [Key,a,b,c,d,e,f]), 2112 true = dets:member(T, Key), 2113 2114 %% If delay, this happens without file access. 2115 %% (This is no longer true; cache lookup has been simplified.) 2116 ok = dets:delete(T,Key), 2117 ok = dets:insert(T,Tuple), 2118 ok = dets:insert(T,Tuple), 2119 [Tuple] = dets:lookup(T, Key), 2120 true = dets:member(T, Key), 2121 ok = dets:sync(T), 2122 [Tuple] = dets:lookup(T, Key), 2123 true = dets:member(T, Key), 2124 2125 %% Key's objects are is on file only, 2126 %% key toto in the cache (if there is one). 2127 ok = dets:delete(T,toto), 2128 false = dets:member(T, toto), 2129 ok = dets:insert(T,[{toto,b},{toto,b}]), 2130 true = sort([Tuple,{toto,b}]) =:= 2131 sort(dets:lookup_keys(T, [Key,toto])), 2132 true = dets:member(T, toto), 2133 2134 ok = dets:delete(T, Key), 2135 ok = dets:sync(T), 2136 Size = dets:info(T, size), 2137 2138 %% No object with the key on the file. 2139 %% Delete, add one object. 2140 Size1 = Size + 2, 2141 del_and_ins(key, T, Size1, Tuple, Key, 1), 2142 del_and_ins(object, T, Size1, Tuple, Key, 1), 2143 del_and_ins(both, T, Size1, Tuple, Key, 1), 2144 2145 %% One object with the key on the file. 2146 %% Delete it, add one object. 2147 Size2 = Size + 2, 2148 del_and_ins(key, T, Size2, Tuple, Key, 1), 2149 del_and_ins(object, T, Size2, Tuple, Key, 1), 2150 del_and_ins(both, T, Size2, Tuple, Key, 1), 2151 2152 %% Overwrite an objekt on file with the same object. 2153 ok = dets:insert(T, Tuple), 2154 ok = dets:sync(T), 2155 [Tuple2] = dets:lookup(T, Key), 2156 true = dets:member(T, Key), 2157 ok = dets:insert(T, Tuple), 2158 ok = dets:sync(T), 2159 [Tuple2] = dets:lookup(T, Key), 2160 true = dets:member(T, Key), 2161 2162 %% A mix of insert and delete. 2163 ok = dets:delete(T, Key), 2164 ok = dets:sync(T), 2165 ok = dets:delete(T, Key), 2166 ok = dets:insert(T, {Key,foo}), 2167 ok = dets:insert(T, {Key,bar}), 2168 [{Key,bar},{Key,foo}] = sort(dets:lookup(T, Key)), 2169 true = dets:member(T, Key), 2170 ok = dets:delete_object(T, {Key,foo}), 2171 ok = dets:insert(T, {Key,kar}), 2172 [{Key,bar},{Key,kar}] = sort(dets:lookup(T, Key)), 2173 true = dets:member(T, Key), 2174 ok = dets:insert(T, [{Key,kar},{Key,kar}]), 2175 [{Key,bar},{Key,kar}] = sort(dets:lookup(T, Key)), 2176 true = dets:member(T, Key), 2177 ok = dets:delete_object(T, {Key,bar}), 2178 ok = dets:delete_object(T, {Key,kar}), 2179 [] = dets:lookup(T, Key), 2180 false = dets:member(T, Key), 2181 ok = dets:sync(T), 2182 [] = dets:lookup(T, Key), 2183 false = dets:member(T, Key), 2184 2185 OtherValues = sort(lookup_keys(T, OtherKeys)), 2186 if 2187 Extra -> 2188 %% Let the table grow for a while, if it needs to. 2189 All1 = get_all_objects(T), 2190 dets:safe_fixtable(T, false), 2191 timer:sleep(1200), 2192 OtherValues = sort(lookup_keys(T, OtherKeys)), 2193 dets:safe_fixtable(T, true), 2194 All2 = get_all_objects(T), 2195 FAll2 = get_all_objects_fast(T), 2196 true = sort(All2) =:= sort(FAll2), 2197 case symdiff(All1, All2) of 2198 {[],[]} -> ok; 2199 {X,Y} -> 2200 NoBad = length(X) + length(Y), 2201 ct:fail({bags,DelayedWrite,Extra,Sz,NoBad}) 2202 end; 2203 true -> 2204 ok 2205 end, 2206 ok = dets:close(T), 2207 file:delete(Fname), 2208 2209 %% Second object of a key added and looked up simultaneously. 2210 R1 = {index_test,1,2,3,4}, 2211 R2 = {index_test,2,2,13,14}, 2212 R3 = {index_test,1,12,13,14}, 2213 {ok, _} = dets:open_file(T,[{type,bag}, {keypos,2},{file,Fname}]), 2214 ok = dets:insert(T,R1), 2215 ok = dets:sync(T), 2216 ok = dets:insert(T,R2), 2217 ok = dets:sync(T), 2218 ok = dets:insert(T,R3), 2219 [R1,R3] = sort(dets:lookup(T,1)), 2220 true = dets:member(T, 1), 2221 [R1,R3] = sort(dets:lookup(T,1)), 2222 true = dets:member(T, 1), 2223 ok = dets:close(T), 2224 file:delete(Fname), 2225 2226 check_pps(P0), 2227 ok. 2228 2229%% Test the write cache for duplicate bags. 2230cache_duplicate_bags(Config) when is_list(Config) -> 2231 Small = 2, 2232 cache_dup_bags(Config, {0,0}, false, Small), 2233 cache_dup_bags(Config, {0,0}, true, Small), 2234 cache_dup_bags(Config, {5000,5000}, false, Small), 2235 cache_dup_bags(Config, {5000,5000}, true, Small), 2236 %% Objects of size greater than 2 kB. 2237 Big = 1200, 2238 cache_dup_bags(Config, {0,0}, false, Big), 2239 cache_dup_bags(Config, {0,0}, true, Big), 2240 cache_dup_bags(Config, {5000,5000}, false, Big), 2241 cache_dup_bags(Config, {5000,5000}, true, Big). 2242 2243cache_dup_bags(Config, DelayedWrite, Extra, Sz) -> 2244 %% Extra = bool(). Insert tuples until the tested key is not alone. 2245 %% Sz = integer(). Size of the inserted tuples. 2246 2247 T = cache, 2248 Fname = filename(cache, Config), 2249 file:delete(Fname), 2250 P0 = pps(), 2251 2252 {ok, _} = dets:open_file(T,[{file,Fname}, {type,duplicate_bag}, 2253 {delayed_write, DelayedWrite}]), 2254 2255 Dups = 2, 2256 {Key, OtherKeys} = 2257 if 2258 Extra -> 2259 %% Insert enough to get three keys in some slot. 2260 dets:safe_fixtable(T, true), 2261 insert_objs(T, 1, Sz, Dups); 2262 true -> 2263 {1,[]} 2264 end, 2265 Tuple = erlang:make_tuple(Sz, Key), 2266 ok = dets:delete(T, Key), 2267 ok = dets:sync(T), 2268 false = dets:member(T, Key), 2269 2270 %% The values of keys in the same slot as Key are checked. 2271 OtherValues = sort(lookup_keys(T, OtherKeys)), 2272 2273 ok = dets:insert(T, Tuple), 2274 [Tuple] = dets:lookup(T, Key), 2275 true = dets:member(T, Key), 2276 ok = dets:insert(T, [Tuple,Tuple]), 2277 %% If no delay, the cache gets filled immediately, and written. 2278 [Tuple,Tuple,Tuple] = dets:lookup_keys(T, [Key,a,b,c,d,e,f]), 2279 true = dets:member(T, Key), 2280 2281 %% If delay, this happens without file access. 2282 %% (This is no longer true; cache lookup has been simplified.) 2283 ok = dets:delete(T,Key), 2284 ok = dets:insert(T,Tuple), 2285 ok = dets:insert(T,Tuple), 2286 [Tuple,Tuple] = dets:lookup(T, Key), 2287 true = dets:member(T, Key), 2288 ok = dets:sync(T), 2289 [Tuple,Tuple] = dets:lookup(T, Key), 2290 true = dets:member(T, Key), 2291 2292 %% One object in the cache, one on the file. 2293 ok = dets:delete(T,Key), 2294 ok = dets:insert(T,Tuple), 2295 ok = dets:sync(T), 2296 ok = dets:insert(T,Tuple), 2297 true = dets:member(T, Key), % should not read the file, but it does.. 2298 2299 %% Key's objects are is on file only, 2300 %% key toto in the cache (if there is one). 2301 ok = dets:delete(T,toto), 2302 ok = dets:insert(T,[{toto,b},{toto,b}]), 2303 true = sort([Tuple,Tuple,{toto,b},{toto,b}]) =:= 2304 sort(dets:lookup_keys(T, [Key,toto])), 2305 true = dets:member(T, toto), 2306 Size = dets:info(T, size), 2307 2308 %% Two objects with the same key on the file. 2309 %% Delete them, add two objects. 2310 del_and_ins(key, T, Size, Tuple, Key, 2), 2311 2312 del_and_ins(object, T, Size, Tuple, Key, 2), 2313 del_and_ins(both, T, Size, Tuple, Key, 2), 2314 2315 %% Two objects with the same key on the file. 2316 %% Delete them, add three objects. 2317 del_and_ins(key, T, Size, Tuple, Key, 3), 2318 del_and_ins(object, T, Size, Tuple, Key, 3), 2319 del_and_ins(both, T, Size, Tuple, Key, 3), 2320 2321 %% Two objects with the same key on the file. 2322 %% Delete them, add one object. 2323 del_and_ins(key, T, Size, Tuple, Key, 1), 2324 del_and_ins(object, T, Size, Tuple, Key, 1), 2325 del_and_ins(both, T, Size, Tuple, Key, 1), 2326 2327 OtherValues = sort(lookup_keys(T, OtherKeys)), 2328 if 2329 Extra -> 2330 %% Let the table grow for a while, if it needs to. 2331 All1 = get_all_objects(T), 2332 dets:safe_fixtable(T, false), 2333 timer:sleep(1200), 2334 OtherValues = sort(lookup_keys(T, OtherKeys)), 2335 dets:safe_fixtable(T, true), 2336 All2 = get_all_objects(T), 2337 FAll2 = get_all_objects_fast(T), 2338 true = sort(All2) =:= sort(FAll2), 2339 case symdiff(All1, All2) of 2340 {[],[]} -> ok; 2341 {X,Y} -> 2342 NoBad = length(X) + length(Y), 2343 ct:fail({dup_bags,DelayedWrite,Extra,Sz,NoBad}) 2344 end; 2345 true -> 2346 ok 2347 end, 2348 ok = dets:close(T), 2349 2350 file:delete(Fname), 2351 check_pps(P0), 2352 ok. 2353 2354lookup_keys(_T, []) -> 2355 []; 2356lookup_keys(T, Keys) -> 2357 dets:lookup_keys(T, Keys). 2358 2359del_and_ins(W, T, Size, Obj, Key, N) -> 2360 case W of 2361 object -> 2362 ok = dets:delete_object(T, Obj); 2363 key -> 2364 2365 ok = dets:delete(T, Key); 2366 both -> 2367 ok = dets:delete(T, Key), 2368 ok = dets:delete_object(T, Obj) 2369 end, 2370 Objs = duplicate(N, Obj), 2371 [] = dets:lookup(T, Key), 2372 ok = dets:insert(T, Objs), 2373 Objs = dets:lookup_keys(T, [snurrespratt,Key]), 2374 true = Size + length(Objs)-2 =:= dets:info(T, size), 2375 Objs = dets:lookup(T, Key). 2376 2377 2378insert_objs(T, N, Sz, Dups) -> 2379 Seq = seq(N,N+255), 2380 L0 = map(fun(I) -> erlang:make_tuple(Sz, I) end, Seq), 2381 L = append(duplicate(Dups, L0)), 2382 ok = dets:insert(T, L), 2383 case search_slot(T, 0) of 2384 false -> 2385 insert_objs(T, N+256, Sz, Dups); 2386 Keys -> 2387 Keys 2388 end. 2389 2390search_slot(T, I) -> 2391 case dets:slot(T, I) of 2392 '$end_of_table' -> 2393 false; 2394 Objs -> 2395 case usort(map(fun(X) -> element(1, X) end, Objs)) of 2396 [_, Key, _ | _] = Keys0 -> 2397 Keys = delete(Key, Keys0), 2398 {Key, Keys}; 2399 _ -> 2400 search_slot(T, I+1) 2401 end 2402 end. 2403 2404symdiff(L1, L2) -> 2405 {X, _, Y} = 2406 sofs:symmetric_partition(sofs:set(L1), sofs:set(L2)), 2407 {sofs:to_external(X), sofs:to_external(Y)}. 2408 2409%% Test read-only tables and traversal caused crashes. 2410otp_4208(Config) when is_list(Config) -> 2411 Tab = otp_4208, 2412 FName = filename(Tab, Config), 2413 Expected = sort([{3,ghi,12},{1,abc,10},{4,jkl,13},{2,def,11}]), 2414 2415 file:delete(FName), 2416 {ok, Tab} = dets:open_file(Tab, [{file,FName}]), 2417 ok = dets:insert(Tab, [{1,abc,10},{2,def,11},{3,ghi,12},{4,jkl,13}]), 2418 Expected = sort(dets:traverse(Tab, fun(X) -> {continue, X} end)), 2419 ok = dets:close(Tab), 2420 2421 {ok, Tab} = dets:open_file(Tab, [{access, read},{file,FName}]), 2422 Expected = sort(dets:traverse(Tab, fun(X) -> {continue, X} end)), 2423 ok = dets:close(Tab), 2424 file:delete(FName), 2425 2426 ok. 2427 2428%% Test read-only tables and growth. 2429otp_4989(Config) when is_list(Config) -> 2430 Tab = otp_4989, 2431 FName = filename(Tab, Config), 2432 2433 %% Do exactly as in the error report. 2434 _Ets = ets:new(Tab, [named_table]), 2435 ets_init(Tab, 100000), 2436 {ok, Tab} = 2437 dets:open_file(Tab, [{access, read_write}, {file,FName}, {keypos,2}]), 2438 ok = dets:from_ets(Tab, Tab), 2439 ok = dets:close(Tab), 2440 %% Restore. 2441 {ok, Tab} = 2442 dets:open_file(Tab, [{access, read}, {keypos, 2}, {file, FName}]), 2443 true = ets:delete_all_objects(Tab), 2444 true = ets:from_dets(Tab, Tab), 2445 ok = dets:close(Tab), 2446 ets:delete(Tab), 2447 file:delete(FName), 2448 ok. 2449 2450ets_init(_Tab, 0) -> 2451 ok; 2452ets_init(Tab, N) -> 2453 ets:insert(Tab, {N,N}), 2454 ets_init(Tab, N - 1). 2455 2456%% OTP-8898. Truncated Dets file. 2457otp_8898(Config) when is_list(Config) -> 2458 Tab = otp_8898, 2459 FName = filename(Tab, Config), 2460 2461 Server = self(), 2462 2463 file:delete(FName), 2464 {ok, _} = dets:open_file(Tab,[{file, FName}]), 2465 [P1,P2,P3] = new_clients(3, Tab), 2466 2467 Seq = [{P1,[sync]},{P2,[{lookup,1,[]}]},{P3,[{insert,{1,b}}]}], 2468 atomic_requests(Server, Tab, [[]], Seq), 2469 true = get_replies([{P1,ok},{P2,ok},{P3,ok}]), 2470 ok = dets:close(Tab), 2471 {ok, _} = dets:open_file(Tab,[{file, FName}]), 2472 file:delete(FName), 2473 2474 ok. 2475 2476%% OTP-8899. Several clients. Updated Head was ignored. 2477otp_8899(Config) when is_list(Config) -> 2478 Tab = many_clients, 2479 FName = filename(Tab, Config), 2480 2481 Server = self(), 2482 2483 file:delete(FName), 2484 {ok, _} = dets:open_file(Tab,[{file, FName}]), 2485 [P1,P2,P3,P4] = new_clients(4, Tab), 2486 2487 MC = [Tab], 2488 Seq6a = [{P1,[{insert,[{used_to_be_skipped_by,match}]}, 2489 {lookup,1,[{1,a}]}]}, 2490 {P2,[{verbose,true,MC}]}, 2491 {P3,[{lookup,1,[{1,a}]}]}, {P4,[{verbose,true,MC}]}], 2492 atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq6a), 2493 true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]), 2494 [{1,a},{2,b},{3,c},{used_to_be_skipped_by,match}] = 2495 lists:sort(dets:match_object(Tab, '_')), 2496 _ = dets:close(Tab), 2497 file:delete(FName), 2498 2499 ok. 2500 2501%% Test several clients accessing a table simultaneously. 2502many_clients(Config) when is_list(Config) -> 2503 Tab = many_clients, 2504 FName = filename(Tab, Config), 2505 2506 Server = self(), 2507 2508 file:delete(FName), 2509 P0 = pps(), 2510 {ok, _} = dets:open_file(Tab,[{file, FName}]), 2511 [P1,P2,P3,P4] = new_clients(4, Tab), 2512 2513 %% dets:init_table/2 is used for making sure that all processes 2514 %% start sending requests before the Dets process begins to handle 2515 %% them; the requests arrive "in parallel". 2516 2517 %% Four processes accessing the same table at almost the same time. 2518 2519 %% One key is read, updated, and read again. 2520 Seq1 = [{P1,[{lookup,1,[{1,a}]}]}, {P2,[{insert,{1,b}}]}, 2521 {P3,[{lookup,1,[{1,b}]}]}, {P4,[{lookup,1,[{1,b}]}]}], 2522 atomic_requests(Server, Tab, [[{1,a}]], Seq1), 2523 true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]), 2524 2525 %% Different keys read by different processes 2526 Seq2 = [{P1,[{member,1,true}]}, {P2,[{lookup,2,[{2,b}]}]}, 2527 {P3,[{lookup,1,[{1,a}]}]}, {P4,[{lookup,3,[{3,c}]}]}], 2528 atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq2), 2529 true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]), 2530 2531 %% Reading deleted key. 2532 Seq3 = [{P1,[{delete_key,2}]}, {P2,[{lookup,1,[{1,a}]}]}, 2533 {P3,[{lookup,1,[{1,a}]}]}, {P4,[{member,2,false}]}], 2534 atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq3), 2535 true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]), 2536 2537 %% Inserting objects. 2538 Seq4 = [{P1,[{insert,[{1,a},{2,b}]}]}, {P2,[{insert,[{2,c},{3,a}]}]}, 2539 {P3,[{insert,[{3,b},{4,d}]}]}, 2540 {P4,[{lookup_keys,[1,2,3,4],[{1,a},{2,c},{3,b},{4,d}]}]}], 2541 atomic_requests(Server, Tab, [], Seq4), 2542 true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]), 2543 2544 %% Deleting objects. 2545 Seq5 = [{P1,[{delete_object,{1,a}}]}, {P2,[{delete_object,{1,a}}]}, 2546 {P3,[{delete_object,{3,c}}]}, 2547 {P4,[{lookup_keys,[1,2,3,4],[{2,b}]}]}], 2548 atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq5), 2549 true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]), 2550 2551 %% Some request not streamed. 2552 Seq6 = [{P1,[{lookup,1,[{1,a}]}]}, {P2,[{info,size,3}]}, 2553 {P3,[{lookup,1,[{1,a}]}]}, {P4,[{info,size,3}]}], 2554 atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq6), 2555 true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]), 2556 2557 %% Some request not streamed. 2558 Seq7 = [{P1,[{insert,[{3,a}]}]}, {P2,[{insert,[{3,b}]}]}, 2559 {P3,[{delete_object,{3,c}}]}, 2560 {P4,[{lookup,3,[{3,b}]}]}], 2561 atomic_requests(Server, Tab, [[{3,c}]], Seq7), 2562 true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]), 2563 2564 put_requests(Server, [{P1,stop},{P2,stop},{P3,stop},{P4,stop}]), 2565 ok = dets:close(Tab), 2566 file:delete(FName), 2567 2568 %% Check that errors are handled correctly by the streaming operators. 2569 {ok, _} = dets:open_file(Tab,[{file, FName}]), 2570 ok = ins(Tab, 100), 2571 Obj = {66,{item,number,66}}, 2572 {ok, ObjPos} = dets:where(Tab, Obj), 2573 ok = dets:close(Tab), 2574 %% Damaged object. 2575 crash(FName, ObjPos+12), 2576 {ok, _} = dets:open_file(Tab,[{file, FName}]), 2577 BadObject1 = dets:lookup_keys(Tab, [65,66,67,68,69]), 2578 bad_object(BadObject1, FName), 2579 _Error = dets:close(Tab), 2580 file:delete(FName), 2581 2582 check_pps(P0), 2583 2584 ok. 2585 2586%% Tab is initiated with the objects in Objs. Objs = [[object()]]. 2587atomic_requests(Server, Tab, Objs, Req) -> 2588 ok = dets:init_table(Tab, atomic_requests(Server, Objs, Req)). 2589 2590atomic_requests(Server, L, Req) -> 2591 fun(close) -> 2592 ok; 2593 (read) when [] =:= L -> 2594 put_requests(Server, Req), 2595 end_of_input; 2596 (read) -> 2597 [E | Es] = L, 2598 {E, atomic_requests(Server, Es, Req)} 2599 end. 2600 2601put_requests(Server, L) -> 2602 lists:foreach(fun({Pid,R}) -> Pid ! {Server,R}, timer:sleep(1) end, L). 2603 2604get_replies(L) -> 2605 lists:all(fun({Pid,Reply}) -> Reply =:= get_reply(Pid) end, L). 2606 2607get_reply(Pid) -> 2608 receive {Pid, Reply} -> Reply end. 2609 2610new_clients(0, _Tab) -> 2611 []; 2612new_clients(N, Tab) -> 2613 [new_client(Tab) | new_clients(N-1, Tab)]. 2614 2615new_client(Tab) -> 2616 spawn(?MODULE, client, [self(), Tab]). 2617 2618client(S, Tab) -> 2619 receive 2620 {S, stop} -> 2621 exit(normal); 2622 {S, ToDo} -> 2623 Reply = eval(ToDo, Tab), 2624 case Reply of 2625 {error, _} -> io:format("~p: ~p~n", [self(), Reply]); 2626 _ -> ok 2627 end, 2628 S ! {self(), Reply} 2629 end, 2630 client(S, Tab). 2631 2632eval([], _Tab) -> 2633 ok; 2634eval([{verbose,Bool,Expected} | L], Tab) -> 2635 case dets:verbose(Bool) of 2636 Expected -> eval(L, Tab); 2637 Error -> {error, {verbose,Error}} 2638 end; 2639eval([sync | L], Tab) -> 2640 case dets:sync(Tab) of 2641 ok -> eval(L, Tab); 2642 Error -> {error, {sync,Error}} 2643 end; 2644eval([{insert,Stuff} | L], Tab) -> 2645 case dets:insert(Tab, Stuff) of 2646 ok -> eval(L, Tab); 2647 Error -> {error, {insert,Stuff,Error}} 2648 end; 2649eval([{lookup,Key,Expected} | L], Tab) -> 2650 case dets:lookup(Tab, Key) of 2651 Expected -> eval(L, Tab); 2652 Else -> {error, {lookup,Key,Expected,Else}} 2653 end; 2654eval([{lookup_keys,Keys,Expected} | L], Tab) -> 2655 %% Time order is destroyed... 2656 case dets:lookup_keys(Tab, Keys) of 2657 R when is_list(R) -> 2658 case lists:sort(Expected) =:= lists:sort(R) of 2659 true -> eval(L, Tab); 2660 false -> {error, {lookup_keys,Keys,Expected,R}} 2661 end; 2662 Else -> {error, {lookup_keys,Keys,Expected,Else}} 2663 end; 2664eval([{member,Key,Expected} | L], Tab) -> 2665 case dets:member(Tab, Key) of 2666 Expected -> eval(L, Tab); 2667 Else -> {error, {member,Key,Expected,Else}} 2668 end; 2669eval([{delete_key,Key} | L], Tab) -> 2670 case dets:delete(Tab, Key) of 2671 ok -> eval(L, Tab); 2672 Else -> {error, {delete_key,Key,Else}} 2673 end; 2674eval([{delete_object,Object} | L], Tab) -> 2675 case dets:delete_object(Tab, Object) of 2676 ok -> eval(L, Tab); 2677 Else -> {error, {delete_object,Object,Else}} 2678 end; 2679eval([{info,Tag,Expected} | L], Tab) -> 2680 case dets:info(Tab, Tag) of 2681 Expected -> eval(L, Tab); 2682 Else -> {error, {info,Tag,Else,Expected}} 2683 end; 2684eval(Else, _Tab) -> 2685 {error, {bad_request,Else}}. 2686 2687%% More than 128k keys caused crash. 2688otp_4906(Config) when is_list(Config) -> 2689 N = 256*512 + 400, 2690 Tab = otp_4906, 2691 FName = filename(Tab, Config), 2692 2693 file:delete(FName), 2694 {ok, Tab} = dets:open_file(Tab, [{file, FName}]), 2695 ok = ins_small(Tab, 0, N), 2696 ok = dets:close(Tab), 2697 {ok, Tab} = dets:open_file(Tab, [{file, FName}]), 2698 ok = read_4906(Tab, N-1), 2699 ok = dets:close(Tab), 2700 file:delete(FName), 2701 2702 %% If the (only) process fixing a table updates the table, the 2703 %% process will no longer be punished with a 1 ms delay (hm, the 2704 %% server is delayed, it should be the client...). In this example 2705 %% the writing process *is* delayed. 2706 {ok,Tab} = dets:open_file(Tab, [{file,FName}]), 2707 Parent = self(), 2708 FixPid = spawn_link(fun() -> 2709 dets:safe_fixtable(Tab, true), 2710 receive {Parent, stop} -> ok end 2711 end), 2712 ok = ins_small(Tab, 0, 1000), 2713 FixPid ! {Parent, stop}, 2714 timer:sleep(1), 2715 ok = dets:close(Tab), 2716 file:delete(FName), 2717 ok. 2718 2719read_4906(_T, N) when N < 0 -> 2720 ok; 2721read_4906(T, N) -> 2722 [_] = dets:lookup(T, N), 2723 read_4906(T, N-1). 2724 2725ins_small(_T, I, N) when I =:= N -> 2726 ok; 2727ins_small(T, I, N) -> 2728 ok = dets:insert(T, {I}), 2729 ins_small(T, I+1, N). 2730 2731%% Unwritable ramfile caused crash. 2732otp_5402(Config) when is_list(Config) -> 2733 Tab = otp_5402, 2734 File = filename:join(["cannot", "write", "this", "file"]), 2735 2736 %% close 2737 {ok, T} = dets:open_file(Tab, [{ram_file,true}, 2738 {file, File}]), 2739 ok = dets:insert(T, {1,a}), 2740 {error,{file_error,_,_}} = dets:close(T), 2741 2742 %% sync 2743 {ok, T} = dets:open_file(Tab, [{ram_file,true}, 2744 {file, File}]), 2745 ok = dets:insert(T, {1,a}), 2746 {error,{file_error,_,_}} = dets:sync(T), 2747 {error,{file_error,_,_}} = dets:close(T), 2748 2749 %% auto_save 2750 {ok, T} = dets:open_file(Tab, [{ram_file,true}, 2751 {auto_save, 2000}, 2752 {file, File}]), 2753 ok = dets:insert(T, {1,a}), 2754 timer:sleep(5000), 2755 {error,{file_error,_,_}} = dets:close(T), 2756 ok. 2757 2758%% Several clients open and close tables simultaneously. 2759simultaneous_open(Config) -> 2760 Tab = sim_open, 2761 File = filename(Tab, Config), 2762 2763 ok = monit(Tab, File), 2764 case feasible() of 2765 false -> {comment, "OK, but did not run all of the test"}; 2766 true -> 2767 ok = kill_while_repairing(Tab, File), 2768 ok = kill_while_init(Tab, File), 2769 ok = open_ro(Tab, File), 2770 ok = open_w(Tab, File, 0, Config), 2771 ok = open_w(Tab, File, 100, Config) 2772 end. 2773 2774feasible() -> 2775 LP = erlang:system_info(logical_processors), 2776 (is_integer(LP) 2777 andalso LP >= erlang:system_info(schedulers_online) 2778 andalso not erlang:system_info(debug_compiled) 2779 andalso not erlang:system_info(lock_checking)). 2780 2781%% One process logs and another process closes the log. Before 2782%% monitors were used, this would make the client never return. 2783monit(Tab, File) -> 2784 file:delete(File), 2785 {ok, Tab} = dets:open_file(Tab, [{file,File}]), 2786 F1 = fun() -> dets:close(Tab) end, 2787 F2 = fun() -> {'EXIT', {badarg, _}} = do_log(Tab) end, 2788 spawn(F2), 2789 timer:sleep(100), 2790 spawn(F1), 2791 dets:close(Tab), 2792 ok = file:delete(File). 2793 2794do_log(Tab) -> 2795 case catch dets:insert(Tab, {hej,san,sa}) of 2796 ok -> do_log(Tab); 2797 Else -> Else 2798 end. 2799 2800%% Kill the Dets process while repair is in progress. 2801kill_while_repairing(Tab, File) -> 2802 create_opened_log(File), 2803 Delay = 1000, 2804 dets:start(), 2805 Parent = self(), 2806 F = fun() -> 2807 R = (catch dets:open_file(Tab, [{file,File}])), 2808 timer:sleep(Delay), 2809 Parent ! {self(), R} 2810 end, 2811 %% One of these will open the file, the other will be pending 2812 %% until the file has been repaired: 2813 P1 = spawn(F), 2814 P2 = spawn(F), 2815 P3 = spawn(F), 2816 DetsPid = find_dets_pid(), 2817 exit(DetsPid, kill), 2818 2819 receive {P1,R1} -> R1 end, 2820 receive {P2,R2} -> R2 end, 2821 receive {P3,R3} -> R3 end, 2822 io:format("Killed pid: ~p~n", [DetsPid]), 2823 io:format("Remaining Dets-pids (should be nil): ~p~n", 2824 [find_dets_pids()]), 2825 {replies,[{'EXIT', {dets_process_died, _}}, {ok,_}, {ok, _}]} = 2826 {replies,lists:sort([R1, R2, R3])}, 2827 2828 timer:sleep(200), 2829 case dets:info(Tab) of 2830 undefined -> 2831 ok; 2832 _Info -> 2833 timer:sleep(5000), 2834 undefined = dets:info(Tab) 2835 end, 2836 2837 file:delete(File), 2838 ok. 2839 2840find_dets_pid() -> 2841 case find_dets_pids() of 2842 [] -> 2843 timer:sleep(100), 2844 find_dets_pid(); 2845 [Pid] -> 2846 Pid 2847 end. 2848 2849find_dets_pids() -> 2850 lists:filter(fun(P) -> dets:pid2name(P) =/= undefined end, 2851 erlang:processes()). 2852 2853%% Kill the Dets process when there are users and an on-going 2854%% initiailization. 2855kill_while_init(Tab, File) -> 2856 file:delete(File), 2857 Parent = self(), 2858 F = fun() -> 2859 R = dets:open_file(Tab, [{file,File}]), 2860 Parent ! {self(), R}, 2861 receive {Parent, die} -> ok end, 2862 {error, not_owner} = dets:close(Tab) 2863 end, 2864 P1 = spawn(F), 2865 P2 = spawn(F), 2866 P3 = spawn(F), 2867 IF = fun() -> 2868 R = dets:open_file(Tab, [{file,File}]), 2869 Parent ! {self(), R}, 2870 Fun = fun(_) -> timer:sleep(100000) end, 2871 {'EXIT', {badarg, _}} = (catch dets:init_table(Tab, Fun)), 2872 receive {Parent, die} -> ok end 2873 end, 2874 P4 = spawn(IF), 2875 receive {P1,R1} -> {ok, _} = R1 end, 2876 receive {P2,R2} -> {ok, _} = R2 end, 2877 receive {P3,R3} -> {ok, _} = R3 end, 2878 receive {P4,R4} -> {ok, _} = R4 end, 2879 DetsPid = find_dets_pid(), 2880 exit(DetsPid, kill), 2881 2882 timer:sleep(1000), 2883 undefined = dets:info(Tab), 2884 P1 ! {Parent, die}, 2885 P2 ! {Parent, die}, 2886 P3 ! {Parent, die}, 2887 P4 ! {Parent, die}, 2888 2889 file:delete(File), 2890 timer:sleep(100), 2891 ok. 2892 2893open_ro(Tab, File) -> 2894 create_opened_log(File), 2895 Delay = 1000, 2896 Parent = self(), 2897 F = fun() -> 2898 R = dets:open_file(Tab, [{file,File},{access,read}]), 2899 timer:sleep(Delay), 2900 Parent ! {self(), R} 2901 end, 2902 P1 = spawn(F), 2903 P2 = spawn(F), 2904 P3 = spawn(F), 2905 2906 receive {P1,R1} -> {error,{not_closed,_}} = R1 end, 2907 receive {P2,R2} -> {error,{not_closed,_}} = R2 end, 2908 receive {P3,R3} -> {error,{not_closed,_}} = R3 end, 2909 ok. 2910 2911open_w(Tab, File, Delay, Config) -> 2912 create_opened_log(File), 2913 2914 Tab2 = t2, 2915 File2 = filename(Tab2, Config), 2916 file:delete(File2), 2917 {ok,Tab2} = dets:open_file(Tab2, [{file,File2}]), 2918 ok = dets:close(Tab2), 2919 2920 Parent = self(), 2921 F = fun() -> 2922 R = dets:open_file(Tab, [{file,File}]), 2923 timer:sleep(Delay), 2924 Parent ! {self(), R} 2925 end, 2926 Pid1 = spawn(F), 2927 Pid2 = spawn(F), 2928 Pid3 = spawn(F), 2929 2930 ok = wait_for_repair_to_start(Tab), 2931 2932 %% It is assumed that it takes some time to repair the file. 2933 {ok,Tab2} = dets:open_file(Tab2, [{file,File2}]), 2934 %% The Dets server managed to handle to open_file request. 2935 0 = qlen(), % still repairing 2936 2937 ok = dets:close(Tab2), 2938 file:delete(File2), 2939 2940 receive {Pid1,R1} -> {ok, Tab} = R1 end, 2941 receive {Pid2,R2} -> {ok, Tab} = R2 end, 2942 receive {Pid3,R3} -> {ok, Tab} = R3 end, 2943 timer:sleep(200), 2944 case dets:info(Tab) of 2945 undefined -> 2946 ok; 2947 _Info -> 2948 timer:sleep(5000), 2949 undefined = dets:info(Tab) 2950 end, 2951 2952 file:delete(File), 2953 ok. 2954 2955wait_for_repair_to_start(Tab) -> 2956 case catch dets_server:get_pid(Tab) of 2957 {'EXIT', _} -> 2958 timer:sleep(1), 2959 wait_for_repair_to_start(Tab); 2960 Pid when is_pid(Pid) -> 2961 ok 2962 end. 2963 2964qlen() -> 2965 {_, {_, N}} = lists:keysearch(message_queue_len, 1, process_info(self())), 2966 N. 2967 2968create_opened_log(File) -> 2969 Tab = t, 2970 file:delete(File), 2971 {ok, Tab} = dets:open_file(Tab, [{file,File}]), 2972 ok = ins(Tab, 60000), 2973 ok = dets:close(Tab), 2974 crash(File, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED), 2975 ok. 2976 2977%% OTP-5075. insert_new/2 2978insert_new(Config) -> 2979 Tab = insert_new, 2980 File = filename(Tab, Config), 2981 file:delete(File), 2982 {ok, T} = dets:open_file(Tab, [{file,File}]), 2983 {'EXIT', {badarg, _}} = (catch dets:insert_new(Tab, 14)), 2984 {'EXIT', {badarg, _}} = (catch dets:insert_new(Tab, {})), 2985 true = dets:insert_new(Tab, {1,a}), 2986 false = dets:insert_new(Tab, {1,a}), 2987 true = dets:insert_new(Tab, [{2,b}, {3,c}]), 2988 false = dets:insert_new(Tab, [{2,b}, {3,c}]), 2989 false = dets:insert_new(Tab, [{1,a}, {4,d}]), 2990 ok = dets:close(Tab), 2991 2992 file:delete(File), 2993 {ok, T} = dets:open_file(Tab, [{file,File},{type,bag}]), 2994 true = dets:insert_new(Tab, {1,a}), 2995 false = dets:insert_new(Tab, {1,b}), 2996 true = dets:insert_new(Tab, [{2,b}, {3,c}]), 2997 false = dets:insert_new(Tab, [{2,a}, {3,d}]), 2998 false = dets:insert_new(Tab, [{1,a}, {4,d}]), 2999 ok = dets:close(Tab), 3000 3001 3002 file:delete(File), 3003 ok. 3004 3005%% OTP-5126. repair_continuation/2 3006repair_continuation(Config) -> 3007 Tab = repair_continuation_table, 3008 Fname = filename(repair_cont, Config), 3009 file:delete(Fname), 3010 {ok, _} = dets:open_file(Tab, [{file,Fname}]), 3011 ok = dets:insert(Tab, [{1,a},{2,b},{3,c}]), 3012 3013 MS = [{'_',[],[true]}], 3014 3015 SRes = term_to_binary(dets:select(Tab, MS, 1)), 3016 %% Get rid of compiled match spec 3017 lists:foreach(fun (P) -> 3018 garbage_collect(P) 3019 end, processes()), 3020 {[true], C2} = binary_to_term(SRes), 3021 3022 {'EXIT', {badarg, _}} = (catch dets:select(C2)), 3023 C3 = dets:repair_continuation(C2, MS), 3024 {[true], C4} = dets:select(C3), 3025 C5 = dets:repair_continuation(C4, MS), 3026 {[true], _} = dets:select(C5), 3027 {'EXIT', {badarg, _}} = (catch dets:repair_continuation(Tab, bu)), 3028 3029 ok = dets:close(Tab), 3030 file:delete(Fname), 3031 ok. 3032 3033%% OTP-5487. Growth of read-only table (again). 3034otp_5487(Config) -> 3035 Tab = otp_5487, 3036 Fname = filename(otp_5487, Config), 3037 file:delete(Fname), 3038 Ets = ets:new(otp_5487, [public, set]), 3039 lists:foreach(fun(I) -> ets:insert(Ets, {I,I+1}) end, 3040 lists:seq(0,1000)), 3041 {ok, _} = dets:open_file(Tab, [{file,Fname}]), 3042 ok = dets:from_ets(Tab, Ets), 3043 ok = dets:sync(Tab), 3044 ok = dets:close(Tab), 3045 {ok, _} = dets:open_file(Tab, [{file,Fname},{access,read}]), 3046 [{1,2}] = dets:lookup(Tab, 1), 3047 ok = dets:close(Tab), 3048 ets:delete(Ets), 3049 file:delete(Fname). 3050 3051%% OTP-6206. Badly formed free lists. 3052otp_6206(Config) -> 3053 Tab = otp_6206, 3054 File = filename(Tab, Config), 3055 3056 file:delete(File), 3057 Options = [{file,File}], 3058 {ok, Tab} = dets:open_file(Tab, Options), 3059 NObjs = 13006, 3060 ok = ins(Tab, NObjs), 3061 ok = del(Tab, NObjs, 2), 3062 ok = dets:close(Tab), 3063 3064 %% Used to return {badmatch,{error,{bad_freelists,File}}. 3065 {ok, Tab} = dets:open_file(Tab, [{repair,false}|Options]), 3066 ok = dets:close(Tab), 3067 file:delete(File), 3068 ok. 3069 3070%% OTP-6359. select and match never return the empty list. 3071otp_6359(Config) -> 3072 Tab = otp_6359, 3073 File = filename(Tab, Config), 3074 3075 file:delete(File), 3076 {ok, _} = dets:open_file(Tab, [{file, File}]), 3077 %% Used to return {[], Cont}: 3078 '$end_of_table' = dets:match(Tab, '_', 100), 3079 ok = dets:close(Tab), 3080 file:delete(File), 3081 ok. 3082 3083%% OTP-4738. ==/2 and =:=/2. 3084otp_4738(Config) -> 3085 otp_4738_set(Config), 3086 otp_4738_bag(Config), 3087 otp_4738_dupbag(Config), 3088 ok. 3089 3090otp_4738_dupbag(Config) -> 3091 Tab = otp_4738, 3092 File = filename(Tab, Config), 3093 file:delete(File), 3094 I = -12857447, 3095 F = float(I), 3096 One = 1, 3097 FOne = float(One), 3098 Args = [{file,File},{type,duplicate_bag}], 3099 {ok, Tab} = dets:open_file(Tab, Args), 3100 ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]), 3101 ok = dets:sync(Tab), 3102 [{F,One},{F,FOne}] = dets:lookup(Tab, F), 3103 [{I,One},{I,FOne}] = dets:lookup(Tab, I), 3104 ok = dets:insert(Tab, [{F,One},{F,FOne}]), 3105 [{I,One},{I,FOne},{F,One},{F,FOne},{F,One},{F,FOne}] = 3106 dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3107 ok = dets:insert(Tab, [{F,FOne},{F,One}]), 3108 [{I,One},{I,FOne},{F,One},{F,FOne},{F,One}, 3109 {F,FOne},{F,FOne},{F,One}] = 3110 dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3111 ok = dets:delete_object(Tab, {I,FOne}), 3112 [{I,One},{F,One},{F,FOne},{F,One},{F,FOne},{F,FOne},{F,One}] = 3113 dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3114 ok = dets:insert(Tab, {I,FOne}), 3115 [{I,One},{I,FOne},{F,One},{F,FOne},{F,One}, 3116 {F,FOne},{F,FOne},{F,One}] = 3117 dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3118 ok = dets:delete_object(Tab, {F,FOne}), 3119 [{I,One},{I,FOne},{F,One},{F,One},{F,One}] = 3120 dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3121 ok = dets:delete(Tab, F), 3122 [{I,One},{I,FOne}] = dets:match_object(Tab, '_'), 3123 ok = dets:close(Tab), 3124 file:delete(File), 3125 3126 Zero = 0, 3127 FZero = float(Zero), 3128 {ok, Tab} = dets:open_file(Tab, Args), 3129 ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]), 3130 ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]), 3131 ok = dets:insert(Tab, [{I,Zero},{F,Zero},{I,FZero},{I,FZero}]), 3132 Objs0 = dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3133 ok = dets:close(Tab), 3134 crash(File, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED), 3135 io:format("Expect repair:~n"), 3136 {ok, Tab} = dets:open_file(Tab, Args), 3137 Objs1 = dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3138 ok = dets:close(Tab), 3139 Objs1 = Objs0, 3140 file:delete(File), 3141 ok. 3142 3143otp_4738_bag(Config) -> 3144 Tab = otp_4738, 3145 File = filename(Tab, Config), 3146 file:delete(File), 3147 I = -12857447, 3148 F = float(I), 3149 One = 1, 3150 FOne = float(One), 3151 Args = [{file,File},{type,bag}], 3152 {ok, Tab} = dets:open_file(Tab, Args), 3153 ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]), 3154 ok = dets:sync(Tab), 3155 [{F,One},{F,FOne}] = dets:lookup(Tab, F), 3156 [{I,One},{I,FOne}] = dets:lookup(Tab, I), 3157 ok = dets:insert(Tab, [{F,One},{F,FOne}]), 3158 [{I,One},{I,FOne},{F,One},{F,FOne}] = 3159 dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3160 ok = dets:insert(Tab, [{F,FOne},{F,One}]), 3161 [{I,One},{I,FOne},{F,FOne},{F,One}] = 3162 dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3163 ok = dets:delete_object(Tab, {I,FOne}), 3164 [{I,One},{F,FOne},{F,One}] = 3165 dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3166 ok = dets:insert(Tab, {I,FOne}), 3167 [{I,One},{I,FOne},{F,FOne},{F,One}] = 3168 dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3169 ok = dets:delete(Tab, F), 3170 [{I,One},{I,FOne}] = dets:match_object(Tab, '_'), 3171 ok = dets:close(Tab), 3172 file:delete(File). 3173 3174otp_4738_set(Config) -> 3175 Tab = otp_4738, 3176 File = filename(Tab, Config), 3177 file:delete(File), 3178 Args = [{file,File},{type,set}], 3179 3180 %% I and F share the same slot. 3181 I = -12857447, 3182 F = float(I), 3183 {ok, Tab} = dets:open_file(Tab, Args), 3184 ok = dets:insert(Tab, [{I},{F}]), 3185 ok = dets:sync(Tab), 3186 [{F}] = dets:lookup(Tab, F), 3187 [{I}] = dets:lookup(Tab, I), 3188 ok = dets:insert(Tab, [{F}]), 3189 [{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3190 ok = dets:close(Tab), 3191 file:delete(File), 3192 3193 {ok, Tab} = dets:open_file(Tab, Args), 3194 ok = dets:insert(Tab, [{I}]), 3195 ok = dets:sync(Tab), 3196 [] = dets:lookup(Tab, F), 3197 [{I}] = dets:lookup(Tab, I), 3198 ok = dets:insert(Tab, [{F}]), 3199 [{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3200 ok = dets:close(Tab), 3201 file:delete(File), 3202 3203 {ok, Tab} = dets:open_file(Tab, Args), 3204 ok = dets:insert(Tab, [{I},{F}]), 3205 %% {insert, ...} in the cache, try lookup: 3206 [{F}] = dets:lookup(Tab, F), 3207 [{I}] = dets:lookup(Tab, I), 3208 %% Both were found, but that cannot be verified. 3209 [{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3210 ok = dets:close(Tab), 3211 file:delete(File), 3212 3213 {ok, Tab} = dets:open_file(Tab, Args), 3214 ok = dets:insert(Tab, [{I}]), 3215 ok = dets:sync(Tab), 3216 ok = dets:insert(Tab, [{F}]), 3217 %% {insert, ...} in the cache, try lookup: 3218 [{F}] = dets:lookup(Tab, F), 3219 [{I}] = dets:lookup(Tab, I), 3220 [{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3221 ok = dets:close(Tab), 3222 file:delete(File), 3223 3224 {ok, Tab} = dets:open_file(Tab, Args), 3225 %% Both operations in the cache: 3226 ok = dets:insert(Tab, [{I}]), 3227 ok = dets:insert(Tab, [{F}]), 3228 [{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')), 3229 ok = dets:close(Tab), 3230 file:delete(File), 3231 ok. 3232 3233%% OTP-7146. Bugfix: missing test when re-hashing. 3234otp_7146(Config) -> 3235 Tab = otp_7146, 3236 File = filename(Tab, Config), 3237 file:delete(File), 3238 3239 Max = 2048, 3240 {ok, Tab} = dets:open_file(Tab, [{max_no_slots,Max}, {file,File}]), 3241 write_dets(Tab, Max), 3242 ok = dets:close(Tab), 3243 3244 file:delete(File), 3245 ok. 3246 3247write_dets(Tab, Max) -> 3248 write_dets(Tab, 0, Max). 3249 3250write_dets(_Tab, N, Max) when N > Max -> 3251 ok; 3252write_dets(Tab, N, Max) -> 3253 ok = dets:insert(Tab,{ N, {entry,N}}), 3254 write_dets(Tab, N+1, Max). 3255 3256%% OTP-8070. Duplicated objects with insert_new() and duplicate_bag. 3257otp_8070(Config) when is_list(Config) -> 3258 Tab = otp_8070, 3259 File = filename(Tab, Config), 3260 file:delete(File), 3261 {ok, _} = dets:open_file(Tab, [{file,File},{type, duplicate_bag}]), 3262 ok = dets:insert(Tab, [{3,0}]), 3263 false = dets:insert_new(Tab, [{3,1},{3,1}]), 3264 [{3,0}] = dets:lookup(Tab, 3), 3265 ok = dets:close(Tab), 3266 file:delete(File), 3267 ok. 3268 3269%% OTP-8856. insert_new() bug. 3270otp_8856(Config) when is_list(Config) -> 3271 Tab = otp_8856, 3272 File = filename(Tab, Config), 3273 file:delete(File), 3274 Me = self(), 3275 {ok, _} = dets:open_file(Tab, [{type, bag}, {file, File}]), 3276 spawn(fun()-> Me ! {1, dets:insert(Tab, [])} end), 3277 spawn(fun()-> Me ! {2, dets:insert_new(Tab, [])} end), 3278 receive {1, ok} -> ok end, 3279 receive {2, true} -> ok end, 3280 ok = dets:close(Tab), 3281 file:delete(File), 3282 3283 {ok, _} = dets:open_file(Tab, [{type, set}, {file, File}]), 3284 spawn(fun() -> dets:delete(Tab, 0) end), 3285 spawn(fun() -> Me ! {3, dets:insert_new(Tab, {0,0})} end), 3286 receive {3, true} -> ok end, 3287 ok = dets:close(Tab), 3288 file:delete(File), 3289 ok. 3290 3291%% OTP-8903. bchunk/match/select bug. 3292otp_8903(Config) when is_list(Config) -> 3293 Tab = otp_8903, 3294 File = filename(Tab, Config), 3295 {ok,T} = dets:open_file(bug, [{file,File}]), 3296 ok = dets:insert(T, [{1,a},{2,b},{3,c}]), 3297 dets:safe_fixtable(T, true), 3298 {[_],C1} = dets:match_object(T, '_', 1), 3299 {BC1,_D} = dets:bchunk(T, start), 3300 ok = dets:close(T), 3301 {'EXIT', {badarg, _}} = (catch {foo,dets:match_object(C1)}), 3302 {'EXIT', {badarg, _}} = (catch {foo,dets:bchunk(T, BC1)}), 3303 {ok,T} = dets:open_file(bug, [{file,File}]), 3304 false = dets:info(T, safe_fixed), 3305 {'EXIT', {badarg, _}} = (catch {foo,dets:match_object(C1)}), 3306 {'EXIT', {badarg, _}} = (catch {foo,dets:bchunk(T, BC1)}), 3307 ok = dets:close(T), 3308 file:delete(File), 3309 ok. 3310 3311%% OTP-8923. rehash due to lookup after initialization. 3312otp_8923(Config) when is_list(Config) -> 3313 Tab = otp_8923, 3314 File = filename(Tab, Config), 3315 %% Create a file with more than 256 keys: 3316 file:delete(File), 3317 Bin = list_to_binary([ 0 || _ <- lists:seq(1, 400) ]), 3318 BigBin = list_to_binary([ 0 ||_ <- lists:seq(1, 4000)]), 3319 Ets = ets:new(temp, [{keypos,1}]), 3320 [ true = ets:insert(Ets, {C,Bin}) || C <- lists:seq(1, 700) ], 3321 true = ets:insert(Ets, {helper_data,BigBin}), 3322 true = ets:insert(Ets, {prim_btree,BigBin}), 3323 true = ets:insert(Ets, {sec_btree,BigBin}), 3324 %% Note: too few slots; re-hash will take place 3325 {ok, Tab} = dets:open_file(Tab, [{file,File}]), 3326 Tab = ets:to_dets(Ets, Tab), 3327 ok = dets:close(Tab), 3328 true = ets:delete(Ets), 3329 3330 {ok,Ref} = dets:open_file(File), 3331 [{1,_}] = dets:lookup(Ref, 1), 3332 ok = dets:close(Ref), 3333 3334 {ok,Ref2} = dets:open_file(File), 3335 [{helper_data,_}] = dets:lookup(Ref2, helper_data), 3336 ok = dets:close(Ref2), 3337 3338 file:delete(File), 3339 ok. 3340 3341%% OTP-9282. The name of a table can be an arbitrary term. 3342otp_9282(Config) when is_list(Config) -> 3343 some_calls(make_ref(), Config), 3344 some_calls({a,typical,name}, Config), 3345 some_calls(fun() -> a_funny_name end, Config), 3346 ok. 3347 3348some_calls(Tab, Config) -> 3349 File = filename(ref, Config), 3350 {ok,T} = dets:open_file(Tab, [{file,File}]), 3351 T = Tab, 3352 false = dets:info(T, safe_fixed), 3353 File = dets:info(T, filename), 3354 ok = dets:insert(Tab, [{3,0}]), 3355 [{3,0}] = dets:lookup(Tab, 3), 3356 [{3,0}] = dets:traverse(Tab, fun(X) -> {continue, X} end), 3357 ok = dets:close(T), 3358 file:delete(File). 3359 3360 3361%% OTP-11245. Tables remained fixed after traversal. 3362otp_11245(Config) when is_list(Config) -> 3363 Tab = otp_11245, 3364 File = filename(Tab, Config), 3365 {ok, Tab} = dets:open_file(Tab, [{file,File}]), 3366 N = 1024, 3367 ins(Tab, N), 3368 N = length(dets:match(Tab, '_')), 3369 false = dets:info(Tab, safe_fixed), 3370 dets:traverse(Tab, fun(_) -> continue end), 3371 false = dets:info(Tab, safe_fixed), 3372 N = dets:foldl(fun(_, N2) -> N2+1 end, 0, Tab), 3373 false = dets:info(Tab, safe_fixed), 3374 N = dets:foldr(fun(_, N2) -> N2+1 end, 0, Tab), 3375 false = dets:info(Tab, safe_fixed), 3376 ok = dets:close(Tab), 3377 file:delete(File), 3378 ok. 3379 3380%% OTP-11709. Bugfixes. 3381otp_11709(Config) when is_list(Config) -> 3382 Short = <<"foo">>, 3383 Long = <<"a sufficiently long text">>, 3384 3385 %% Bug: leaking file descriptor 3386 P0 = pps(), 3387 File = filename(otp_11709, Config), 3388 ok = file:write_file(File, Long), 3389 false = dets:is_dets_file(File), 3390 check_pps(P0), 3391 3392 %% Bug: deleting file 3393 Args = [[{access, A}, {repair, R}] || 3394 A <- [read, read_write], 3395 R <- [true, false, force]], 3396 Fun1 = fun(S, As) -> 3397 P1 = pps(), 3398 ok = file:write_file(File, S), 3399 {error,{not_a_dets_file,File}} = dets:open_file(File, As), 3400 {ok, S} = file:read_file(File), 3401 check_pps(P1) 3402 end, 3403 Fun2 = fun(S) -> 3404 _ = [Fun1(S, As) || As <- Args], 3405 ok 3406 end, 3407 ok = Fun2(Long), % no change here 3408 ok = Fun2(Short), % mimic the behaviour for longer files 3409 3410 %% open_file/1 3411 ok = file:write_file(File, Long), 3412 {error,{not_a_dets_file,File}} = dets:open_file(File), % no change 3413 ok = file:write_file(File, Short), 3414 {error,{not_a_dets_file,File}} = dets:open_file(File), % mimic 3415 3416 _ = file:delete(File), 3417 ok. 3418 3419%% OTP-13229. open_file() exits with badarg when given binary file name. 3420%% Also OTP-15253. 3421otp_13229(_Config) -> 3422 F = <<"binfile.tab">>, 3423 try dets:open_file(name, [{file, F}]) of 3424 R -> 3425 exit({open_succeeded, R}) 3426 catch 3427 error:badarg -> 3428 ok 3429 end, 3430 try dets:open_file(F, []) of % OTP-15253 3431 R2 -> 3432 exit({open_succeeded, R2}) 3433 catch 3434 error:badarg -> 3435 ok 3436 end, 3437 try dets:open_file(F) of 3438 R3 -> 3439 exit({open_succeeded, R3}) 3440 catch 3441 error:badarg -> 3442 ok 3443 end. 3444 3445%% OTP-13260. Race when opening a table. 3446otp_13260(Config) -> 3447 [ok] = lists:usort([otp_13260_1(Config) || _ <- lists:seq(1, 3)]), 3448 ok. 3449 3450otp_13260_1(Config) -> 3451 Tab = otp_13260, 3452 File = filename(Tab, Config), 3453 N = 20, 3454 P = self(), 3455 Pids = [spawn_link(fun() -> counter(P, Tab, File) end) || 3456 _ <- lists:seq(1, N)], 3457 Rs = rec(Pids), 3458 true = lists:all(fun(R) -> is_integer(R) end, Rs), 3459 wait_for_close(Tab). 3460 3461rec([]) -> 3462 []; 3463rec([Pid | Pids]) -> 3464 receive {Pid, R} -> 3465 [R | rec(Pids)] 3466 end. 3467 3468%% One may have to run the test several times to trigger the bug. 3469counter(P, Tab, File) -> 3470 Key = key, 3471 N = case catch dets:update_counter(Tab, Key, 1) of 3472 {'EXIT', _} -> 3473 {ok, Tab} = dets:open_file(Tab, [{file, File}]), 3474 ok = dets:insert(Tab, {Key, 1}), 3475 dets:update_counter(Tab, Key, 1); 3476 N1 when is_integer(N1) -> 3477 N1; 3478 DetsBug -> 3479 DetsBug 3480 end, 3481 P ! {self(), N}. 3482 3483wait_for_close(Tab) -> 3484 case dets:info(Tab, owner) of 3485 undefined -> 3486 ok; 3487 _ -> 3488 timer:sleep(100), 3489 wait_for_close(Tab) 3490 end. 3491 3492%% OTP-13830. Format 8 is no longer supported. 3493otp_13830(Config) -> 3494 Tab = otp_13830, 3495 File8 = filename:join(?datadir(Config), "version_8.dets"), 3496 {error,{format_8_no_longer_supported,_}} = 3497 dets:open_file(Tab, [{file, File8}]), 3498 File = filename(Tab, Config), 3499 %% Check the 'version' option, for backwards compatibility: 3500 {ok, Tab} = dets:open_file(Tab, [{file, File}, {version, 9}]), 3501 ok = dets:close(Tab), 3502 {ok, Tab} = dets:open_file(Tab, [{file, File}, {version, default}]), 3503 ok = dets:close(Tab). 3504 3505%% 3506%% Parts common to several test cases 3507%% 3508 3509crash(File, Where) -> 3510 crash(File, Where, 10). 3511 3512crash(File, Where, What) when is_integer(What) -> 3513 {ok, Fd} = file:open(File, [read,write]), 3514 file:position(Fd, Where), 3515 ok = file:write(Fd, [What]), 3516 ok = file:close(Fd). 3517 3518args(Config) -> 3519 {Sets, Bags, Dups} = 3520 {[ 3521 [], 3522 [{type, set}, {estimated_no_objects, 300}, 3523 {ram_file, true}], 3524 [{type, set}, {estimated_no_objects, 300}], 3525 [{type, set}, {estimated_no_objects, 300}], 3526 [{auto_save,20}, {type, set}, 3527 {estimated_no_objects, 300}] 3528 ], 3529 3530 [ 3531 [{type, bag}, {estimated_no_objects, 300}, {ram_file, true}], 3532 [{type, bag}], 3533 [{type, bag}, {estimated_no_objects, 300}], 3534 [{type, bag}, {estimated_no_objects, 300}], 3535 [{type, bag}, 3536 {auto_save,20}, {estimated_no_objects, 300}], 3537 [{type, bag}, {estimated_no_objects, 300}, {ram_file, true}] 3538 ], 3539 3540 [ 3541 [{type, duplicate_bag}, {estimated_no_objects, 300}, 3542 {ram_file, true}], 3543 [{type, duplicate_bag}], 3544 [{type, duplicate_bag}, {estimated_no_objects, 300}], 3545 [{type, duplicate_bag}, {estimated_no_objects, 300}], 3546 [{type, duplicate_bag}, 3547 {auto_save,20}, {estimated_no_objects, 300}], 3548 [{type, duplicate_bag}, {estimated_no_objects, 300}, 3549 {ram_file, true}] 3550 ] 3551 }, 3552 zip_filename(Sets, Bags, Dups, Config). 3553 3554zip_filename(S, B, D, Conf) -> 3555 zip_filename(S, B, D, [], [], [], 1, Conf). 3556 3557zip_filename([H|T], B, D, S1, B1, D1, I, Conf) -> 3558 zip_filename(T, B, D, [[{file, new_filename(I, Conf)} | H] | S1], 3559 B1, D1, I+1, Conf); 3560zip_filename([], [H|B], D, S1, B1, D1, I, Conf) -> 3561 zip_filename([], B, D, S1, [[{file, new_filename(I, Conf)} | H] | B1], 3562 D1, I+1, Conf); 3563zip_filename([], [], [H|T], S1, B1, D1, I, Conf) -> 3564 zip_filename([], [], T, S1, B1, [[{file, new_filename(I, Conf)} | H] | D1], 3565 I+1, Conf); 3566zip_filename([], [], [], S1, B1, D1, _, _Conf) -> 3567 {reverse(S1), reverse(B1), reverse(D1)}. 3568 3569del_test(Tab) -> 3570 ?format("Deltest on ~p~n", [Tab]), 3571 Objs = safe_get_all_objects(Tab), 3572 Keys = map(fun(X) -> element(1, X) end, Objs), 3573 foreach(fun(Key) -> dets:delete(Tab, Key) end, Keys), 3574 0 = length(get_all_objects(Tab)), 3575 [] = get_all_objects_fast(Tab), 3576 0 = dets:info(Tab, size). 3577 3578del_obj_test(Tab) -> 3579 ?format("Delobjtest on ~p~n", [Tab]), 3580 Objs = safe_get_all_objects(Tab), 3581 LL = length(Objs), 3582 LL = dets:info(Tab, size), 3583 foreach(fun(Obj) -> dets:delete_object(Tab, Obj) end, Objs), 3584 0 = length(get_all_objects(Tab)), 3585 [] = get_all_objects_fast(Tab), 3586 0 = dets:info(Tab, size). 3587 3588match_del_test(Tab) -> 3589 ?format("Match delete test on ~p~n", [Tab]), 3590 ok = dets:match_delete(Tab, {'_','_','_'}), 3591 Sz = dets:info(Tab, size), 3592 true = Sz =:= length(dets:match_object(Tab, '_')), 3593 ok = dets:match_delete(Tab, '_'), 3594 0 = dets:info(Tab, size), 3595 0 = length(get_all_objects(Tab)), 3596 [] = get_all_objects_fast(Tab). 3597 3598trav_test(_Data, Len, Tab) -> 3599 ?format("Travtest on ~p~n", [Tab]), 3600 _X0 = dets:traverse(Tab, fun(_X) -> continue end), 3601 XX = dets:traverse(Tab, fun(X) -> {continue, X} end), 3602 case Len =:= length(XX) of 3603 false -> ?format("DIFF ~p~n", [XX -- _Data]); 3604 true -> ok 3605 end, 3606 1 = length(dets:traverse(Tab, fun(X) -> {done, X} end)). 3607 3608match_test(Data, Tab) -> 3609 ?format("Match test on ~p~n", [Tab]), 3610 Data1 = sort(filter(fun(X) when tuple_size(X) =:= 3 -> true; 3611 (_X) -> false 3612 end, Data)), 3613 Data1 = sort(dets:match_object(Tab, {'$1', '$2', '$3'})), 3614 3615 Len = length(Data), 3616 Len = length(dets:match(Tab, '_')), 3617 Len2 = length(Data1), 3618 Len2 = length(dets:match(Tab, {'$1', '_', '_'})), 3619 3620 Data3 = 3621 filter(fun(X) -> 3622 K = element(1, X), 3623 if 3624 tuple_size(X) =:= 3, tuple_size(K) =:= 2 -> true; 3625 true -> false 3626 end 3627 end, Data), 3628 Len3 = length(Data3), 3629 Len3 = length(dets:match(Tab, {{'$1', '$2'}, '_', '_'})), 3630 Len3 = length(dets:match_object(Tab, {{'$1', '$2'}, '_', '_'})), 3631 3632 R = make_ref(), 3633 dets:insert(Tab, {{R, R}, 33 ,44}), 3634 1 = length(dets:match(Tab, {{R, R}, '_', '_'})), 3635 1 = length(dets:match_object(Tab, {{R, R}, '_', '_'})). 3636 3637%% 3638%% Utilities 3639%% 3640 3641headsz() -> 3642 ?HEADSZ_v9. 3643 3644unwritable(Fname) -> 3645 {ok, Info} = file:read_file_info(Fname), 3646 Mode = Info#file_info.mode - 8#00200, 3647 file:write_file_info(Fname, Info#file_info{mode = Mode}). 3648 3649writable(Fname) -> 3650 {ok, Info} = file:read_file_info(Fname), 3651 Mode = Info#file_info.mode bor 8#00200, 3652 file:write_file_info(Fname, Info#file_info{mode = Mode}). 3653 3654truncate(File, Where) -> 3655 {ok, Fd} = file:open(File, [read,write]), 3656 file:position(Fd, Where), 3657 ok = file:truncate(Fd), 3658 ok = file:close(Fd). 3659 3660new_filename(Name, _Config) when is_integer(Name) -> 3661 filename:join(?privdir(_Config), 3662 integer_to_list(Name) ++ ".DETS"). 3663 3664filename(Name, Config) when is_atom(Name) -> 3665 filename(atom_to_list(Name), Config); 3666filename(Name, _Config) -> 3667 filename:join(?privdir(_Config), Name). 3668 3669open_files(_Name, []) -> 3670 []; 3671open_files(Name0, [Args | Tail]) -> 3672 ?format("init ~p~n", [Args]), 3673 Name = list_to_atom(integer_to_list(Name0)), 3674 {ok, Name} = dets:open_file(Name, Args), 3675 [Name | open_files(Name0+1, Tail)]. 3676 3677close_all(Tabs) -> foreach(fun(Tab) -> ok = dets:close(Tab) end, Tabs). 3678 3679delete_files(Args) -> 3680 Fun = fun(F) -> 3681 {value, {file, File}} = keysearch(file, 1, F), 3682 file:delete(File), 3683 File 3684 end, 3685 map(Fun, Args). 3686 3687%% Initialize all tables 3688initialize(Tabs, Data) -> 3689 foreach(fun(Tab) -> 3690 Fun = fun(Obj) -> ok = dets:insert(Tab, Obj) end, 3691 foreach(Fun, Data), 3692 dets:sync(Tab) 3693 end, Tabs). 3694 3695%% need more than 512 objects to really trig overflow 3696make_data(Kp) -> 3697 make_data(Kp, set). 3698 3699make_data(Kp, Type) -> 3700 dup(Type, make_data(Kp, Type, 520)). 3701 3702dup(duplicate_bag, [H1, H2 |T]) -> 3703 [H1,H2, H1, H2 | dup(duplicate_bag, T)]; 3704dup(_, Other) -> 3705 Other. 3706 3707make_data(_Kp, Type, 0) -> 3708 odd_keys(Type); 3709make_data(1, set, I) -> 3710 [{I, q,w} | make_data(1, set, I-1)]; 3711make_data(2, set, I) -> 3712 [{hh, I, q,w} | make_data(2, set, I-1)]; 3713make_data(1, bag, I) -> 3714 [{I, q,w} , {I, hah, 77} | make_data(1, bag, I-1)]; 3715make_data(2, bag, I) -> 3716 [{hh, I, q,w} , {hh, I, lalal, 900} | make_data(2, bag, I-1)]; 3717make_data(1, duplicate_bag, I) -> 3718 [{I, q,w} , {I, hah, 77} | make_data(1, duplicate_bag, I-1)]; 3719make_data(2, duplicate_bag, I) -> 3720 [{hh, I, q,w} , {hh, I, lalal, 900} | make_data(2, duplicate_bag, I-1)]. 3721 3722odd_keys(_) -> 3723 [{foo, 1 ,2}, 3724 {{foo, foo}, 2,3}, 3725 {"kakaka", {{{}}}, jj}, 3726 {{"kallll", "kkk", []}, 66.7777}, 3727 {make_ref(), 99, 66}, 3728 {{1},2,3,4,5,6,7,duplicate(50, 8)}, 3729 {self(), 7,8,88}, 3730 {[self()], 8, 11}]. 3731 3732 3733ins(_T, 0) -> 3734 ok; 3735ins(T, N) -> 3736 case dets:insert(T, {N, item(N)}) of 3737 ok -> ins(T, N-1); 3738 Error -> Error 3739 end. 3740 3741item(N) when N rem 2 =:= 0 -> 3742 {item, number, N}; 3743item(N) -> 3744 {item, number, N, a, much, bigger, one, i, think}. 3745 3746del(_T, N, _I) when N =< 0 -> 3747 ok; 3748del(T, N, I) -> 3749 ok = dets:delete(T, N), 3750 del(T, N-I, I). 3751 3752ensure_node(0, _Node) -> 3753 could_not_start_node; 3754ensure_node(N, Node) -> 3755 case net_adm:ping(Node) of 3756 pang -> 3757 receive after 1000 -> 3758 ok 3759 end, 3760 ensure_node(N-1,Node); 3761 pong -> 3762 ok 3763 end. 3764 3765size_test(Len, Tabs) -> 3766 foreach(fun(Tab) -> 3767 Len = dets:info(Tab, size) 3768 end, Tabs). 3769 3770no_keys_test([T | Ts]) -> 3771 no_keys_test(T), 3772 no_keys_test(Ts); 3773no_keys_test([]) -> 3774 ok; 3775no_keys_test(T) -> 3776 Kp = dets:info(T, keypos), 3777 All = dets:match_object(T, '_'), 3778 L = lists:map(fun(X) -> element(Kp, X) end, All), 3779 NoKeys = length(lists:usort(L)), 3780 case {dets:info(T, no_keys), NoKeys} of 3781 {N, N} -> 3782 ok; 3783 {N1, N2} -> 3784 exit({no_keys_test, N1, N2}) 3785 end. 3786 3787safe_get_all_objects(Tab) -> 3788 dets:safe_fixtable(Tab, true), 3789 Objects = get_all_objects(Tab), 3790 dets:safe_fixtable(Tab, false), 3791 Objects. 3792 3793%% Caution: unless the table has been fixed, strange results can be returned. 3794get_all_objects(Tab) -> get_all_objects(dets:first(Tab), Tab, []). 3795 3796%% Assuming no key matches {error, Reason}... 3797get_all_objects('$end_of_table', _Tab, L) -> L; 3798get_all_objects({error, Reason}, _Tab, _L) -> 3799 exit({get_all_objects, {error, Reason}}); 3800get_all_objects(Key, Tab, L) -> 3801 Objs = dets:lookup(Tab, Key), 3802 get_all_objects(dets:next(Tab, Key), Tab, Objs ++ L). 3803 3804count_objects_quite_fast(Tab) -> 3805 R1 = dets:match_object(Tab, '_', 1), 3806 count_objs_1(R1, 0). 3807 3808count_objs_1('$end_of_table', N) -> 3809 N; 3810count_objs_1({Ts,C}, N) when is_list(Ts) -> 3811 count_objs_1(dets:match_object(C), length(Ts) + N). 3812 3813get_all_objects_fast(Tab) -> 3814 dets:match_object(Tab, '_'). 3815 3816histogram(Tab) -> 3817 OnePercent = case dets:info(Tab, no_slots) of 3818 undefined -> undefined; 3819 {_, NoSlots, _} -> NoSlots/100 3820 end, 3821 histogram(Tab, OnePercent). 3822 3823histogram(Tab, OnePercent) -> 3824 E = ets:new(histo, []), 3825 dets:safe_fixtable(Tab, true), 3826 Hist = histo(Tab, E, 0, OnePercent, OnePercent), 3827 dets:safe_fixtable(Tab, false), 3828 case Hist of 3829 ok -> 3830 H = ets:tab2list(E), 3831 true = ets:delete(E), 3832 sort(H); 3833 Error -> 3834 ets:delete(E), 3835 Error 3836 end. 3837 3838histo(T, E, I, One, Count) when is_number(Count), I > Count -> 3839 io:format("."), 3840 histo(T, E, I, One, Count+One); 3841histo(T, E, I, One, Count) -> 3842 case dets:slot(T, I) of 3843 '$end_of_table' when is_number(Count) -> 3844 io:format("~n"), 3845 ok; 3846 '$end_of_table' -> 3847 ok; 3848 Objs when is_list(Objs) -> 3849 L = length(Objs), 3850 case catch ets:update_counter(E, L, 1) of 3851 {'EXIT', _} -> 3852 ets:insert(E, {L, 1}); 3853 _ -> 3854 ok 3855 end, 3856 histo(T, E, I+1, One, Count); 3857 Error -> 3858 Error 3859 end. 3860 3861sum_histogram(H) -> 3862 sum_histogram(H, 0). 3863 3864sum_histogram([{S,N1} | H], N) -> 3865 sum_histogram(H, N + S*N1); 3866sum_histogram([], N) -> 3867 N. 3868 3869ave_histogram(H) -> 3870 ave_histogram(H, 0)/sum_histogram(H). 3871 3872ave_histogram([{S,N1} | H], N) -> 3873 ave_histogram(H, N + S*S*N1); 3874ave_histogram([], N) -> 3875 N. 3876 3877bad_object({error,{{bad_object,_}, FileName}}, FileName) -> 3878 ok; % No debug. 3879bad_object({error,{{{bad_object,_,_},_,_,_}, FileName}}, FileName) -> 3880 ok. % Debug. 3881 3882check_badarg({'EXIT', {badarg, [{M,F,Args,_} | _]}}, M, F, Args) -> 3883 true. 3884 3885check_pps({Ports0,Procs0} = P0) -> 3886 ok = check_dets_tables(), 3887 case pps() of 3888 P0 -> 3889 ok; 3890 _ -> 3891 %% On some (rare) occasions the dets process is still 3892 %% running although the call to close() has returned, as 3893 %% it seems... 3894 timer:sleep(500), 3895 case pps() of 3896 P0 -> 3897 ok; 3898 {Ports1,Procs1} = P1 -> 3899 case {Ports1 -- Ports0, Procs1 -- Procs0} of 3900 {[], []} -> ok; 3901 {PortsDiff,ProcsDiff} -> 3902 io:format("failure, got ~p~n, expected ~p\n", [P1, P0]), 3903 show("Old port", Ports0 -- Ports1), 3904 show("New port", PortsDiff), 3905 show("Old proc", Procs0 -- Procs1), 3906 show("New proc", ProcsDiff), 3907 ct:fail(failed) 3908 end 3909 end 3910 end. 3911 3912%% Copied from dets_server.erl: 3913-define(REGISTRY, dets_registry). 3914-define(OWNERS, dets_owners). 3915-define(STORE, dets). 3916 3917check_dets_tables() -> 3918 Store = [T || 3919 T <- ets:all(), 3920 ets:info(T, name) =:= ?STORE, 3921 owner(T) =:= dets], 3922 S = case Store of 3923 [Tab] -> ets:tab2list(Tab); 3924 [] -> [] 3925 end, 3926 case {ets:tab2list(?REGISTRY), ets:tab2list(?OWNERS), S} of 3927 {[], [], []} -> ok; 3928 {R, O, _} -> 3929 io:format("Registry: ~p~n", [R]), 3930 io:format("Owners: ~p~n", [O]), 3931 io:format("Store: ~p~n", [S]), 3932 not_ok 3933 end. 3934 3935owner(Tab) -> 3936 Owner = ets:info(Tab, owner), 3937 case process_info(Owner, registered_name) of 3938 {registered_name, Name} -> Name; 3939 _ -> Owner 3940 end. 3941 3942show(_S, []) -> 3943 ok; 3944show(S, [{Pid, Name, InitCall}|Pids]) when is_pid(Pid) -> 3945 io:format("~s: ~w (~w), ~w: ~p~n", 3946 [S, Pid, proc_reg_name(Name), InitCall, 3947 erlang:process_info(Pid)]), 3948 show(S, Pids); 3949show(S, [{Port, _}|Ports]) when is_port(Port)-> 3950 io:format("~s: ~w: ~p~n", [S, Port, erlang:port_info(Port)]), 3951 show(S, Ports). 3952 3953pps() -> 3954 dets:start(), 3955 {port_list(), process_list()}. 3956 3957port_list() -> 3958 [{P,safe_second_element(erlang:port_info(P, name))} || 3959 P <- erlang:ports()]. 3960 3961process_list() -> 3962 [{P,process_info(P, registered_name), 3963 safe_second_element(process_info(P, initial_call))} || 3964 P <- processes()]. 3965 3966proc_reg_name({registered_name, Name}) -> Name; 3967proc_reg_name([]) -> no_reg_name. 3968 3969safe_second_element({_,Info}) -> Info; 3970safe_second_element(Other) -> Other. 3971