1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 1996-2018. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20-module(dets_SUITE).
21
22%%-define(debug, true).
23
24-ifdef(debug).
25-define(format(S, A), io:format(S, A)).
26-define(config(X,Y), foo).
27-define(t, test_server).
28-define(privdir(_), "./dets_SUITE_priv").
29-define(datadir(_), "./dets_SUITE_data").
30-else.
31-include_lib("common_test/include/ct.hrl").
32-define(format(S, A), ok).
33-define(privdir(Conf), proplists:get_value(priv_dir, Conf)).
34-define(datadir(Conf), proplists:get_value(data_dir, Conf)).
35-endif.
36
37-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
38	 init_per_group/2,end_per_group/2, newly_started/1, basic/1,
39         open/1, sets/1, bags/1, duplicate_bags/1, access/1, dirty_mark/1,
40         dirty_mark2/1, bag_next/1, oldbugs/1,
41         truncated_segment_array/1, open_file/1, init_table/1, repair/1,
42	 phash/1, fold/1, fixtable/1, match/1, select/1, update_counter/1,
43         badarg/1, cache_sets/1, cache_bags/1, cache_duplicate_bags/1,
44	 otp_4208/1, otp_4989/1, many_clients/1, otp_4906/1, otp_5402/1,
45         simultaneous_open/1, insert_new/1, repair_continuation/1,
46         otp_5487/1, otp_6206/1, otp_6359/1, otp_4738/1, otp_7146/1,
47         otp_8070/1, otp_8856/1, otp_8898/1, otp_8899/1, otp_8903/1,
48         otp_8923/1, otp_9282/1, otp_11245/1, otp_11709/1, otp_13229/1,
49         otp_13260/1, otp_13830/1]).
50
51-export([dets_dirty_loop/0]).
52
53-export([histogram/1, sum_histogram/1, ave_histogram/1]).
54
55-export([init_per_testcase/2, end_per_testcase/2]).
56
57%% Internal export.
58-export([client/2]).
59
60-import(lists,
61	[append/1, delete/2, duplicate/2, filter/2, foreach/2, keysearch/3,
62	 last/1, map/2, member/2, reverse/1, seq/2, sort/1, usort/1]).
63
64-include_lib("kernel/include/file.hrl").
65
66-define(DETS_SERVER, dets).
67
68%% HEADSZ taken from dets_v9.erl.
69-define(HEADSZ_v9, (56+28*4+16)).
70-define(NO_KEYS_POS_v9, 36).
71-define(CLOSED_PROPERLY_POS, 8).
72
73-define(NOT_PROPERLY_CLOSED,0).
74-define(CLOSED_PROPERLY,1).
75
76init_per_testcase(_Case, Config) ->
77    Config.
78
79end_per_testcase(_Case, _Config) ->
80    ok.
81
82suite() ->
83    [{ct_hooks,[ts_install_cth]},
84     {timetrap,{minutes,15}}].
85
86all() ->
87    [
88	basic, open, sets, bags, duplicate_bags, newly_started, open_file,
89	init_table, repair, access, oldbugs,
90	truncated_segment_array, dirty_mark, dirty_mark2, bag_next,
91        phash, fold, fixtable, match, select, update_counter, badarg,
92	cache_sets, cache_bags, cache_duplicate_bags, otp_4208, otp_4989,
93	many_clients, otp_4906, otp_5402, simultaneous_open,
94	insert_new, repair_continuation, otp_5487, otp_6206,
95	otp_6359, otp_4738, otp_7146, otp_8070, otp_8856, otp_8898,
96	otp_8899, otp_8903, otp_8923, otp_9282, otp_11245, otp_11709,
97        otp_13229, otp_13260, otp_13830
98    ].
99
100groups() ->
101    [].
102
103init_per_suite(Config) ->
104    Config.
105
106end_per_suite(_Config) ->
107    ok.
108
109init_per_group(_GroupName, Config) ->
110    Config.
111
112end_per_group(_GroupName, Config) ->
113    Config.
114
115%% OTP-3621
116newly_started(Config) when is_list(Config) ->
117    true = is_alive(),
118    {ok, Node} = test_server:start_node(slave1, slave, []),
119    [] = rpc:call(Node, dets, all, []),
120    test_server:stop_node(Node),
121    ok.
122
123basic(Config) when is_list(Config) ->
124    Tab = dets_basic_test,
125    FName = filename(Tab, Config),
126
127    P0 = pps(),
128    {ok, _} = dets:open_file(Tab,[{file, FName}]),
129    ok = dets:insert(Tab,{mazda,japan}),
130    ok = dets:insert(Tab,{toyota,japan}),
131    ok = dets:insert(Tab,{suzuki,japan}),
132    ok = dets:insert(Tab,{honda,japan}),
133    ok = dets:insert(Tab,{renault,france}),
134    ok = dets:insert(Tab,{citroen,france}),
135    ok = dets:insert(Tab,{opel,germany}),
136    ok = dets:insert(Tab,{saab,sweden}),
137    ok = dets:insert(Tab,{volvo,sweden}),
138    [{opel,germany}] = dets:lookup(Tab,opel),
139    Japs = dets:traverse(Tab, fun(Obj) ->
140                                      case Obj of
141                                          {_, japan} -> {continue, Obj};
142                                          _ -> continue
143                                      end
144                              end),
145    4  = length(Japs),
146    ok = dets:close(Tab),
147    file:delete(FName),
148    check_pps(P0),
149    ok.
150
151
152open(Config) when is_list(Config) ->
153    %% Running this test twice means that the Dets server is restarted
154    %% twice. dets_sup specifies a maximum of 4 restarts in an hour.
155    %% If this becomes a problem, one should consider running this
156    %% test on a slave node.
157
158    {Sets, Bags, Dups} = args(Config),
159
160    All = Sets ++ Bags ++ Dups,
161    delete_files(All),
162
163    Data = make_data(1),
164
165    P0 = pps(),
166    Tabs = open_files(1, All),
167    initialize(Tabs, Data),
168    check(Tabs, Data),
169
170    foreach(fun(Tab) -> ok = dets:close(Tab) end, Tabs),
171    %% Now reopen the files
172    ?format("Reopening closed files \n", []),
173    Tabs = open_files(1, All),
174    ?format("Checking contents of reopened files \n", []),
175    check(Tabs, Data),
176    %% crash the dets server
177
178    ?format("Crashing dets server \n", []),
179    process_flag(trap_exit, true),
180    Procs = [whereis(?DETS_SERVER) | [dets:info(Tab, pid) || Tab <- Tabs]],
181    foreach(fun(Pid) -> exit(Pid, kill) end, Procs),
182    timer:sleep(100),
183    c:flush(),  %% flush all the EXIT sigs
184    timer:sleep(200),
185
186    %% Now reopen the files again
187    ?format("Reopening crashed files \n", []),
188    open_files(1, All),
189    ?format("Checking contents of repaired files \n", []),
190    check(Tabs, Data),
191
192    close_all(Tabs),
193    delete_files(All),
194
195    {Ports0, Procs0} = P0,
196    Test = fun() ->
197                   P1 = pps(),
198                   {Ports1, Procs1} = P1,
199                   show("Old port", Ports0 -- Ports1),
200                   show("New port", Ports1 -- Ports0),
201                   show("Old procs", Procs0 -- Procs1),
202                   show("New procs", Procs1 -- Procs0),
203                   io:format("Remaining Dets-pids (should be nil): ~p~n",
204                             [find_dets_pids()]),
205                   true = Ports1 =:= Ports0,
206                   %% The dets_server process has been restarted:
207                   [_] = Procs0 -- Procs1,
208                   [_] = Procs1 -- Procs0,
209                   ok
210           end,
211    case catch Test() of
212        ok -> ok;
213        _ ->
214            timer:sleep(500),
215            ok = Test()
216    end.
217
218check(Tabs, Data) ->
219    foreach(fun(Tab) ->
220		    Kp = dets:info(Tab, keypos),
221		    ?format("checking ~p~n", [Tab]),
222		    foreach(fun(Item) ->
223				    case dets:lookup(Tab, k(Kp,Item)) of
224					[Item] -> ok;
225					_Other -> bad(Tab,Item)
226				    end
227			    end, Data)
228	    end, Tabs),
229    ok.
230
231k(Kp, Obj) -> element(Kp, Obj).
232
233bad(_Tab, _Item) ->
234    ?format("Can't find item ~p in ~p ~n", [_Item, _Tab]),
235    exit(badtab).
236
237%% Perform traversal and match testing on set type dets tables.
238sets(Config) when is_list(Config) ->
239    {Sets, _, _} = args(Config),
240
241    Data = make_data(1),
242    delete_files(Sets),
243    P0 = pps(),
244    Tabs = open_files(1, Sets),
245    Bigger = [{17,q,w,w}, {48,q,w,w,w,w,w,w}], % 48 requires a bigger buddy
246    initialize(Tabs, Data++Bigger++Data), % overwrite
247    Len = length(Data),
248    foreach(fun(Tab) -> trav_test(Data, Len, Tab) end, Tabs),
249    size_test(Len, Tabs),
250    no_keys_test(Tabs),
251    foreach(fun(Tab) -> del_test(Tab) end, Tabs),
252    initialize(Tabs, Data),
253    foreach(fun(Tab) -> del_obj_test(Tab) end, Tabs),
254    initialize(Tabs, Data),
255    foreach(fun(Tab) ->
256                    Len = dets:info(Tab, size) end,
257            Tabs),
258    foreach(fun(Tab) -> match_test(Data, Tab) end, Tabs),
259    foreach(fun(Tab) -> match_del_test(Tab) end, Tabs),
260
261    close_all(Tabs),
262    delete_files(Sets),
263    check_pps(P0),
264    ok.
265
266%% Perform traversal and match testing on bag type dets tables.
267bags(Config) when is_list(Config) ->
268    {_, Bags, _} = args(Config),
269    Data = make_data(1, bag),  %% gives twice as many objects
270    delete_files(Bags),
271    P0 = pps(),
272    Tabs = open_files(1, Bags),
273    initialize(Tabs, Data++Data),
274    Len = length(Data),
275    foreach(fun(Tab) -> trav_test(Data, Len, Tab) end, Tabs),
276    size_test(Len, Tabs),
277    no_keys_test(Tabs),
278    foreach(fun(Tab) -> del_test(Tab) end, Tabs),
279    initialize(Tabs, Data),
280    foreach(fun(Tab) -> del_obj_test(Tab) end, Tabs),
281    initialize(Tabs, Data),
282    foreach(fun(Tab) ->
283                    Len = dets:info(Tab, size) end,
284            Tabs),
285    foreach(fun(Tab) -> match_test(Data, Tab) end, Tabs),
286    foreach(fun(Tab) -> match_del_test(Tab) end, Tabs),
287    close_all(Tabs),
288    delete_files(Bags),
289    check_pps(P0),
290    ok.
291
292
293%% Perform traversal and match testing on duplicate_bag type dets tables.
294duplicate_bags(Config) when is_list(Config) ->
295    {_, _, Dups} = args(Config),
296    Data = make_data(1, duplicate_bag), %% gives twice as many objects
297    delete_files(Dups),
298    P0 = pps(),
299    Tabs = open_files(1, Dups),
300    initialize(Tabs, Data),
301    Len = length(Data),
302    foreach(fun(Tab) -> trav_test(Data, Len, Tab) end, Tabs),
303    size_test(Len, Tabs),
304    no_keys_test(Tabs),
305    foreach(fun(Tab) -> del_test(Tab) end, Tabs),
306    initialize(Tabs, Data),
307    foreach(fun(Tab) -> del_obj_test(Tab) end, Tabs),
308    initialize(Tabs, Data),
309    foreach(fun(Tab) ->
310                    Len = dets:info(Tab, size) end,
311            Tabs),
312    foreach(fun(Tab) -> match_test(Data, Tab) end, Tabs),
313    foreach(fun(Tab) -> match_del_test(Tab) end, Tabs),
314    close_all(Tabs),
315    delete_files(Dups),
316    check_pps(P0),
317    ok.
318
319
320access(Config) when is_list(Config) ->
321    Args_acc = [[{ram_file, true}, {access, read}],
322		[{access, read}]],
323    Args = [[{ram_file, true}],
324	    []],
325
326    {Args_acc_1, _, _} = zip_filename(Args_acc, [], [], Config),
327    delete_files(Args_acc_1),
328    {Args_1, _, _} = zip_filename(Args, [], [], Config),
329
330    P0 = pps(),
331    {error, {file_error,_,enoent}} = dets:open_file('1', hd(Args_acc_1)),
332
333    Tabs = open_files(1, Args_1),
334    close_all(Tabs),
335    Tabs = open_files(1, Args_acc_1),
336
337    foreach(fun(Tab) ->
338                    {error, {access_mode,_}} = dets:insert(Tab, {1,2}),
339                    [] = dets:lookup(Tab, 11),
340                    '$end_of_table' = dets:first(Tab),
341                    {error, {access_mode,_}} = dets:delete(Tab, 22)
342            end, Tabs),
343    close_all(Tabs),
344    delete_files(Args_acc_1),
345    check_pps(P0),
346    ok.
347
348
349%% Test that the table is not marked dirty if not written.
350dirty_mark(Config) when is_list(Config) ->
351    true = is_alive(),
352    Tab = dets_dirty_mark_test,
353    FName = filename(Tab, Config),
354    P0 = pps(),
355    dets:open_file(Tab,[{file, FName}]),
356    dets:insert(Tab,{mazda,japan}),
357    dets:insert(Tab,{toyota,japan}),
358    dets:insert(Tab,{suzuki,japan}),
359    dets:insert(Tab,{honda,japan}),
360    dets:insert(Tab,{renault,france}),
361    dets:insert(Tab,{citroen,france}),
362    dets:insert(Tab,{opel,germany}),
363    dets:insert(Tab,{saab,sweden}),
364    dets:insert(Tab,{volvo,sweden}),
365    [{opel,germany}] = dets:lookup(Tab,opel),
366    ok = dets:close(Tab),
367    Call = fun(P,A) ->
368		   P ! {self(), A},
369		   receive
370		       {P, Ans} ->
371			   Ans
372		   after 5000 ->
373			   exit(other_process_dead)
374		   end
375	   end,
376    {ok, Node} = test_server:start_node(dets_dirty_mark,
377                                        slave,
378                                        [{linked, false},
379                                         {args, "-pa " ++
380                                              filename:dirname
381						(code:which(?MODULE))}]),
382    ok = ensure_node(20, Node),
383    %% io:format("~p~n",[rpc:call(Node, code, get_path, [])]),
384    %% io:format("~p~n",[rpc:call(Node, file, get_cwd, [])]),
385    %% io:format("~p~n",[Config]),
386    Pid = rpc:call(Node,erlang, spawn,
387			 [?MODULE, dets_dirty_loop, []]),
388    {ok, Tab} = Call(Pid, [open, Tab, [{file, FName}]]),
389    [{opel,germany}] = Call(Pid, [read,Tab,opel]),
390    test_server:stop_node(Node),
391    {ok, Tab} = dets:open_file(Tab,[{file, FName},
392                                    {repair,false}]),
393    ok = dets:close(Tab),
394    file:delete(FName),
395    check_pps(P0),
396    ok.
397
398%% Test that the table is flushed when auto_save is in effect.
399dirty_mark2(Config) when is_list(Config) ->
400    true = is_alive(),
401    Tab = dets_dirty_mark2_test,
402    FName = filename(Tab, Config),
403    P0 = pps(),
404    dets:open_file(Tab,[{file, FName}]),
405    dets:insert(Tab,{toyota,japan}),
406    dets:insert(Tab,{suzuki,japan}),
407    dets:insert(Tab,{honda,japan}),
408    dets:insert(Tab,{renault,france}),
409    dets:insert(Tab,{citroen,france}),
410    dets:insert(Tab,{opel,germany}),
411    dets:insert(Tab,{saab,sweden}),
412    dets:insert(Tab,{volvo,sweden}),
413    [{opel,germany}] = dets:lookup(Tab,opel),
414    ok = dets:close(Tab),
415    Call = fun(P,A) ->
416		   P ! {self(), A},
417		   receive
418		       {P, Ans} ->
419			   Ans
420		   after 5000 ->
421			   exit(other_process_dead)
422		   end
423	   end,
424    {ok, Node} = test_server:start_node(dets_dirty_mark2,
425                                        slave,
426                                        [{linked, false},
427                                         {args, "-pa " ++
428                                              filename:dirname
429						(code:which(?MODULE))}]),
430    ok = ensure_node(20, Node),
431    Pid = rpc:call(Node,erlang, spawn,
432                   [?MODULE, dets_dirty_loop, []]),
433    {ok, Tab} = Call(Pid, [open, Tab, [{file, FName},{auto_save,1000}]]),
434    ok = Call(Pid, [write,Tab,{mazda,japan}]),
435    timer:sleep(2100),
436    %% Read something, just to give auto save time to finish.
437    [{opel,germany}] = Call(Pid, [read,Tab,opel]),
438    test_server:stop_node(Node),
439    {ok, Tab} = dets:open_file(Tab, [{file, FName}, {repair,false}]),
440    ok = dets:close(Tab),
441    file:delete(FName),
442    check_pps(P0),
443    ok.
444
445dets_dirty_loop() ->
446    receive
447	{From, [open, Name, Args]} ->
448	    Ret = dets:open_file(Name, Args),
449	    From ! {self(), Ret},
450	    dets_dirty_loop();
451	{From, [read, Name, Key]} ->
452	    Ret = dets:lookup(Name, Key),
453	    From ! {self(), Ret},
454	    dets_dirty_loop();
455	{From, [write, Name, Value]} ->
456	    Ret = dets:insert(Name, Value),
457	    From ! {self(), Ret},
458	    dets_dirty_loop();
459        {From, [close, Name]} ->
460            Ret = dets:close(Name),
461            From ! {self(), Ret},
462            dets_dirty_loop()
463    end.
464
465
466%% Check that bags and next work as expected.
467bag_next(Config) when is_list(Config) ->
468    Tab = dets_bag_next_test,
469    FName = filename(Tab, Config),
470
471    %% first and next crash upon error
472    dets:open_file(Tab,[{file, FName}, {type, bag}]),
473    ok = dets:insert(Tab, [{1,1},{2,2},{3,3},{4,4}]),
474    FirstKey = dets:first(Tab),
475    NextKey = dets:next(Tab, FirstKey),
476    [FirstObj | _] = dets:lookup(Tab, FirstKey),
477    [NextObj | _] = dets:lookup(Tab, NextKey),
478    {ok, FirstPos} = dets:where(Tab, FirstObj),
479    {ok, NextPos} = dets:where(Tab, NextObj),
480    crash(FName, NextPos+12),
481    {'EXIT',BadObject1} = (catch dets:next(Tab, FirstKey)),
482    bad_object(BadObject1, FName),
483    crash(FName, FirstPos+12),
484    {'EXIT',BadObject2} = (catch dets:first(Tab)),
485    bad_object(BadObject2, FName),
486    dets:close(Tab),
487    file:delete(FName),
488
489    P0 = pps(),
490    dets:open_file(Tab,[{file, FName}, {type, bag}]),
491    dets:insert(Tab,{698,hopp}),
492    dets:insert(Tab,{186,hopp}),
493    dets:insert(Tab,{hej,hopp}),
494    dets:insert(Tab,{186,plopp}),
495    Loop = fun(N, Last, Self) ->
496		   case N of
497		       0 ->
498			   exit({unterminated_first_next_sequence, N, Last});
499		       _ ->
500			   case Last of
501			       '$end_of_table' ->
502				   ok;
503			       _ ->
504				   Self(N-1, dets:next(Tab,Last), Self)
505			   end
506		   end
507	   end,
508    ok = Loop(4,dets:first(Tab),Loop),
509    dets:close(Tab),
510    file:delete(FName),
511    check_pps(P0),
512    ok.
513
514oldbugs(Config) when is_list(Config) ->
515    FName = filename(dets_suite_oldbugs_test, Config),
516    P0 = pps(),
517    {ok, ob} = dets:open_file(ob, [{type, bag}, {file, FName}]),
518    ok = dets:insert(ob, {1, 2}),
519    ok = dets:insert(ob, {1,3}),
520    ok = dets:insert(ob, {1, 2}),
521    2 = dets:info(ob, size),  %% assertion
522    ok = dets:close(ob),
523    file:delete(FName),
524    check_pps(P0),
525    ok.
526
527%% Test that a file where the segment array has been truncated
528%% is possible to repair.
529truncated_segment_array(Config) when is_list(Config) ->
530    TabRef = dets_suite_truncated_segment_array_test,
531    Fname = filename(TabRef, Config),
532    %% Create file that needs to be repaired
533    file:delete(Fname),
534    P0 = pps(),
535    {ok, TabRef} = dets:open_file(TabRef, [{file, Fname}]),
536    ok = dets:close(TabRef),
537
538    %% Truncate the file
539    HeadSize = headsz(),
540    truncate(Fname, HeadSize + 10),
541
542    %% Open the truncated file
543    io:format("Expect repair:~n"),
544    {ok, TabRef} = dets:open_file(TabRef,
545					[{file, Fname}, {repair, true}]),
546    ok = dets:close(TabRef),
547    file:delete(Fname),
548    check_pps(P0),
549    ok.
550
551%% Test open_file/1.
552open_file(Config) when is_list(Config) ->
553    T = open_v9,
554    Fname = filename(T, Config),
555    {ok, _} = dets:open_file(T, [{file,Fname}]),
556    9 = dets:info(T, version), % Backwards compatibility.
557    true = [self()] =:= dets:info(T, users),
558    {ok, _} = dets:open_file(T, [{file,Fname}]),
559    true = [self(),self()] =:= dets:info(T, users),
560    ok = dets:close(T),
561    true = [self()] =:= dets:info(T, users),
562    ok = dets:close(T),
563    undefined = ets:info(T, users),
564    file:delete(Fname),
565
566    open_1(Config).
567
568open_1(Config) ->
569    TabRef = open_file_1_test,
570    Fname = filename(TabRef, Config),
571    file:delete(Fname),
572
573    P0 = pps(),
574    {error,{file_error,Fname,enoent}} = dets:open_file(Fname),
575
576    ok = file:write_file(Fname, duplicate(100,65)),
577    {error,{not_a_dets_file,Fname}} = dets:open_file(Fname),
578    file:delete(Fname),
579
580    HeadSize = headsz(),
581    {ok, TabRef} = dets:open_file(TabRef, [{file, Fname}]),
582    ok = dets:close(TabRef),
583    truncate(Fname, HeadSize + 10),
584    true = dets:is_dets_file(Fname),
585    io:format("Expect repair:~n"),
586    {ok, Ref} = dets:open_file(Fname), % repairing
587    ok = dets:close(Ref),
588    file:delete(Fname),
589
590    %% truncated file header, invalid type
591    {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]),
592    ok = ins(TabRef, 3000),
593    ok = dets:close(TabRef),
594    TypePos = 12,
595    crash(Fname, TypePos),
596    {error, {invalid_type_code,Fname}} = dets:open_file(Fname),
597    truncate(Fname, HeadSize - 10),
598    {error,{not_a_dets_file,Fname}} = dets:open_file(Fname),
599    {error,{not_a_dets_file,Fname}} =
600        dets:open_file(TabRef, [{file,Fname}]),
601    file:delete(Fname),
602
603    {error,{file_error,{foo,bar},_}} = dets:is_dets_file({foo,bar}),
604    check_pps(P0),
605    ok.
606
607%% Test initialize_table/2 and from_ets/2.
608init_table(Config) when is_list(Config) ->
609    %% Objects are returned in "time order".
610    T = init_table_v9,
611    Fname = filename(T, Config),
612    file:delete(Fname),
613    L = [{1,a},{2,b},{1,c},{2,c},{1,c},{2,a},{1,b}],
614    Input = init([L]),
615    {ok, _} = dets:open_file(T, [{file,Fname},{type,duplicate_bag}]),
616    ok = dets:init_table(T, Input),
617    [{1,a},{1,c},{1,c},{1,b}] = dets:lookup(T, 1),
618    [{2,b},{2,c},{2,a}] = dets:lookup(T, 2),
619    ok = dets:close(T),
620    file:delete(Fname),
621
622    init_table_1(Config),
623    fast_init_table(Config).
624
625init_table_1(Config) ->
626    TabRef = init_table_test,
627    Fname = filename(TabRef, Config),
628    file:delete(Fname),
629    P0 = pps(),
630
631    Args = [{file,Fname},{auto_save,120000}],
632    {ok, _} = dets:open_file(TabRef, Args),
633    {'EXIT', _} =
634	(catch dets:init_table(TabRef, fun(foo) -> bar end)),
635    dets:close(TabRef),
636    {ok, _} = dets:open_file(TabRef, Args),
637    {'EXIT', _} = (catch dets:init_table(TabRef, fun() -> foo end)),
638    dets:close(TabRef),
639    {ok, _} = dets:open_file(TabRef, Args),
640    {'EXIT', {badarg, _}} = (catch dets:init_table(TabRef, nofun)),
641    {'EXIT', {badarg, _}} =
642	(catch dets:init_table(TabRef, fun(_X) -> end_of_input end,
643			       [{foo,bar}])),
644    dets:close(TabRef),
645    {ok, _} = dets:open_file(TabRef, Args),
646    away = (catch dets:init_table(TabRef, fun(_) -> throw(away) end)),
647    dets:close(TabRef),
648    {ok, _} = dets:open_file(TabRef, Args),
649    {error, {init_fun, fopp}} =
650	dets:init_table(TabRef, fun(read) -> fopp end),
651    dets:close(TabRef),
652
653    {ok, _} = dets:open_file(TabRef, Args),
654    dets:safe_fixtable(TabRef, true),
655    {error, {fixed_table, TabRef}} = dets:init_table(TabRef, init([])),
656    dets:safe_fixtable(TabRef, false),
657    ET = ets:new(foo,[]),
658    ok = dets:from_ets(TabRef, ET),
659    [] = get_all_objects(TabRef),
660    [] = get_all_objects_fast(TabRef),
661    true = ets:insert(ET, {1,a}),
662    true = ets:insert(ET, {2,b}),
663    ok = dets:from_ets(TabRef, ET),
664    [{1,a},{2,b}] = sort(get_all_objects(TabRef)),
665    [{1,a},{2,b}] = sort(get_all_objects_fast(TabRef)),
666    true = ets:delete(ET),
667    120000 = dets:info(TabRef, auto_save),
668    ok = dets:close(TabRef),
669
670    {ok, _} = dets:open_file(TabRef, [{access,read} | Args]),
671    {error, {access_mode, Fname}} = dets:init_table(TabRef, init([])),
672    ok = dets:close(TabRef),
673
674    {ok, _} = dets:open_file(TabRef, Args),
675    {error, invalid_objects_list} =
676	(catch dets:init_table(TabRef, init([[{1,2},bad,{3,4}]]))),
677    _ = dets:close(TabRef),
678    file:delete(Fname),
679
680    L1 = [[{1,a},{2,b}],[],[{3,c}],[{4,d}],[]],
681    bulk_init(L1, set, 4, Config),
682    L2 = [[{1,a},{2,b}],[],[{2,q},{3,c}],[{4,d}],[{4,e},{2,q}]],
683    bulk_init(L2, set, 4, Config),
684    bulk_init(L2, bag, 6, Config),
685    bulk_init(L2, duplicate_bag, 7, Config),
686    bulk_init(L1, set, 4, 512, Config),
687    bulk_init([], set, 0, 10000, Config),
688    file:delete(Fname),
689
690    %% Initiate a file that contains a lot of objects.
691    {ok, _} = dets:open_file(TabRef, [{min_no_slots,10000} | Args]),
692    ok = ins(TabRef, 6000),
693    Fun = init_fun(0, 10000),
694    ok = dets:init_table(TabRef, Fun,{format,term}),
695    All = sort(get_all_objects(TabRef)),
696    FAll = get_all_objects_fast(TabRef),
697    true = All =:= sort(FAll),
698    true = length(All) =:= 10000,
699    ok = dets:close(TabRef),
700    file:delete(Fname),
701
702    {ok, _} = dets:open_file(TabRef, [{min_no_slots,4000} | Args]),
703    ok = ins(TabRef, 6000),
704    FileSize1 = dets:info(TabRef, file_size),
705    Fun2 = init_fun(0, 4000),
706    ok = dets:init_table(TabRef, Fun2),
707    FileSize2 = dets:info(TabRef, file_size),
708    ok = dets:close(TabRef),
709    true = FileSize1 > FileSize2,
710    file:delete(Fname),
711
712    check_pps(P0),
713    ok.
714
715bulk_init(Ls, Type, N, Config) ->
716    bulk_init(Ls, Type, N, 256, Config).
717
718bulk_init(Ls, Type, N, Est, Config) ->
719    T = init_table_test,
720    Fname = filename(T, Config),
721    file:delete(Fname),
722    Input = init(Ls),
723    Args = [{ram_file,false}, {type,Type},{keypos,1},{file,Fname},
724	    {estimated_no_objects, Est}],
725    {ok, T} = dets:open_file(T, Args),
726    ok = dets:init_table(T, Input),
727    All = sort(get_all_objects(T)),
728    FAll = get_all_objects_fast(T),
729    true = All =:= sort(FAll),
730    true = length(All) =:= N,
731    true = dets:info(T, size) =:= N,
732    ok = dets:close(T),
733
734    {ok, T} = dets:open_file(T, Args),
735    All2 = sort(get_all_objects(T)),
736    FAll2 = get_all_objects_fast(T),
737    true = All =:= All2,
738    true = All =:= sort(FAll2),
739    ok = dets:close(T),
740    file:delete(Fname).
741
742init(L) ->
743    fun(close) ->
744	    ok;
745       (read) when [] =:= L ->
746	    end_of_input;
747       (read) ->
748	    [E | Es] = L,
749	    {E, init(Es)}
750    end.
751
752init_fun(I, N) ->
753    fun(read) when I =:= N ->
754	    end_of_input;
755       (read) ->
756	    {NewN, Items} = items(I, N, 1000, []),
757	    {Items, init_fun(NewN, N)};
758       (close) ->
759	    ignored
760    end.
761
762fast_init_table(Config) ->
763    TabRef = init_table_test,
764    Fname = filename(TabRef, Config),
765    file:delete(Fname),
766    P0 = pps(),
767
768    Args = [{file,Fname},{auto_save,120000}],
769
770    Source = init_table_test_source,
771    SourceFname = filename(Source, Config),
772    file:delete(SourceFname),
773    SourceArgs = [{file,SourceFname},{auto_save,120000}],
774
775    {ok, Source} = dets:open_file(Source, SourceArgs),
776
777    {ok, _} = dets:open_file(TabRef, Args),
778    {'EXIT', _} =
779	(catch dets:init_table(TabRef, fun(foo) -> bar end, {format,bchunk})),
780    dets:close(TabRef),
781    file:delete(Fname),
782    {ok, _} = dets:open_file(TabRef, Args),
783    {'EXIT', _} = (catch dets:init_table(TabRef, fun() -> foo end,
784					       {format,bchunk})),
785    dets:close(TabRef),
786    file:delete(Fname),
787    {ok, _} = dets:open_file(TabRef, Args),
788    {'EXIT', {badarg, _}} =
789	(catch dets:init_table(TabRef, nofun, {format,bchunk})),
790    dets:close(TabRef),
791    {ok, _} = dets:open_file(TabRef, Args),
792    away = (catch dets:init_table(TabRef, fun(_) -> throw(away) end,
793					{format,bchunk})),
794    dets:close(TabRef),
795    file:delete(Fname),
796    {ok, _} = dets:open_file(TabRef, Args),
797    {error, {init_fun, fopp}} =
798	dets:init_table(TabRef, fun(read) -> fopp end, {format,bchunk}),
799    dets:close(TabRef),
800    file:delete(Fname),
801    {ok, _} = dets:open_file(TabRef, Args),
802    dets:safe_fixtable(TabRef, true),
803    {error, {fixed_table, TabRef}} =
804	dets:init_table(TabRef, init([]), {format,bchunk}),
805    dets:safe_fixtable(TabRef, false),
806    ok = dets:close(TabRef),
807
808    {ok, _} = dets:open_file(TabRef, [{access,read} | Args]),
809    {error, {access_mode, Fname}} =
810	dets:init_table(TabRef, init([]), {format,bchunk}),
811    ok = dets:close(TabRef),
812
813    {ok, _} = dets:open_file(TabRef, Args),
814    {error, {init_fun,{1,2}}} =
815	dets:init_table(TabRef, init([[{1,2},bad,{3,4}]]), {format,bchunk}),
816    _ = dets:close(TabRef),
817    file:delete(Fname),
818
819    {ok, _} = dets:open_file(TabRef, Args),
820    {error, {init_fun, end_of_input}} =
821	dets:init_table(TabRef, init([]),{format,bchunk}),
822    _ = dets:close(TabRef),
823    file:delete(Fname),
824
825    {ok, _} = dets:open_file(TabRef, Args),
826    {'EXIT', {badarg, _}} =
827	(catch dets:init_table(TabRef, init([]),{format,foppla})),
828    _ = dets:close(TabRef),
829    file:delete(Fname),
830
831    {ok, _} = dets:open_file(TabRef, Args),
832    ok = ins(TabRef, 100),
833
834    [BParms | Objs] = collect_bchunk(TabRef, init_bchunk(TabRef)),
835    Parms = binary_to_term(BParms),
836    {error, {init_fun, <<"foobar">>}} =
837	dets:init_table(TabRef, init([[<<"foobar">>]]),{format,bchunk}),
838    _ = dets:close(TabRef),
839    file:delete(Fname),
840
841    {ok, _} = dets:open_file(TabRef, Args),
842    Parms1 = setelement(1, Parms, foobar),
843    BParms1 = term_to_binary(Parms1),
844    {error, {init_fun, BParms1}} =
845	dets:init_table(TabRef, init([[BParms1 | Objs]]),{format,bchunk}),
846    _ = dets:close(TabRef),
847    file:delete(Fname),
848
849    {ok, _} = dets:open_file(TabRef, Args),
850    [{Sz1,No1} | NoColls17] = element(tuple_size(Parms), Parms),
851    Parms2 = setelement(tuple_size(Parms), Parms, [{Sz1,No1+1} | NoColls17]),
852    BParms2 = term_to_binary(Parms2),
853    {error, invalid_objects_list} =
854	dets:init_table(TabRef, init([[BParms2 | Objs]]),{format,bchunk}),
855    _ = dets:close(TabRef),
856    file:delete(Fname),
857
858    {ok, _} = dets:open_file(TabRef, Args),
859    [{LSize1,Slot1,Obj1} | ObjsRest] = Objs,
860
861    BadSize = byte_size(Obj1)-1,
862    <<BadSizeObj:BadSize/binary,_:1/binary>> = Obj1,
863    BadObjs = [{LSize1,Slot1,BadSizeObj} | ObjsRest],
864    {error, invalid_objects_list} =
865	dets:init_table(TabRef, init([[BParms | BadObjs]]),{format,bchunk}),
866    _ = dets:close(TabRef),
867    file:delete(Fname),
868
869    {ok, _} = dets:open_file(TabRef, Args),
870    <<Size:32,BigObj0/binary>> = list_to_binary(lists:duplicate(16,Obj1)),
871    BigObj = <<(Size*16):32,BigObj0/binary>>,
872    BadColl = [BParms, {LSize1+4,Slot1,BigObj} | ObjsRest],
873    {error, invalid_objects_list} =
874         dets:init_table(TabRef, init([BadColl]),{format,bchunk}),
875    _ = dets:close(TabRef),
876    file:delete(Fname),
877
878    {ok, _} = dets:open_file(TabRef, Args),
879    BadObj = <<"foobar">>,
880    {error, invalid_objects_list} =
881	dets:init_table(TabRef, init([[BParms, BadObj]]),{format,bchunk}),
882    _ = dets:close(TabRef),
883    file:delete(Fname),
884
885    {ok, _} = dets:open_file(TabRef, [{type,bag} | Args]),
886    {error, {init_fun, _}} =
887	dets:init_table(TabRef, init([[BParms]]),{format,bchunk}),
888    _ = dets:close(TabRef),
889    file:delete(Fname),
890
891    ok = dets:close(Source),
892    file:delete(SourceFname),
893
894    L1 = [{1,a},{2,b},{3,c},{4,d}],
895    fast_bulk_init(L1, set, 4, 4, Config),
896    L2 = [{1,a},{2,b},{2,q},{3,c},{4,d},{4,e},{2,q}],
897    fast_bulk_init(L2, set, 4, 4, Config),
898    fast_bulk_init(L2, bag, 6, 4, Config),
899    fast_bulk_init(L2, duplicate_bag, 7, 4, Config),
900    fast_bulk_init(L1, set, 4, 4, 512, Config),
901    fast_bulk_init([], set, 0, 0, 10000, Config),
902    file:delete(Fname),
903
904    %% Initiate a file that contains a lot of objects.
905    {ok, _} = dets:open_file(Source, [{min_no_slots,10000} | SourceArgs]),
906    Fun1 = init_fun(0, 10000),
907    ok = dets:init_table(Source, Fun1, {format,term}),
908
909    {ok, _} = dets:open_file(TabRef, [{min_no_slots,10000} | Args]),
910    ok = ins(TabRef, 6000),
911    Fun2 = init_bchunk(Source),
912    true =
913        dets:is_compatible_bchunk_format(TabRef,
914                                         dets:info(Source, bchunk_format)),
915    false = dets:is_compatible_bchunk_format(TabRef, <<"foobar">>),
916    ok = dets:init_table(TabRef, Fun2, {format, bchunk}),
917    ok = dets:close(Source),
918    file:delete(SourceFname),
919    All = sort(get_all_objects(TabRef)),
920    FAll = get_all_objects_fast(TabRef),
921    true = All =:= sort(FAll),
922    true = length(All) =:= 10000,
923    ok = dets:close(TabRef),
924    file:delete(Fname),
925
926    %% Initiate inserts fewer objects than the table contains.
927    {ok, _} = dets:open_file(Source, [{min_no_slots,1000} | SourceArgs]),
928    ok = ins(Source, 4000),
929
930    {ok, _} = dets:open_file(TabRef, [{min_no_slots,1000} | Args]),
931    ok = ins(TabRef, 6000),
932    FileSize1 = dets:info(TabRef, file_size),
933    Fun4 = init_bchunk(Source),
934    ok = dets:init_table(TabRef, Fun4, {format, bchunk}),
935    ok = dets:close(Source),
936    file:delete(SourceFname),
937    FileSize2 = dets:info(TabRef, file_size),
938    All_2 = sort(get_all_objects(TabRef)),
939    FAll_2 = get_all_objects_fast(TabRef),
940    true = All_2 =:= sort(FAll_2),
941    true = length(All_2) =:= 4000,
942    ok = dets:close(TabRef),
943    true = FileSize1 > FileSize2,
944
945    %% Bchunk and fixed table.
946    {ok, _} = dets:open_file(TabRef, Args),
947    NoItems = dets:info(TabRef, no_objects),
948    AllObjects1 = sort(get_all_objects_fast(TabRef)),
949    dets:safe_fixtable(TabRef, true),
950    true = dets:info(TabRef, fixed),
951    Cont1 = init_bchunk(TabRef),
952    NoDel =
953	dets:select_delete(TabRef, [{{'_',{item,'_','_'}},[],[true]}]),
954    true = (NoDel > 0),
955    AllObjects2 = sort(get_all_objects_fast(TabRef)),
956    true = dets:info(TabRef, fixed),
957    Cont2 = init_bchunk(TabRef),
958    NoItems2 = dets:info(TabRef, no_objects),
959    true = (NoItems =:= NoItems2 + NoDel),
960    NoDel2 = dets:select_delete(TabRef, [{'_',[],[true]}]),
961    true = (NoDel2 > 0),
962    AllObjects3 = sort(get_all_objects_fast(TabRef)),
963    NoItems3 = dets:info(TabRef, no_objects),
964    true = (NoItems3 =:= 0),
965    true = dets:info(TabRef, fixed),
966    true = (NoItems2 =:= NoItems3 + NoDel2),
967    Cont3 = init_bchunk(TabRef),
968
969    BinColl1 = collect_bchunk(TabRef, Cont1),
970    BinColl2 = collect_bchunk(TabRef, Cont2),
971    BinColl3 = collect_bchunk(TabRef, Cont3),
972    dets:safe_fixtable(TabRef, false),
973    ok = dets:close(TabRef),
974    file:delete(Fname),
975
976    %% Now check that the above collected binaries are correct.
977    {ok, _} = dets:open_file(TabRef, Args),
978    ok = dets:init_table(TabRef, init([BinColl1]),{format,bchunk}),
979    true = (AllObjects1 =:= sort(get_all_objects_fast(TabRef))),
980    true = (length(AllObjects1) =:= dets:info(TabRef, no_objects)),
981    ok = dets:init_table(TabRef, init([BinColl2]),{format,bchunk}),
982    true = (AllObjects2 =:= sort(get_all_objects_fast(TabRef))),
983    true = (length(AllObjects2) =:= dets:info(TabRef, no_objects)),
984    ok = dets:init_table(TabRef, init([BinColl3]),{format,bchunk}),
985    true = (AllObjects3 =:= sort(get_all_objects_fast(TabRef))),
986    true = (length(AllObjects3) =:= dets:info(TabRef, no_objects)),
987    ok = dets:close(TabRef),
988    file:delete(Fname),
989    check_pps(P0),
990    ok.
991
992fast_bulk_init(L, Type, N, NoKeys, Config) ->
993    fast_bulk_init(L, Type, N, NoKeys, 256, Config).
994
995fast_bulk_init(L, Type, N, NoKeys, Est, Config) ->
996    T = init_table_test,
997    Fname = filename(T, Config),
998    file:delete(Fname),
999
1000    Args0 = [{ram_file,false}, {type,Type},{keypos,1},
1001	    {estimated_no_objects, Est}],
1002    Args = [{file,Fname} | Args0],
1003    S = init_table_test_source,
1004    SFname = filename(S, Config),
1005    file:delete(SFname),
1006    SArgs = [{file,SFname} | Args0],
1007
1008    {ok, S} = dets:open_file(S, SArgs),
1009    ok = dets:insert(S, L),
1010
1011    Input = init_bchunk(S),
1012    {ok, T} = dets:open_file(T, Args),
1013    ok = dets:init_table(T, Input, [{format,bchunk}]),
1014    All = sort(get_all_objects(T)),
1015    FAll = get_all_objects_fast(T),
1016    true = All =:= sort(FAll),
1017    true = length(All) =:= N,
1018    true = dets:info(T, size) =:= N,
1019    true = dets:info(T, no_keys) =:= NoKeys,
1020    ok = dets:close(T),
1021
1022    {ok, T} = dets:open_file(T, Args),
1023    All2 = sort(get_all_objects(T)),
1024    FAll2 = get_all_objects_fast(T),
1025    true = All =:= All2,
1026    true = All =:= sort(FAll2),
1027    ok = dets:close(T),
1028    file:delete(Fname),
1029
1030    ok = dets:close(S),
1031    file:delete(SFname),
1032    ok.
1033
1034init_bchunk(T) ->
1035    Start = dets:bchunk(T, start),
1036    init_bchunk(T, Start).
1037
1038init_bchunk(Tab, State) ->
1039    fun(read) when State =:= '$end_of_table' ->
1040	    end_of_input;
1041       (read) when element(1, State) =:= error ->
1042	    State;
1043       (read) ->
1044	    {Cont, Objs} = State,
1045	    {Objs, init_bchunk(Tab, dets:bchunk(Tab, Cont))};
1046       (close) ->
1047	    ok
1048    end.
1049
1050collect_bchunk(Tab, Fun) ->
1051    collect_bchunk(Tab, Fun, []).
1052
1053collect_bchunk(Tab, Fun, L) ->
1054    case Fun(read) of
1055	end_of_input ->
1056	    lists:append(lists:reverse(L));
1057	{Objs, Fun2} when is_list(Objs) ->
1058	    collect_bchunk(Tab, Fun2, [Objs | L]);
1059	Error ->
1060	    Error
1061    end.
1062
1063items(I, N, C, L) when I =:= N; C =:= 0 ->
1064    {I, L};
1065items(I, N, C, L) ->
1066    items(I+1, N, C-1, [{I, item(I)} | L]).
1067
1068%% Test open_file and repair.
1069repair(Config) when is_list(Config) ->
1070    %% The short lived format 9(a).
1071    %% Not very throughly tested here.
1072    A9 = a9,
1073    Version9aS = filename:join(?datadir(Config), "version_9a.dets"),
1074    Version9aT = filename('v9a.dets', Config),
1075    {ok, _} = file:copy(Version9aS, Version9aT),
1076    {ok, A9} = dets:open_file(A9, [{file,Version9aT}]),
1077    undefined = dets:info(A9, bchunk_format),
1078    [{1,a},{2,b},{3,c}] = sort(dets:match_object(A9, '_')),
1079    ok = dets:insert(A9, {4,d}),
1080    ok = dets:close(A9),
1081    {ok, A9} = dets:open_file(A9, [{file,Version9aT}]),
1082    {error, old_version} = dets:bchunk(A9, start),
1083    ok = dets:close(A9),
1084    io:format("Expect forced repair:~n"),
1085    {ok, A9} = dets:open_file(A9, [{file,Version9aT},{repair,force}]),
1086    {_, _} = dets:bchunk(A9, start),
1087    ok = dets:close(A9),
1088    file:delete(Version9aT),
1089
1090    repair_1(Config).
1091
1092repair_1(Config) ->
1093    TabRef = repair_test,
1094    Fname = filename(TabRef, Config),
1095    file:delete(Fname),
1096    HeadSize = headsz(),
1097
1098    P0 = pps(),
1099    {'EXIT', {badarg, _}} =
1100	(catch dets:open_file(TabRef, [{min_no_slots,1000},
1101				       {max_no_slots,500}])),
1102    {error,{file_error,hoppla,enoent}} = dets:file_info(hoppla),
1103    {error,{file_error,Fname,enoent}} =
1104	dets:open_file(TabRef, [{file, Fname}, {access, read}]),
1105
1106    %% compacting, and some kind of test that free lists are saved OK on file
1107    {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]),
1108    0 = dets:info(TabRef, size),
1109    ok = ins(TabRef, 30000),
1110    ok = del(TabRef, 30000, 3),
1111    ok = dets:close(TabRef),
1112    {error, {access_mode,Fname}} =
1113        dets:open_file(foo, [{file,Fname},{repair,force},{access,read}]),
1114    {ok, Ref3} = dets:open_file(Fname), % no repair!
1115    20000 = dets:info(Ref3, size),
1116    20000 = dets:foldl(fun(_, N) -> N+1 end, 0, Ref3),
1117    20000 = count_objects_quite_fast(Ref3), % actually a test of match
1118    no_keys_test(Ref3),
1119    ok = dets:close(Ref3),
1120    {error,{keypos_mismatch,Fname}} =
1121	dets:open_file(TabRef, [{file, Fname},{keypos,17}]),
1122    {error,{type_mismatch,Fname}} =
1123	dets:open_file(TabRef, [{file, Fname},{type,duplicate_bag}]),
1124
1125    %% make one of the temporary files unwritable
1126    TmpFile = Fname ++ ".TMP.1",
1127    file:delete(TmpFile),
1128    {ok, TmpFd} = file:open(TmpFile, [read,write]),
1129    ok = file:close(TmpFd),
1130    unwritable(TmpFile),
1131    {error,{file_error,TmpFile,eacces}} = dets:fsck(Fname),
1132    {ok, _} = dets:open_file(TabRef,
1133                             [{repair,false},{file, Fname}]),
1134    20000 = length(get_all_objects(TabRef)),
1135    _ = histogram(TabRef, silent),
1136    20000 = length(get_all_objects_fast(TabRef)),
1137    ok = dets:close(TabRef),
1138    writable(TmpFile),
1139    file:delete(TmpFile),
1140
1141    truncate(Fname, HeadSize + 10),
1142    {error,{not_closed, Fname}} =
1143	dets:open_file(TabRef, [{file, Fname}, {access, read}]),
1144    {error,{not_closed, Fname}} =
1145	dets:open_file(TabRef, [{file, Fname}, {access, read},
1146                                {repair,force}]),
1147    {error,{needs_repair, Fname}} =
1148	dets:open_file(TabRef, [{file, Fname}, {repair, false}]),
1149    file:delete(Fname),
1150
1151    %% truncated file header
1152    {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]),
1153    ok = ins(TabRef, 100),
1154    ok = dets:close(TabRef),
1155    file:delete(Fname),
1156
1157    %% FIXME.
1158    %% will fail unless the slots are properly sorted when repairing (v8)
1159    BArgs = [{file, Fname},{type,duplicate_bag},
1160	     {delayed_write,{3000,10000}}],
1161    {ok, TabRef} = dets:open_file(TabRef, BArgs),
1162    Seq = seq(1, 500),
1163    Small = map(fun(X) -> {X,X} end, Seq),
1164    Big = map(fun(X) -> erlang:make_tuple(20, X) end, Seq),
1165    ok = dets:insert(TabRef, Small),
1166    ok = dets:insert(TabRef, Big),
1167    ok = dets:insert(TabRef, Small),
1168    ok = dets:insert(TabRef, Big),
1169    All = sort(safe_get_all_objects(TabRef)),
1170    ok = dets:close(TabRef),
1171    io:format("Expect forced repair:~n"),
1172    {ok, _} =
1173         dets:open_file(TabRef, [{repair,force},{min_no_slots,2000} | BArgs]),
1174
1175    {MinNoSlots,_,MaxNoSlots} = dets:info(TabRef, no_slots),
1176    ok = dets:close(TabRef),
1177    io:format("Expect compaction:~n"),
1178    {ok, _} =
1179        dets:open_file(TabRef, [{repair,force},
1180                                {min_no_slots,MinNoSlots},
1181                                {max_no_slots,MaxNoSlots} | BArgs]),
1182    All2 = get_all_objects(TabRef),
1183    true = All =:= sort(All2),
1184    FAll2 = get_all_objects_fast(TabRef),
1185    true = All =:= sort(FAll2),
1186    true = length(All) =:= dets:info(TabRef, size),
1187    no_keys_test(TabRef),
1188    Fun = fun(X) -> 4 = length(dets:lookup(TabRef, X)) end,
1189    foreach(Fun, Seq),
1190    _ = histogram(TabRef, silent),
1191    ok = dets:close(TabRef),
1192    file:delete(Fname),
1193
1194    %% object bigger than segments, the "hole" is taken care of
1195    {ok, TabRef} = dets:open_file(TabRef, [{file, Fname}]),
1196    Tuple = erlang:make_tuple(1000, foobar), % > 2 kB
1197    ok = dets:insert(TabRef, Tuple),
1198    %% at least one full segment (objects smaller than 2 kB):
1199    ins(TabRef, 2000),
1200    ok = dets:close(TabRef),
1201
1202    {ok, _} =
1203        dets:open_file(TabRef, [{repair,false},{file, Fname}]),
1204    {ok, ObjPos} = dets:where(TabRef, {66,{item,number,66}}),
1205    ok = dets:close(TabRef),
1206    %% Damaged object.
1207    Pos = 12, % v9: compaction fails, proper repair follows
1208    crash(Fname, ObjPos+Pos),
1209    io:format(
1210	    "Expect forced repair (possibly after attempted compaction):~n"),
1211    {ok, _} = dets:open_file(TabRef, [{repair,force},{file, Fname}]),
1212    true = dets:info(TabRef, size) < 2001,
1213    ok = dets:close(TabRef),
1214    file:delete(Fname),
1215
1216    %% The file is smaller than the padded object.
1217    {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]),
1218    ok = dets:insert(TabRef, Tuple),
1219    ok = dets:close(TabRef),
1220    io:format("Expect forced repair or compaction:~n"),
1221    {ok, _} =
1222	dets:open_file(TabRef, [{repair,force},{file, Fname}]),
1223    true = 1 =:= dets:info(TabRef, size),
1224    ok = dets:close(TabRef),
1225    file:delete(Fname),
1226
1227    %% Damaged free lists.
1228    {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]),
1229    ok = ins(TabRef, 300),
1230    ok = dets:sync(TabRef),
1231    ok = del(TabRef, 300, 3),
1232    %% FileSize is approximately where the free lists will be written.
1233    FileSize = dets:info(TabRef, memory),
1234    ok = dets:close(TabRef),
1235    crash(Fname, FileSize+20),
1236    %% Used to return bad_freelists, but that changed in OTP-9622
1237    {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]),
1238    ok = dets:close(TabRef),
1239    file:delete(Fname),
1240
1241    %% File not closed, opening with read and read_write access tried.
1242    {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]),
1243    ok = ins(TabRef, 300),
1244    ok = dets:close(TabRef),
1245    crash(Fname, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED),
1246    {error, {not_closed, Fname}} =
1247       dets:open_file(foo, [{file,Fname},{repair,force},
1248                            {access,read}]),
1249    {error, {not_closed, Fname}} =
1250       dets:open_file(foo, [{file,Fname},{repair,true},
1251                            {access,read}]),
1252    io:format("Expect repair:~n"),
1253    {ok, TabRef} =
1254       dets:open_file(TabRef, [{file,Fname},{repair,true},
1255                               {access,read_write}]),
1256    ok = dets:close(TabRef),
1257    crash(Fname, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED),
1258    io:format("Expect forced repair:~n"),
1259    {ok, TabRef} =
1260       dets:open_file(TabRef, [{file,Fname},{repair,force},
1261                               {access,read_write}]),
1262    ok = dets:close(TabRef),
1263    file:delete(Fname),
1264
1265    %% The size of an object is huge.
1266    {ok, TabRef} = dets:open_file(TabRef, [{file,Fname}]),
1267    ok = dets:insert(TabRef, [{1,2,3},{2,3,4}]),
1268    {ok, ObjPos2} = dets:where(TabRef, {1,2,3}),
1269    ok = dets:close(TabRef),
1270    crash(Fname, ObjPos2, 255),
1271    io:format("Expect forced repair:~n"),
1272    {ok, TabRef} = dets:open_file(TabRef, [{file,Fname},{repair,force}]),
1273    ok = dets:close(TabRef),
1274    file:delete(Fname),
1275
1276    check_pps(P0),
1277    ok.
1278
1279
1280%% Test version 9(b) with erlang:phash/2 as hash function.
1281phash(Config) when is_list(Config) ->
1282    T = phash,
1283    Phash_v9bS = filename:join(?datadir(Config), "version_9b_phash.dat"),
1284    Fname = filename('v9b.dets', Config),
1285    {ok, _} = file:copy(Phash_v9bS, Fname),
1286
1287    %% Deleting all objects changes the hash function.
1288    %% A feature... (it's for free)
1289    {ok, T} = dets:open_file(T, [{file, Fname}]),
1290    phash = dets:info(T, hash),
1291    dets:delete_all_objects(T),
1292    phash2 = dets:info(T, hash),
1293    [] = get_all_objects(T),
1294    [] = get_all_objects_fast(T),
1295    ok = dets:close(T),
1296
1297    %% The hash function is kept when compacting a table.
1298    {ok, _} = file:copy(Phash_v9bS, Fname),
1299    io:format("Expect compaction:~n"),
1300    {ok, T} = dets:open_file(T, [{file, Fname},{repair,force}]),
1301    phash = dets:info(T, hash),
1302    [{1,a},{2,b},{3,c},{4,d},{5,e}] =
1303	lists:sort(dets:lookup_keys(T, [1,2,3,4,5])),
1304    ok = dets:close(T),
1305
1306    %% The hash function is updated when repairing a table (no cost).
1307    {ok, _} = file:copy(Phash_v9bS, Fname),
1308    crash(Fname, ?CLOSED_PROPERLY_POS+3, 0),
1309    io:format("Expect repair:~n"),
1310    {ok, T} = dets:open_file(T, [{file, Fname}]),
1311    phash2 = dets:info(T, hash),
1312    [{1,a},{2,b},{3,c},{4,d},{5,e}] =
1313	lists:sort(dets:lookup_keys(T, [1,2,3,4,5])),
1314    ok = dets:close(T),
1315
1316    %% One cannot use the bchunk format when copying between a phash
1317    %% table and a phash2 table. (There is no test for the case an
1318    %% Erlang/OTP R9 (or later) node (using phash2) copies a table to
1319    %% an Erlang/OTP R8 node (using phash).) See also the comment on
1320    %% HASH_PARMS in dets_v9.erl.
1321    {ok, _} = file:copy(Phash_v9bS, Fname),
1322    {ok, T} = dets:open_file(T, [{file, Fname}]),
1323    Type = dets:info(T, type),
1324    KeyPos = dets:info(T, keypos),
1325    Input = init_bchunk(T),
1326    T2 = phash_table,
1327    Fname2 = filename(T2, Config),
1328    Args = [{type,Type},{keypos,KeyPos},{file,Fname2}],
1329    {ok, T2} = dets:open_file(T2, Args),
1330    {error, {init_fun, _}} =
1331	dets:init_table(T2, Input, {format,bchunk}),
1332    _ = dets:close(T2),
1333    ok = dets:close(T),
1334    file:delete(Fname2),
1335
1336    file:delete(Fname),
1337    ok.
1338
1339%% Test foldl, foldr, to_ets.
1340fold(Config) when is_list(Config) ->
1341    T = test_table,
1342    N = 100,
1343    Fname = filename(T, Config),
1344    file:delete(Fname),
1345    P0 = pps(),
1346
1347    Args = [{file,Fname}, {estimated_no_objects, N}],
1348    {ok, _} = dets:open_file(T, Args),
1349
1350    ok = ins(T, N),
1351
1352    Ets = ets:new(to_ets, [public]),
1353    dets:to_ets(T, Ets),
1354    true = N =:= ets:info(Ets, size),
1355    ets:delete(Ets),
1356
1357    Ets2 = ets:new(to_ets, [private]),
1358    dets:to_ets(T, Ets2),
1359    true = N =:= ets:info(Ets2, size),
1360    ets:delete(Ets2),
1361
1362    {'EXIT', {badarg, _}} = (catch dets:to_ets(T, not_an_ets_table)),
1363
1364    F0 = fun(X, A) -> [X | A] end,
1365    true = N =:= length(dets:foldl(F0, [], T)),
1366    true = N =:= length(dets:foldr(F0, [], T)),
1367
1368    F1 = fun(_X, _A) -> throw(away) end,
1369    away = (catch dets:foldl(F1, [], T)),
1370    away = (catch dets:foldr(F1, [], T)),
1371
1372    F2 = fun(X, A) -> X + A end,
1373    {'EXIT', _} = (catch dets:foldl(F2, [], T)),
1374    {'EXIT', _} = (catch dets:foldr(F2, [], T)),
1375
1376    F3 = fun(_X) -> throw(away) end,
1377    away = (catch dets:traverse(T, F3)),
1378
1379    F4 = fun(X) -> X + 17 end,
1380    {'EXIT', _} = (catch dets:traverse(T, F4)),
1381
1382    F5 = fun(_X) -> done end,
1383    done = dets:traverse(T, F5),
1384
1385    {ok, ObjPos} = dets:where(T, {66,{item,number,66}}),
1386    ok = dets:close(T),
1387
1388    %% Damaged object.
1389    Pos = 8,
1390    crash(Fname, ObjPos+Pos),
1391    {ok, _} = dets:open_file(T, Args),
1392    io:format("Expect corrupt table:~n"),
1393    BadObject1 = dets:foldl(F0, [], T),
1394    bad_object(BadObject1, Fname),
1395    BadObject2 = dets:close(T),
1396    bad_object(BadObject2, Fname),
1397
1398    file:delete(Fname),
1399    check_pps(P0),
1400    ok.
1401
1402%% Add objects to a fixed table.
1403fixtable(Config) when is_list(Config) ->
1404    T = fixtable,
1405    Fname = filename(fixtable, Config),
1406    file:delete(Fname),
1407    Args = [{file,Fname}],
1408    P0 = pps(),
1409    {ok, _} = dets:open_file(T, Args),
1410
1411    %% badarg
1412    check_badarg(catch dets:safe_fixtable(no_table,true),
1413		       dets, safe_fixtable, [no_table,true]),
1414    check_badarg(catch dets:safe_fixtable(T,undefined),
1415		       dets, safe_fixtable, [T,undefined]),
1416
1417    %% The table is not allowed to grow while the elements are inserted:
1418
1419    ok = ins(T, 500),
1420    dets:safe_fixtable(T, false),
1421    %% Now the table can grow. At the same time as elements are inserted,
1422    %% the table tries to catch up with the previously inserted elements.
1423    ok = ins(T, 1000),
1424    1000 = dets:info(T, size),
1425    ok = dets:close(T),
1426    file:delete(Fname),
1427
1428    {ok, _} = dets:open_file(T, [{type, duplicate_bag} | Args]),
1429    %% In a fixed table, delete and re-insert an object.
1430    ok = dets:insert(T, {1, a, b}),
1431    SysBefore = erlang:timestamp(),
1432    MonBefore = erlang:monotonic_time(),
1433    dets:safe_fixtable(T, true),
1434    MonAfter = erlang:monotonic_time(),
1435    SysAfter = erlang:timestamp(),
1436    Self = self(),
1437    {FixMonTime,[{Self,1}]} = dets:info(T,safe_fixed_monotonic_time),
1438    {FixSysTime,[{Self,1}]} = dets:info(T,safe_fixed),
1439    true = is_integer(FixMonTime),
1440    true = MonBefore =< FixMonTime,
1441    true = FixMonTime =< MonAfter,
1442    {FstMs,FstS,FstUs} = FixSysTime,
1443    true = is_integer(FstMs),
1444    true = is_integer(FstS),
1445    true = is_integer(FstUs),
1446    case erlang:system_info(time_warp_mode) of
1447	no_time_warp ->
1448	    true = timer:now_diff(FixSysTime, SysBefore) >= 0,
1449	    true = timer:now_diff(SysAfter, FixSysTime) >= 0;
1450	_ ->
1451	    %% ets:info(Tab,safe_fixed) not timewarp safe...
1452	    ignore
1453    end,
1454    ok = dets:match_delete(T, {1, a, b}),
1455    ok = dets:insert(T, {1, a, b}),
1456    {FixMonTime,[{Self,1}]} = dets:info(T,safe_fixed_monotonic_time),
1457    {FixSysTime,[{Self,1}]} = dets:info(T,safe_fixed),
1458    dets:safe_fixtable(T, false),
1459    1 = length(dets:match_object(T, '_')),
1460
1461    ok = dets:match_delete(T, '_'),
1462    %% In a fixed table, delete and insert a smaller object.
1463    ok = dets:insert(T, {1, duplicate(100, e)}),
1464    dets:safe_fixtable(T, true),
1465    ok = dets:match_delete(T, {1, '_'}),
1466    ok = dets:insert(T, {1, a, b}),
1467    dets:safe_fixtable(T, false),
1468    1 = length(dets:match_object(T, '_')),
1469
1470    ok = dets:delete_all_objects(T),
1471    %% Like the last one, but one extra object.
1472    ok = dets:insert(T, {1, duplicate(100, e)}),
1473    ok = dets:insert(T, {2, duplicate(100, e)}),
1474    dets:safe_fixtable(T, true),
1475    ok = dets:match_delete(T, {1, '_'}),
1476    ok = dets:insert(T, {1, a, b}),
1477    dets:safe_fixtable(T, false),
1478    2 = length(dets:match_object(T, '_')),
1479    dets:safe_fixtable(T, true),
1480    ok = dets:delete_all_objects(T),
1481    true = dets:info(T, fixed),
1482    0 = length(dets:match_object(T, '_')),
1483
1484    ok = dets:close(T),
1485    file:delete(Fname),
1486    check_pps(P0),
1487    ok.
1488
1489%% Matching objects of a fixed table.
1490match(Config) when is_list(Config) ->
1491    T = match,
1492    Fname = filename(match, Config),
1493    file:delete(Fname),
1494    P0 = pps(),
1495
1496    Args = [{file,Fname}, {type, duplicate_bag}, {estimated_no_objects,550}],
1497    {ok, _} = dets:open_file(T, Args),
1498    ok = dets:insert(T, {1, a, b}),
1499    ok = dets:insert(T, {1, b, a}),
1500    ok = dets:insert(T, {2, a, b}),
1501    ok = dets:insert(T, {2, b, a}),
1502
1503    %% match, badarg
1504    MSpec = [{'_',[],['$_']}],
1505    check_badarg(catch dets:match(no_table, '_'),
1506                 dets, match, [no_table,'_']),
1507    check_badarg(catch dets:match(T, '_', not_a_number),
1508                 dets, match, [T,'_',not_a_number]),
1509    {EC1, _} = dets:select(T, MSpec, 1),
1510    check_badarg(catch dets:match(EC1),
1511                 dets, match, [EC1]),
1512
1513    %% match_object, badarg
1514    check_badarg(catch dets:match_object(no_table, '_'),
1515                 dets, match_object, [no_table,'_']),
1516    check_badarg(catch dets:match_object(T, '_', not_a_number),
1517                 dets, match_object, [T,'_',not_a_number]),
1518    {EC2, _} = dets:select(T, MSpec, 1),
1519    check_badarg(catch dets:match_object(EC2),
1520                 dets, match_object, [EC2]),
1521
1522    dets:safe_fixtable(T, true),
1523    {[_, _], C1} = dets:match_object(T, '_', 2),
1524    {[_, _], C2} = dets:match_object(C1),
1525    '$end_of_table' = dets:match_object(C2),
1526    {[_, _], C3} = dets:match_object(T, {1, '_', '_'}, 100),
1527    '$end_of_table' = dets:match_object(C3),
1528    '$end_of_table' = dets:match_object(T, {'_'}, default),
1529    dets:safe_fixtable(T, false),
1530
1531    dets:safe_fixtable(T, true),
1532    {[_, _], C30} = dets:match(T, '$1', 2),
1533    {[_, _], C31} = dets:match(C30),
1534    '$end_of_table' = dets:match(C31),
1535    {[_, _], C32} = dets:match(T, {1, '$1', '_'}, 100),
1536    '$end_of_table' = dets:match(C32),
1537    '$end_of_table' = dets:match(T, {'_'}, default),
1538    dets:safe_fixtable(T, false),
1539    [[1],[1],[2],[2]] = sort(dets:match(T, {'$1','_','_'})),
1540
1541    %% delete and insert while chunking
1542    %% (this case almost worthless after changes in OTP-5232)
1543    ok = dets:match_delete(T, '_'),
1544    L500 = seq(1, 500),
1545    Fun = fun(X) -> ok = dets:insert(T, {X, a, b, c, d}) end,
1546    foreach(Fun, L500),
1547    %% Select one object DI in L3 below to be deleted.
1548    {_, TmpCont} = dets:match_object(T, '_', 200),
1549    {_, TmpCont1} = dets:match_object(TmpCont),
1550    {TTL, _} = dets:match_object(TmpCont1),
1551    DI = hd(TTL),
1552    dets:safe_fixtable(T, true),
1553    {L1, C20} = dets:match_object(T, '_', 200),
1554    true = 200 =< length(L1),
1555    ok = dets:match_delete(T, {'2','_','_'}), % no match
1556    ok = dets:match_delete(T, DI), % last object
1557    Tiny = {1050},
1558    ok = dets:insert(T, Tiny),
1559    true = member(Tiny, dets:match_object(T, '_')),
1560    {_L2, C21} = dets:match_object(C20),
1561    {_L3, _C22} = dets:match_object(C21),
1562    %% It used to be that Tiny was not visible here, but since the
1563    %% scanning of files was changed to inspect the free lists every
1564    %% now and then it may very well be visible here.
1565    %% false = member(Tiny, _L3),
1566    %% DI used to visible here, but the above mentioned modification
1567    %% has changed that; it may or may not be visible.
1568    %% true = member(DI, _L3),
1569    dets:safe_fixtable(T, false),
1570    true = dets:member(T, 1050),
1571    true = member(Tiny, dets:match_object(T, '_')),
1572    false = member(DI, dets:match_object(T, '_')),
1573
1574    ok = dets:close(T),
1575    file:delete(Fname),
1576
1577    N = 100,
1578    {ok, _} = dets:open_file(T, [{estimated_no_objects,N} | Args]),
1579    ok = ins(T, N),
1580    Obj = {66,{item,number,66}},
1581    Spec = {'_','_'},
1582    {ok, ObjPos} = dets:where(T, Obj),
1583    ok = dets:close(T),
1584    %% Damaged object.
1585    crash(Fname, ObjPos+12),
1586    {ok, _} = dets:open_file(T, Args),
1587    io:format("Expect corrupt table:~n"),
1588    case ins(T, N) of
1589	      ok ->
1590                  bad_object(dets:sync(T), Fname);
1591	      Else ->
1592		  bad_object(Else, Fname)
1593	  end,
1594    io:format("Expect corrupt table:~n"),
1595    bad_object(dets:match(T, Spec), Fname),
1596    io:format("Expect corrupt table:~n"),
1597    bad_object(dets:match_delete(T, Spec), Fname),
1598    bad_object(dets:close(T), Fname),
1599    file:delete(Fname),
1600
1601    {ok, _} = dets:open_file(T, [{estimated_no_objects,N} | Args]),
1602    ok = ins(T, N),
1603    {ok, ObjPos2} = dets:where(T, Obj),
1604    ok = dets:close(T),
1605
1606    %% Damaged size of object.
1607    CrashPos = 1,
1608    crash(Fname, ObjPos2+CrashPos),
1609    {ok, _} = dets:open_file(T, Args),
1610    case dets:insert_new(T, Obj) of % OTP-12024
1611        ok ->
1612            bad_object(dets:sync(T), Fname);
1613        Else3 ->
1614            bad_object(Else3, Fname)
1615    end,
1616    io:format("Expect corrupt table:~n"),
1617    case ins(T, N) of
1618        ok ->
1619            bad_object(dets:sync(T), Fname);
1620        Else2 ->
1621            bad_object(Else2, Fname)
1622    end,
1623    %% Just echoes...
1624    bad_object(dets:match(T, Spec), Fname),
1625    bad_object(dets:match_delete(T, Spec), Fname),
1626    bad_object(dets:close(T), Fname),
1627    file:delete(Fname),
1628
1629    {ok, _} = dets:open_file(T, [{estimated_no_objects,N} | Args]),
1630    ok = ins(T, N),
1631    {ok, ObjPos3} = dets:where(T, Obj),
1632    ok = dets:close(T),
1633
1634    %% match_delete finds an error
1635    CrashPos3 = 16,
1636    crash(Fname, ObjPos3+CrashPos3),
1637    {ok, _} = dets:open_file(T, Args),
1638    bad_object(dets:match_delete(T, Spec), Fname),
1639    bad_object(dets:close(T), Fname),
1640    file:delete(Fname),
1641
1642    %% The key is not fixed, but not all objects with the key are removed.
1643    {ok, _} = dets:open_file(T, Args),
1644    ok = dets:insert(T, [{1,a},{1,b},{1,c},{1,a},{1,b},{1,c}]),
1645    6 = dets:info(T, size),
1646    ok = dets:match_delete(T, {'_',a}),
1647    4 = dets:info(T, size),
1648    [{1,b},{1,b},{1,c},{1,c}] =
1649	sort(dets:match_object(T,{'_','_'})),
1650    ok = dets:close(T),
1651    file:delete(Fname),
1652
1653    check_pps(P0),
1654    ok.
1655
1656%% Selecting objects of a fixed table.
1657select(Config) when is_list(Config) ->
1658    T = select,
1659    Fname = filename(select, Config),
1660    file:delete(Fname),
1661    P0 = pps(),
1662
1663    Args = [{file,Fname}, {type, duplicate_bag},{estimated_no_objects,550}],
1664    {ok, _} = dets:open_file(T, Args),
1665    ok = dets:insert(T, {1, a, b}),
1666    ok = dets:insert(T, {1, b, a}),
1667    ok = dets:insert(T, {2, a, b}),
1668    ok = dets:insert(T, {2, b, a}),
1669    ok = dets:insert(T, {3, a, b}),
1670    ok = dets:insert(T, {3, b, a}),
1671
1672    %% badarg
1673    MSpec = [{'_',[],['$_']}],
1674    check_badarg(catch dets:select(no_table, MSpec),
1675                 dets, select, [no_table,MSpec]),
1676    check_badarg(catch dets:select(T, <<17>>),
1677                 dets, select, [T,<<17>>]),
1678    check_badarg(catch dets:select(T, []),
1679                 dets, select, [T,[]]),
1680    check_badarg(catch dets:select(T, MSpec, not_a_number),
1681                 dets, select, [T,MSpec,not_a_number]),
1682    {EC, _} = dets:match(T, '_', 1),
1683    check_badarg(catch dets:select(EC),
1684                 dets, select, [EC]),
1685
1686    AllSpec = [{'_',[],['$_']}],
1687
1688    dets:safe_fixtable(T, true),
1689    {[_, _], C1} = dets:select(T, AllSpec, 2),
1690    {[_, _], C2} = dets:select(C1),
1691    {[_, _], C2a} = dets:select(C2),
1692    '$end_of_table' = dets:select(C2a),
1693    {[_, _], C3} = dets:select(T, [{{1,'_','_'},[],['$_']}], 100),
1694    '$end_of_table' = dets:select(C3),
1695    '$end_of_table' = dets:select(T, [{{'_'},[],['$_']}], default),
1696    dets:safe_fixtable(T, false),
1697    Sp1 = [{{1,'_','_'},[],['$_']},{{1,'_','_'},[],['$_']},
1698	   {{2,'_','_'},[],['$_']}],
1699    [_,_,_,_] = dets:select(T, Sp1),
1700    Sp2 = [{{1,'_','_'},[],['$_']},{{1,'_','_'},[],['$_']},
1701	   {{'_','_','_'},[],['$_']}],
1702    [_,_,_,_,_,_] = dets:select(T, Sp2),
1703
1704    AllDeleteSpec = [{'_',[],[true]}],
1705    %% delete and insert while chunking
1706    %% (this case almost worthless after changes in OTP-5232)
1707    6 = dets:select_delete(T, AllDeleteSpec),
1708    L500 = seq(1, 500),
1709    Fun = fun(X) -> ok = dets:insert(T, {X, a, b, c, d}) end,
1710    foreach(Fun, L500),
1711    %% Select one object DI in L3 below to be deleted.
1712    {_, TmpCont} = dets:match_object(T, '_', 200),
1713    {_, TmpCont1} = dets:match_object(TmpCont),
1714    {TTL, _} = dets:match_object(TmpCont1),
1715    DI = hd(TTL),
1716    dets:safe_fixtable(T, true),
1717    {L1, C20} = dets:select(T, AllSpec, 200),
1718    true = 200 =< length(L1),
1719    0 = dets:select_delete(T, [{{2,'_','_'},[],[true]}]),
1720    1 = dets:select_delete(T, [{DI,[],[true]}]), % last object
1721    Tiny = {1050},
1722    ok = dets:insert(T, Tiny),
1723    true = member(Tiny, dets:select(T, AllSpec)),
1724    {_L2, C21} = dets:select(C20),
1725    {_L3, _C22} = dets:select(C21),
1726    %% It used to be that Tiny was not visible here, but since the
1727    %% scanning of files was changed to inspect the free lists every
1728    %% now and then it may very well be visible here.
1729    %% false = member(Tiny, _L3),
1730    %% DI used to visible here, but the above mentioned modification
1731    %% has changed that; it may or may not be visible.
1732    %% true = member(DI, _L3),
1733    true = dets:member(T, 1050),
1734    true = member(Tiny, dets:select(T, AllSpec)),
1735    false = member(DI, dets:select(T, AllSpec)),
1736    dets:safe_fixtable(T, false),
1737    true = dets:member(T, 1050),
1738    true = member(Tiny, dets:select(T, AllSpec)),
1739    false = member(DI, dets:select(T, AllSpec)),
1740    ok = dets:close(T),
1741    file:delete(Fname),
1742
1743    %% The key is not fixed, but not all objects with the key are removed.
1744    {ok, _} = dets:open_file(T, Args),
1745    ok = dets:insert(T, [{1,a},{1,b},{1,c},{1,a},{1,b},{1,c}]),
1746    6 = dets:info(T, size),
1747    2 = dets:select_delete(T, [{{'_',a},[],[true]}]),
1748    4 = dets:info(T, size),
1749    [{1,b},{1,b},{1,c},{1,c}] = sort(dets:select(T, AllSpec)),
1750    ok = dets:close(T),
1751    file:delete(Fname),
1752
1753    check_pps(P0),
1754    ok.
1755
1756%% Test update_counter/1.
1757update_counter(Config) when is_list(Config) ->
1758    T = update_counter,
1759    Fname = filename(select, Config),
1760    file:delete(Fname),
1761    P0 = pps(),
1762
1763    check_badarg(catch dets:update_counter(no_table, 1, 1),
1764                 dets, update_counter, [no_table,1,1]),
1765
1766    Args = [{file,Fname},{keypos,2}],
1767    {ok, _} = dets:open_file(T, [{type,set} | Args]),
1768    {'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)),
1769    ok = dets:insert(T, {1,a}),
1770    {'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)),
1771    ok = dets:insert(T, {0,1}),
1772    {'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)),
1773    ok = dets:insert(T, {0,1,0}),
1774    1 = dets:update_counter(T, 1, 1),
1775    2 = dets:update_counter(T, 1, 1),
1776    6 = dets:update_counter(T, 1, {3,4}),
1777    {'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, {0,3})),
1778    ok = dets:close(T),
1779    file:delete(Fname),
1780
1781    {ok, _} = dets:open_file(T, [{type,bag} | Args]),
1782    ok = dets:insert(T, {0,1,0}),
1783    {'EXIT', {badarg, _}} = (catch dets:update_counter(T, 1, 1)),
1784    ok = dets:close(T),
1785    file:delete(Fname),
1786    check_pps(P0),
1787
1788    ok.
1789
1790%% Call some functions with bad arguments.
1791badarg(Config) when is_list(Config) ->
1792    T = badarg,
1793    Fname = filename(select, Config),
1794    file:delete(Fname),
1795    P0 = pps(),
1796
1797    Args = [{file,Fname},{keypos,3}],
1798    {ok, _} = dets:open_file(T, [{type,set} | Args]),
1799
1800    %% badargs are tested in match, select and fixtable too.
1801
1802    %% open
1803    check_badarg(catch dets:open_file({a,tuple},[]),
1804                 dets, open_file, [{a,tuple},[]]),
1805    check_badarg(catch dets:open_file({a,tuple}),
1806                 dets, open_file,[{a,tuple}]),
1807    check_badarg(catch dets:open_file(file,[foo]),
1808                 dets, open_file, [file,[foo]]),
1809    check_badarg(catch dets:open_file({hej,san},[{type,set}|3]),
1810                 dets, open_file, [{hej,san},[{type,set}|3]]),
1811
1812    %% insert
1813    check_badarg(catch dets:insert(no_table, {1,2}),
1814                 dets, insert, [no_table,{1,2}]),
1815    check_badarg(catch dets:insert(no_table, [{1,2}]),
1816                 dets, insert, [no_table,[{1,2}]]),
1817    check_badarg(catch dets:insert(T, {1,2}),
1818                 dets, insert, [T,{1,2}]),
1819    check_badarg(catch dets:insert(T, [{1,2}]),
1820                 dets, insert, [T,[{1,2}]]),
1821    check_badarg(catch dets:insert(T, [{1,2,3} | 3]),
1822                 dets, insert, [T,[{1,2,3}|3]]),
1823
1824    %% lookup{_keys}
1825    check_badarg(catch dets:lookup_keys(T, []),
1826                 dets, lookup_keys, [badarg,[]]),
1827    check_badarg(catch dets:lookup(no_table, 1),
1828                 dets, lookup, [no_table,1]),
1829    check_badarg(catch dets:lookup_keys(T, [1 | 2]),
1830                 dets, lookup_keys, [T,[1|2]]),
1831
1832    %% member
1833    check_badarg(catch dets:member(no_table, 1),
1834                 dets, member, [no_table,1]),
1835
1836    %% sync
1837    check_badarg(catch dets:sync(no_table),
1838                 dets, sync, [no_table]),
1839
1840    %% delete{_keys}
1841    check_badarg(catch dets:delete(no_table, 1),
1842                 dets, delete, [no_table,1]),
1843
1844    %% delete_object
1845    check_badarg(catch dets:delete_object(no_table, {1,2,3}),
1846                 dets, delete_object, [no_table,{1,2,3}]),
1847    check_badarg(catch dets:delete_object(T, {1,2}),
1848                 dets, delete_object, [T,{1,2}]),
1849    check_badarg(catch dets:delete_object(no_table, [{1,2,3}]),
1850                 dets, delete_object, [no_table,[{1,2,3}]]),
1851    check_badarg(catch dets:delete_object(T, [{1,2}]),
1852                 dets, delete_object, [T,[{1,2}]]),
1853    check_badarg(catch dets:delete_object(T, [{1,2,3} | 3]),
1854                 dets, delete_object, [T,[{1,2,3}|3]]),
1855
1856    %% first,next,slot
1857    check_badarg(catch dets:first(no_table),
1858                 dets, first, [no_table]),
1859    check_badarg(catch dets:next(no_table, 1),
1860                 dets, next, [no_table,1]),
1861    check_badarg(catch dets:slot(no_table, 0),
1862                 dets, slot, [no_table,0]),
1863
1864    %% info
1865    undefined = dets:info(no_table),
1866    undefined = dets:info(no_table, foo),
1867    undefined = dets:info(T, foo),
1868
1869    %% match_delete
1870    check_badarg(catch dets:match_delete(no_table, '_'),
1871                 dets, match_delete, [no_table,'_']),
1872
1873    %% delete_all_objects
1874    check_badarg(catch dets:delete_all_objects(no_table),
1875                 dets, delete_all_objects, [no_table]),
1876
1877    %% select_delete
1878    MSpec = [{'_',[],['$_']}],
1879    check_badarg(catch dets:select_delete(no_table, MSpec),
1880                 dets, select_delete, [no_table,MSpec]),
1881    check_badarg(catch dets:select_delete(T, <<17>>),
1882                 dets, select_delete, [T, <<17>>]),
1883
1884    %% traverse, fold
1885    TF = fun(_) -> continue end,
1886    check_badarg(catch dets:traverse(no_table, TF),
1887                 dets, traverse, [no_table,TF]),
1888    FF = fun(_, A) -> A end,
1889    check_badarg(catch dets:foldl(FF, [], no_table),
1890                 dets, foldl, [FF,[],no_table]),
1891    check_badarg(catch dets:foldr(FF, [], no_table),
1892                 dets, foldl, [FF,[],no_table]),
1893
1894    %% close
1895    ok = dets:close(T),
1896    {error, not_owner} = dets:close(T),
1897    {error, not_owner} = dets:close(T),
1898
1899    %% init_table
1900    IF = fun(X) -> X end,
1901    check_badarg(catch dets:init_table(no_table, IF),
1902                 dets, init_table, [no_table,IF,[]]),
1903    check_badarg(catch dets:init_table(no_table, IF, []),
1904                 dets, init_table, [no_table,IF,[]]),
1905
1906    %% from_ets
1907    Ets = ets:new(ets,[]),
1908    check_badarg(catch dets:from_ets(no_table, Ets),
1909                 dets, from_ets, [no_table,Ets]),
1910    ets:delete(Ets),
1911
1912    {ok, T} = dets:open_file(T, Args),
1913    {error,incompatible_arguments} =
1914	dets:open_file(T, [{type,bag} | Args]),
1915    ok = dets:close(T),
1916
1917    file:delete(Fname),
1918    check_pps(P0),
1919    ok.
1920
1921%% Test the write cache for sets.
1922cache_sets(Config) when is_list(Config) ->
1923    Small = 2,
1924    cache_sets(Config, {0,0}, false, Small),
1925    cache_sets(Config, {0,0}, true, Small),
1926    cache_sets(Config, {5000,5000}, false, Small),
1927    cache_sets(Config, {5000,5000}, true, Small),
1928    %% Objects of size greater than 2 kB.
1929    Big = 1200,
1930    cache_sets(Config, {0,0}, false, Big),
1931    cache_sets(Config, {0,0}, true, Big),
1932    cache_sets(Config, {5000,5000}, false, Big),
1933    cache_sets(Config, {5000,5000}, true, Big),
1934    ok.
1935
1936cache_sets(Config, DelayedWrite, Extra, Sz) ->
1937    %% Extra = bool(). Insert tuples until the tested key is not alone.
1938    %% Sz = integer(). Size of the inserted tuples.
1939
1940    T = cache,
1941    Fname = filename(cache, Config),
1942    file:delete(Fname),
1943    P0 = pps(),
1944
1945    {ok, _} = dets:open_file(T,[{file,Fname}, {type,set},
1946                                {delayed_write, DelayedWrite}]),
1947
1948    Dups = 1,
1949    {Key, OtherKeys} =
1950	if
1951	    Extra ->
1952		%% Insert enough to get three keys in some slot.
1953		dets:safe_fixtable(T, true),
1954		insert_objs(T, 1, Sz, Dups);
1955	    true ->
1956		{1,[]}
1957	  end,
1958    Tuple = erlang:make_tuple(Sz, Key),
1959    ok = dets:delete(T, Key),
1960    ok = dets:sync(T),
1961
1962    %% The values of keys in the same slot as Key are checked.
1963    OtherValues = sort(lookup_keys(T, OtherKeys)),
1964
1965    ok = dets:insert(T, Tuple),
1966    [Tuple] = dets:lookup(T, Key),
1967    true = dets:member(T, Key),
1968    ok = dets:insert(T, [Tuple,Tuple]),
1969    %% If no delay, the cache gets filled immediately, and written.
1970    [Tuple] = dets:lookup_keys(T, [Key,a,b,c,d,e,f]),
1971    true = dets:member(T, Key),
1972
1973    %% If delay, this happens without file access.
1974    ok = dets:delete(T,Key),
1975    ok = dets:insert(T,Tuple),
1976    ok = dets:insert(T,Tuple),
1977    [Tuple] = dets:lookup(T, Key),
1978    true = dets:member(T, Key),
1979    ok = dets:sync(T),
1980    [Tuple] = dets:lookup(T, Key),
1981    true = dets:member(T, Key),
1982
1983    %% Key's objects are is on file only,
1984    %% key 'toto' in the cache (if there is one).
1985    ok = dets:delete(T,toto),
1986    ok = dets:insert(T,[{toto,b},{toto,b}]),
1987    true = sort([Tuple,{toto,b}]) =:=
1988        sort(dets:lookup_keys(T, [Key,toto])),
1989    true = dets:member(T, toto),
1990
1991    ok = dets:delete(T, Key),
1992    ok = dets:sync(T),
1993    false = dets:member(T, Key),
1994    Size = dets:info(T, size),
1995
1996    %% No object with the key on the file.
1997    %% Delete, add one object.
1998    Size1 = Size + 2,
1999    del_and_ins(key, T, Size1, Tuple, Key, 1),
2000    del_and_ins(object, T, Size1, Tuple, Key, 1),
2001    del_and_ins(both, T, Size1, Tuple, Key, 1),
2002
2003    %% One object with the key on the file.
2004    %% Delete it, add one object.
2005    Size2 = Size + 2,
2006    del_and_ins(key, T, Size2, Tuple, Key, 1),
2007    del_and_ins(object, T, Size2, Tuple, Key, 1),
2008    del_and_ins(both, T, Size2, Tuple, Key, 1),
2009
2010    %% Overwrite an old objekt with exactly the same size.
2011    Element = case element(2, Tuple) of
2012		  255 -> 254;
2013		  E -> E + 1
2014	      end,
2015    Tuple2 = setelement(2, Tuple, Element),
2016    ok = dets:sync(T),
2017    ok = dets:insert(T, Tuple2),
2018    [Tuple2] = dets:lookup(T, Key),
2019    true = dets:member(T, Key),
2020    ok = dets:sync(T),
2021    [Tuple2] = dets:lookup(T, Key),
2022    true = dets:member(T, Key),
2023
2024    ok = dets:insert(T, {3,a}),
2025    ok = dets:insert(T, {3,b}),
2026    ok = dets:delete_object(T, {3,c}),
2027    ok = dets:delete_object(T, {3,d}),
2028    [{3,b}] = dets:lookup(T, 3),
2029
2030    ok = dets:delete(T, 3),
2031    ok = dets:delete_object(T, {3,c}),
2032    ok = dets:delete_object(T, {3,d}),
2033    [] = dets:lookup(T, 3),
2034
2035    OtherValues = sort(lookup_keys(T, OtherKeys)),
2036    if
2037	Extra ->
2038	    %% Let the table grow a while, if it needs to.
2039	    All1 = get_all_objects(T),
2040	    dets:safe_fixtable(T, false),
2041	    timer:sleep(1000),
2042	    OtherValues = sort(lookup_keys(T, OtherKeys)),
2043	    dets:safe_fixtable(T, true),
2044	    All2 = get_all_objects(T),
2045	    FAll2 = get_all_objects_fast(T),
2046	    true = sort(All2) =:= sort(FAll2),
2047            case symdiff(All1, All2) of
2048		{[],[]} ->  ok;
2049		{X,Y} ->
2050		    NoBad = length(X) + length(Y),
2051		    ct:fail({sets,DelayedWrite,Extra,Sz,NoBad})
2052	    end;
2053	true ->
2054	    ok
2055    end,
2056    ok = dets:close(T),
2057
2058    file:delete(Fname),
2059    check_pps(P0),
2060    ok.
2061
2062%% Test the write cache for bags.
2063cache_bags(Config) when is_list(Config) ->
2064    Small = 2,
2065    cache_bags(Config, {0,0}, false, Small),
2066    cache_bags(Config, {0,0}, true, Small),
2067    cache_bags(Config, {5000,5000}, false, Small),
2068    cache_bags(Config, {5000,5000}, true, Small),
2069    %% Objects of size greater than 2 kB.
2070    Big = 1200,
2071    cache_bags(Config, {0,0}, false, Big),
2072    cache_bags(Config, {0,0}, true, Big),
2073    cache_bags(Config, {5000,5000}, false, Big),
2074    cache_bags(Config, {5000,5000}, true, Big),
2075    ok.
2076
2077cache_bags(Config, DelayedWrite, Extra, Sz) ->
2078    %% Extra = bool(). Insert tuples until the tested key is not alone.
2079    %% Sz = integer(). Size of the inserted tuples.
2080
2081    T = cache,
2082    Fname = filename(cache, Config),
2083    file:delete(Fname),
2084    P0 = pps(),
2085
2086    {ok, _} = dets:open_file(T,[{file,Fname}, {type,bag},
2087                                {delayed_write, DelayedWrite}]),
2088
2089    Dups = 1,
2090    {Key, OtherKeys} =
2091	if
2092	    Extra ->
2093		%% Insert enough to get three keys in some slot.
2094		dets:safe_fixtable(T, true),
2095		insert_objs(T, 1, Sz, Dups);
2096	    true ->
2097		{1,[]}
2098	  end,
2099    Tuple = erlang:make_tuple(Sz, Key),
2100    ok = dets:delete(T, Key),
2101    ok = dets:sync(T),
2102
2103    %% The values of keys in the same slot as Key are checked.
2104    OtherValues = sort(lookup_keys(T, OtherKeys)),
2105
2106    ok = dets:insert(T, Tuple),
2107    [Tuple] = dets:lookup(T, Key),
2108    true = dets:member(T, Key),
2109    ok = dets:insert(T, [Tuple,Tuple]),
2110    %% If no delay, the cache gets filled immediately, and written.
2111    [Tuple] = dets:lookup_keys(T, [Key,a,b,c,d,e,f]),
2112    true = dets:member(T, Key),
2113
2114    %% If delay, this happens without file access.
2115    %% (This is no longer true; cache lookup has been simplified.)
2116    ok = dets:delete(T,Key),
2117    ok = dets:insert(T,Tuple),
2118    ok = dets:insert(T,Tuple),
2119    [Tuple] = dets:lookup(T, Key),
2120    true = dets:member(T, Key),
2121    ok = dets:sync(T),
2122    [Tuple] = dets:lookup(T, Key),
2123    true = dets:member(T, Key),
2124
2125    %% Key's objects are is on file only,
2126    %% key toto in the cache (if there is one).
2127    ok = dets:delete(T,toto),
2128    false = dets:member(T, toto),
2129    ok = dets:insert(T,[{toto,b},{toto,b}]),
2130    true = sort([Tuple,{toto,b}]) =:=
2131        sort(dets:lookup_keys(T, [Key,toto])),
2132    true = dets:member(T, toto),
2133
2134    ok = dets:delete(T, Key),
2135    ok = dets:sync(T),
2136    Size = dets:info(T, size),
2137
2138    %% No object with the key on the file.
2139    %% Delete, add one object.
2140    Size1 = Size + 2,
2141    del_and_ins(key, T, Size1, Tuple, Key, 1),
2142    del_and_ins(object, T, Size1, Tuple, Key, 1),
2143    del_and_ins(both, T, Size1, Tuple, Key, 1),
2144
2145    %% One object with the key on the file.
2146    %% Delete it, add one object.
2147    Size2 = Size + 2,
2148    del_and_ins(key, T, Size2, Tuple, Key, 1),
2149    del_and_ins(object, T, Size2, Tuple, Key, 1),
2150    del_and_ins(both, T, Size2, Tuple, Key, 1),
2151
2152    %% Overwrite an objekt on file with the same object.
2153    ok = dets:insert(T, Tuple),
2154    ok = dets:sync(T),
2155    [Tuple2] = dets:lookup(T, Key),
2156    true = dets:member(T, Key),
2157    ok = dets:insert(T, Tuple),
2158    ok = dets:sync(T),
2159    [Tuple2] = dets:lookup(T, Key),
2160    true = dets:member(T, Key),
2161
2162    %% A mix of insert and delete.
2163    ok = dets:delete(T, Key),
2164    ok = dets:sync(T),
2165    ok = dets:delete(T, Key),
2166    ok = dets:insert(T, {Key,foo}),
2167    ok = dets:insert(T, {Key,bar}),
2168    [{Key,bar},{Key,foo}] = sort(dets:lookup(T, Key)),
2169    true = dets:member(T, Key),
2170    ok = dets:delete_object(T, {Key,foo}),
2171    ok = dets:insert(T, {Key,kar}),
2172    [{Key,bar},{Key,kar}] = sort(dets:lookup(T, Key)),
2173    true = dets:member(T, Key),
2174    ok = dets:insert(T, [{Key,kar},{Key,kar}]),
2175    [{Key,bar},{Key,kar}] = sort(dets:lookup(T, Key)),
2176    true = dets:member(T, Key),
2177    ok = dets:delete_object(T, {Key,bar}),
2178    ok = dets:delete_object(T, {Key,kar}),
2179    [] = dets:lookup(T, Key),
2180    false = dets:member(T, Key),
2181    ok = dets:sync(T),
2182    [] = dets:lookup(T, Key),
2183    false = dets:member(T, Key),
2184
2185    OtherValues = sort(lookup_keys(T, OtherKeys)),
2186    if
2187	Extra ->
2188	    %% Let the table grow for a while, if it needs to.
2189	    All1 = get_all_objects(T),
2190	    dets:safe_fixtable(T, false),
2191	    timer:sleep(1200),
2192	    OtherValues = sort(lookup_keys(T, OtherKeys)),
2193	    dets:safe_fixtable(T, true),
2194	    All2 = get_all_objects(T),
2195	    FAll2 = get_all_objects_fast(T),
2196	    true = sort(All2) =:= sort(FAll2),
2197            case symdiff(All1, All2) of
2198		{[],[]} ->  ok;
2199		{X,Y} ->
2200		    NoBad = length(X) + length(Y),
2201		    ct:fail({bags,DelayedWrite,Extra,Sz,NoBad})
2202	    end;
2203	true ->
2204	    ok
2205    end,
2206    ok = dets:close(T),
2207    file:delete(Fname),
2208
2209    %% Second object of a key added and looked up simultaneously.
2210    R1 = {index_test,1,2,3,4},
2211    R2 = {index_test,2,2,13,14},
2212    R3 = {index_test,1,12,13,14},
2213    {ok, _} = dets:open_file(T,[{type,bag}, {keypos,2},{file,Fname}]),
2214    ok = dets:insert(T,R1),
2215    ok = dets:sync(T),
2216    ok = dets:insert(T,R2),
2217    ok = dets:sync(T),
2218    ok = dets:insert(T,R3),
2219    [R1,R3] = sort(dets:lookup(T,1)),
2220    true = dets:member(T, 1),
2221    [R1,R3] = sort(dets:lookup(T,1)),
2222    true = dets:member(T, 1),
2223    ok = dets:close(T),
2224    file:delete(Fname),
2225
2226    check_pps(P0),
2227    ok.
2228
2229%% Test the write cache for duplicate bags.
2230cache_duplicate_bags(Config) when is_list(Config) ->
2231    Small = 2,
2232    cache_dup_bags(Config, {0,0}, false, Small),
2233    cache_dup_bags(Config, {0,0}, true, Small),
2234    cache_dup_bags(Config, {5000,5000}, false, Small),
2235    cache_dup_bags(Config, {5000,5000}, true, Small),
2236    %% Objects of size greater than 2 kB.
2237    Big = 1200,
2238    cache_dup_bags(Config, {0,0}, false, Big),
2239    cache_dup_bags(Config, {0,0}, true, Big),
2240    cache_dup_bags(Config, {5000,5000}, false, Big),
2241    cache_dup_bags(Config, {5000,5000}, true, Big).
2242
2243cache_dup_bags(Config, DelayedWrite, Extra, Sz) ->
2244    %% Extra = bool(). Insert tuples until the tested key is not alone.
2245    %% Sz = integer(). Size of the inserted tuples.
2246
2247    T = cache,
2248    Fname = filename(cache, Config),
2249    file:delete(Fname),
2250    P0 = pps(),
2251
2252    {ok, _} = dets:open_file(T,[{file,Fname}, {type,duplicate_bag},
2253                                {delayed_write, DelayedWrite}]),
2254
2255    Dups = 2,
2256    {Key, OtherKeys} =
2257	if
2258	    Extra ->
2259		%% Insert enough to get three keys in some slot.
2260		dets:safe_fixtable(T, true),
2261		insert_objs(T, 1, Sz, Dups);
2262	    true ->
2263		{1,[]}
2264	  end,
2265    Tuple = erlang:make_tuple(Sz, Key),
2266    ok = dets:delete(T, Key),
2267    ok = dets:sync(T),
2268    false = dets:member(T, Key),
2269
2270    %% The values of keys in the same slot as Key are checked.
2271    OtherValues = sort(lookup_keys(T, OtherKeys)),
2272
2273    ok = dets:insert(T, Tuple),
2274    [Tuple] = dets:lookup(T, Key),
2275    true = dets:member(T, Key),
2276    ok = dets:insert(T, [Tuple,Tuple]),
2277    %% If no delay, the cache gets filled immediately, and written.
2278    [Tuple,Tuple,Tuple] = dets:lookup_keys(T, [Key,a,b,c,d,e,f]),
2279    true = dets:member(T, Key),
2280
2281    %% If delay, this happens without file access.
2282    %% (This is no longer true; cache lookup has been simplified.)
2283    ok = dets:delete(T,Key),
2284    ok = dets:insert(T,Tuple),
2285    ok = dets:insert(T,Tuple),
2286    [Tuple,Tuple] = dets:lookup(T, Key),
2287    true = dets:member(T, Key),
2288    ok = dets:sync(T),
2289    [Tuple,Tuple] = dets:lookup(T, Key),
2290    true = dets:member(T, Key),
2291
2292    %% One object in the cache, one on the file.
2293    ok = dets:delete(T,Key),
2294    ok = dets:insert(T,Tuple),
2295    ok = dets:sync(T),
2296    ok = dets:insert(T,Tuple),
2297    true = dets:member(T, Key), % should not read the file, but it does..
2298
2299    %% Key's objects are is on file only,
2300    %% key toto in the cache (if there is one).
2301    ok = dets:delete(T,toto),
2302    ok = dets:insert(T,[{toto,b},{toto,b}]),
2303    true = sort([Tuple,Tuple,{toto,b},{toto,b}]) =:=
2304                 sort(dets:lookup_keys(T, [Key,toto])),
2305    true = dets:member(T, toto),
2306    Size = dets:info(T, size),
2307
2308    %% Two objects with the same key on the file.
2309    %% Delete them, add two objects.
2310    del_and_ins(key, T, Size, Tuple, Key, 2),
2311
2312    del_and_ins(object, T, Size, Tuple, Key, 2),
2313    del_and_ins(both, T, Size, Tuple, Key, 2),
2314
2315    %% Two objects with the same key on the file.
2316    %% Delete them, add three objects.
2317    del_and_ins(key, T, Size, Tuple, Key, 3),
2318    del_and_ins(object, T, Size, Tuple, Key, 3),
2319    del_and_ins(both, T, Size, Tuple, Key, 3),
2320
2321    %% Two objects with the same key on the file.
2322    %% Delete them, add one object.
2323    del_and_ins(key, T, Size, Tuple, Key, 1),
2324    del_and_ins(object, T, Size, Tuple, Key, 1),
2325    del_and_ins(both, T, Size, Tuple, Key, 1),
2326
2327    OtherValues = sort(lookup_keys(T, OtherKeys)),
2328    if
2329	Extra ->
2330	    %% Let the table grow for a while, if it needs to.
2331	    All1 = get_all_objects(T),
2332	    dets:safe_fixtable(T, false),
2333	    timer:sleep(1200),
2334	    OtherValues = sort(lookup_keys(T, OtherKeys)),
2335	    dets:safe_fixtable(T, true),
2336	    All2 = get_all_objects(T),
2337	    FAll2 = get_all_objects_fast(T),
2338	    true = sort(All2) =:= sort(FAll2),
2339            case symdiff(All1, All2) of
2340		{[],[]} ->  ok;
2341		{X,Y} ->
2342		    NoBad = length(X) + length(Y),
2343		    ct:fail({dup_bags,DelayedWrite,Extra,Sz,NoBad})
2344	    end;
2345	true ->
2346	    ok
2347    end,
2348    ok = dets:close(T),
2349
2350    file:delete(Fname),
2351    check_pps(P0),
2352    ok.
2353
2354lookup_keys(_T, []) ->
2355    [];
2356lookup_keys(T, Keys) ->
2357    dets:lookup_keys(T, Keys).
2358
2359del_and_ins(W, T, Size, Obj, Key, N) ->
2360    case W of
2361	object ->
2362	    ok = dets:delete_object(T, Obj);
2363	key ->
2364
2365	    ok = dets:delete(T, Key);
2366	both ->
2367	    ok = dets:delete(T, Key),
2368	    ok = dets:delete_object(T, Obj)
2369    end,
2370    Objs = duplicate(N, Obj),
2371    [] = dets:lookup(T, Key),
2372    ok = dets:insert(T, Objs),
2373    Objs = dets:lookup_keys(T, [snurrespratt,Key]),
2374    true = Size + length(Objs)-2 =:= dets:info(T, size),
2375    Objs = dets:lookup(T, Key).
2376
2377
2378insert_objs(T, N, Sz, Dups) ->
2379    Seq = seq(N,N+255),
2380    L0 = map(fun(I) -> erlang:make_tuple(Sz, I) end, Seq),
2381    L = append(duplicate(Dups, L0)),
2382    ok = dets:insert(T, L),
2383    case search_slot(T, 0) of
2384        false ->
2385            insert_objs(T, N+256, Sz, Dups);
2386        Keys ->
2387            Keys
2388    end.
2389
2390search_slot(T, I) ->
2391    case dets:slot(T, I) of
2392        '$end_of_table' ->
2393            false;
2394        Objs ->
2395            case usort(map(fun(X) -> element(1, X) end, Objs)) of
2396                [_, Key, _ | _] = Keys0 ->
2397                    Keys = delete(Key, Keys0),
2398                    {Key, Keys};
2399                _ ->
2400                    search_slot(T, I+1)
2401            end
2402    end.
2403
2404symdiff(L1, L2) ->
2405    {X, _, Y} =
2406	sofs:symmetric_partition(sofs:set(L1), sofs:set(L2)),
2407    {sofs:to_external(X), sofs:to_external(Y)}.
2408
2409%% Test read-only tables and traversal caused crashes.
2410otp_4208(Config) when is_list(Config) ->
2411    Tab = otp_4208,
2412    FName = filename(Tab, Config),
2413    Expected = sort([{3,ghi,12},{1,abc,10},{4,jkl,13},{2,def,11}]),
2414
2415    file:delete(FName),
2416    {ok, Tab} = dets:open_file(Tab, [{file,FName}]),
2417    ok = dets:insert(Tab, [{1,abc,10},{2,def,11},{3,ghi,12},{4,jkl,13}]),
2418    Expected = sort(dets:traverse(Tab, fun(X) -> {continue, X} end)),
2419    ok = dets:close(Tab),
2420
2421    {ok, Tab} = dets:open_file(Tab, [{access, read},{file,FName}]),
2422    Expected = sort(dets:traverse(Tab, fun(X) -> {continue, X} end)),
2423    ok = dets:close(Tab),
2424    file:delete(FName),
2425
2426    ok.
2427
2428%% Test read-only tables and growth.
2429otp_4989(Config) when is_list(Config) ->
2430    Tab = otp_4989,
2431    FName = filename(Tab, Config),
2432
2433    %% Do exactly as in the error report.
2434    _Ets = ets:new(Tab, [named_table]),
2435    ets_init(Tab, 100000),
2436    {ok, Tab} =
2437        dets:open_file(Tab, [{access, read_write}, {file,FName}, {keypos,2}]),
2438    ok = dets:from_ets(Tab, Tab),
2439    ok = dets:close(Tab),
2440    %% Restore.
2441     {ok, Tab} =
2442        dets:open_file(Tab, [{access, read}, {keypos, 2}, {file, FName}]),
2443    true = ets:delete_all_objects(Tab),
2444    true = ets:from_dets(Tab, Tab),
2445    ok = dets:close(Tab),
2446    ets:delete(Tab),
2447    file:delete(FName),
2448    ok.
2449
2450ets_init(_Tab, 0) ->
2451    ok;
2452ets_init(Tab, N) ->
2453    ets:insert(Tab, {N,N}),
2454    ets_init(Tab, N - 1).
2455
2456%% OTP-8898. Truncated Dets file.
2457otp_8898(Config) when is_list(Config) ->
2458    Tab = otp_8898,
2459    FName = filename(Tab, Config),
2460
2461    Server = self(),
2462
2463    file:delete(FName),
2464    {ok, _} = dets:open_file(Tab,[{file, FName}]),
2465    [P1,P2,P3] = new_clients(3, Tab),
2466
2467    Seq = [{P1,[sync]},{P2,[{lookup,1,[]}]},{P3,[{insert,{1,b}}]}],
2468    atomic_requests(Server, Tab, [[]], Seq),
2469    true = get_replies([{P1,ok},{P2,ok},{P3,ok}]),
2470    ok = dets:close(Tab),
2471    {ok, _} = dets:open_file(Tab,[{file, FName}]),
2472    file:delete(FName),
2473
2474    ok.
2475
2476%% OTP-8899. Several clients. Updated Head was ignored.
2477otp_8899(Config) when is_list(Config) ->
2478    Tab = many_clients,
2479    FName = filename(Tab, Config),
2480
2481    Server = self(),
2482
2483    file:delete(FName),
2484    {ok, _} = dets:open_file(Tab,[{file, FName}]),
2485    [P1,P2,P3,P4] = new_clients(4, Tab),
2486
2487    MC = [Tab],
2488    Seq6a = [{P1,[{insert,[{used_to_be_skipped_by,match}]},
2489                  {lookup,1,[{1,a}]}]},
2490             {P2,[{verbose,true,MC}]},
2491             {P3,[{lookup,1,[{1,a}]}]}, {P4,[{verbose,true,MC}]}],
2492    atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq6a),
2493    true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
2494    [{1,a},{2,b},{3,c},{used_to_be_skipped_by,match}] =
2495        lists:sort(dets:match_object(Tab, '_')),
2496    _ = dets:close(Tab),
2497    file:delete(FName),
2498
2499    ok.
2500
2501%% Test several clients accessing a table simultaneously.
2502many_clients(Config) when is_list(Config) ->
2503    Tab = many_clients,
2504    FName = filename(Tab, Config),
2505
2506    Server = self(),
2507
2508    file:delete(FName),
2509    P0 = pps(),
2510    {ok, _} = dets:open_file(Tab,[{file, FName}]),
2511    [P1,P2,P3,P4] = new_clients(4, Tab),
2512
2513    %% dets:init_table/2 is used for making sure that all processes
2514    %% start sending requests before the Dets process begins to handle
2515    %% them; the requests arrive "in parallel".
2516
2517    %% Four processes accessing the same table at almost the same time.
2518
2519    %% One key is read, updated, and read again.
2520    Seq1 = [{P1,[{lookup,1,[{1,a}]}]}, {P2,[{insert,{1,b}}]},
2521	    {P3,[{lookup,1,[{1,b}]}]}, {P4,[{lookup,1,[{1,b}]}]}],
2522    atomic_requests(Server, Tab, [[{1,a}]], Seq1),
2523    true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
2524
2525    %% Different keys read by different processes
2526    Seq2 = [{P1,[{member,1,true}]}, {P2,[{lookup,2,[{2,b}]}]},
2527	    {P3,[{lookup,1,[{1,a}]}]}, {P4,[{lookup,3,[{3,c}]}]}],
2528    atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq2),
2529    true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
2530
2531    %% Reading deleted key.
2532    Seq3 = [{P1,[{delete_key,2}]}, {P2,[{lookup,1,[{1,a}]}]},
2533	    {P3,[{lookup,1,[{1,a}]}]}, {P4,[{member,2,false}]}],
2534    atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq3),
2535    true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
2536
2537    %% Inserting objects.
2538    Seq4 = [{P1,[{insert,[{1,a},{2,b}]}]}, {P2,[{insert,[{2,c},{3,a}]}]},
2539	    {P3,[{insert,[{3,b},{4,d}]}]},
2540	    {P4,[{lookup_keys,[1,2,3,4],[{1,a},{2,c},{3,b},{4,d}]}]}],
2541    atomic_requests(Server, Tab, [], Seq4),
2542    true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
2543
2544    %% Deleting objects.
2545    Seq5 = [{P1,[{delete_object,{1,a}}]}, {P2,[{delete_object,{1,a}}]},
2546	    {P3,[{delete_object,{3,c}}]},
2547	    {P4,[{lookup_keys,[1,2,3,4],[{2,b}]}]}],
2548    atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq5),
2549    true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
2550
2551    %% Some request not streamed.
2552    Seq6 = [{P1,[{lookup,1,[{1,a}]}]}, {P2,[{info,size,3}]},
2553	    {P3,[{lookup,1,[{1,a}]}]}, {P4,[{info,size,3}]}],
2554    atomic_requests(Server, Tab, [[{1,a},{2,b},{3,c}]], Seq6),
2555    true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
2556
2557    %% Some request not streamed.
2558    Seq7 = [{P1,[{insert,[{3,a}]}]}, {P2,[{insert,[{3,b}]}]},
2559	    {P3,[{delete_object,{3,c}}]},
2560	    {P4,[{lookup,3,[{3,b}]}]}],
2561    atomic_requests(Server, Tab, [[{3,c}]], Seq7),
2562    true = get_replies([{P1,ok}, {P2,ok}, {P3,ok}, {P4,ok}]),
2563
2564    put_requests(Server, [{P1,stop},{P2,stop},{P3,stop},{P4,stop}]),
2565    ok = dets:close(Tab),
2566    file:delete(FName),
2567
2568    %% Check that errors are handled correctly by the streaming operators.
2569    {ok, _} = dets:open_file(Tab,[{file, FName}]),
2570    ok = ins(Tab, 100),
2571    Obj = {66,{item,number,66}},
2572    {ok, ObjPos} = dets:where(Tab, Obj),
2573    ok = dets:close(Tab),
2574    %% Damaged object.
2575    crash(FName, ObjPos+12),
2576    {ok, _} = dets:open_file(Tab,[{file, FName}]),
2577    BadObject1 = dets:lookup_keys(Tab, [65,66,67,68,69]),
2578    bad_object(BadObject1, FName),
2579    _Error = dets:close(Tab),
2580    file:delete(FName),
2581
2582    check_pps(P0),
2583
2584    ok.
2585
2586%% Tab is initiated with the objects in Objs. Objs = [[object()]].
2587atomic_requests(Server, Tab, Objs, Req) ->
2588    ok = dets:init_table(Tab, atomic_requests(Server, Objs, Req)).
2589
2590atomic_requests(Server, L, Req) ->
2591    fun(close) ->
2592	    ok;
2593       (read) when [] =:= L ->
2594	    put_requests(Server, Req),
2595	    end_of_input;
2596       (read) ->
2597	    [E | Es] = L,
2598	    {E, atomic_requests(Server, Es, Req)}
2599    end.
2600
2601put_requests(Server, L) ->
2602    lists:foreach(fun({Pid,R}) -> Pid ! {Server,R}, timer:sleep(1) end, L).
2603
2604get_replies(L) ->
2605    lists:all(fun({Pid,Reply}) -> Reply =:= get_reply(Pid) end, L).
2606
2607get_reply(Pid) ->
2608    receive {Pid, Reply} -> Reply end.
2609
2610new_clients(0, _Tab) ->
2611    [];
2612new_clients(N, Tab) ->
2613    [new_client(Tab) | new_clients(N-1, Tab)].
2614
2615new_client(Tab) ->
2616    spawn(?MODULE, client, [self(), Tab]).
2617
2618client(S, Tab) ->
2619    receive
2620	{S, stop} ->
2621	    exit(normal);
2622	{S, ToDo} ->
2623	    Reply = eval(ToDo, Tab),
2624	    case Reply of
2625		{error, _} -> io:format("~p: ~p~n", [self(), Reply]);
2626		_ -> ok
2627	    end,
2628	    S ! {self(), Reply}
2629    end,
2630    client(S, Tab).
2631
2632eval([], _Tab) ->
2633    ok;
2634eval([{verbose,Bool,Expected} | L], Tab) ->
2635    case dets:verbose(Bool) of
2636        Expected -> eval(L, Tab);
2637        Error -> {error, {verbose,Error}}
2638    end;
2639eval([sync | L], Tab) ->
2640    case dets:sync(Tab) of
2641        ok -> eval(L, Tab);
2642        Error -> {error, {sync,Error}}
2643    end;
2644eval([{insert,Stuff} | L], Tab) ->
2645    case dets:insert(Tab, Stuff) of
2646        ok -> eval(L, Tab);
2647        Error -> {error, {insert,Stuff,Error}}
2648    end;
2649eval([{lookup,Key,Expected} | L], Tab) ->
2650    case dets:lookup(Tab, Key) of
2651        Expected -> eval(L, Tab);
2652        Else -> {error, {lookup,Key,Expected,Else}}
2653    end;
2654eval([{lookup_keys,Keys,Expected} | L], Tab) ->
2655    %% Time order is destroyed...
2656    case dets:lookup_keys(Tab, Keys) of
2657        R when is_list(R) ->
2658            case lists:sort(Expected) =:= lists:sort(R) of
2659                true -> eval(L, Tab);
2660                false -> {error, {lookup_keys,Keys,Expected,R}}
2661            end;
2662        Else -> {error, {lookup_keys,Keys,Expected,Else}}
2663    end;
2664eval([{member,Key,Expected} | L], Tab) ->
2665    case dets:member(Tab, Key) of
2666        Expected -> eval(L, Tab);
2667        Else -> {error, {member,Key,Expected,Else}}
2668    end;
2669eval([{delete_key,Key} | L], Tab) ->
2670    case dets:delete(Tab, Key) of
2671        ok -> eval(L, Tab);
2672        Else -> {error, {delete_key,Key,Else}}
2673    end;
2674eval([{delete_object,Object} | L], Tab) ->
2675    case dets:delete_object(Tab, Object) of
2676        ok -> eval(L, Tab);
2677        Else -> {error, {delete_object,Object,Else}}
2678    end;
2679eval([{info,Tag,Expected} | L], Tab) ->
2680    case dets:info(Tab, Tag) of
2681        Expected -> eval(L, Tab);
2682        Else -> {error, {info,Tag,Else,Expected}}
2683    end;
2684eval(Else, _Tab) ->
2685    {error, {bad_request,Else}}.
2686
2687%% More than 128k keys caused crash.
2688otp_4906(Config) when is_list(Config) ->
2689    N = 256*512 + 400,
2690    Tab = otp_4906,
2691    FName = filename(Tab, Config),
2692
2693    file:delete(FName),
2694    {ok, Tab} = dets:open_file(Tab, [{file, FName}]),
2695    ok = ins_small(Tab, 0, N),
2696    ok = dets:close(Tab),
2697    {ok, Tab} = dets:open_file(Tab, [{file, FName}]),
2698    ok = read_4906(Tab, N-1),
2699    ok = dets:close(Tab),
2700    file:delete(FName),
2701
2702    %% If the (only) process fixing a table updates the table, the
2703    %% process will no longer be punished with a 1 ms delay (hm, the
2704    %% server is delayed, it should be the client...). In this example
2705    %% the writing process *is* delayed.
2706    {ok,Tab} = dets:open_file(Tab, [{file,FName}]),
2707    Parent = self(),
2708    FixPid = spawn_link(fun() ->
2709                                dets:safe_fixtable(Tab, true),
2710                                receive {Parent, stop} -> ok end
2711                        end),
2712    ok = ins_small(Tab, 0, 1000),
2713    FixPid ! {Parent, stop},
2714    timer:sleep(1),
2715    ok = dets:close(Tab),
2716    file:delete(FName),
2717    ok.
2718
2719read_4906(_T, N) when N < 0 ->
2720    ok;
2721read_4906(T, N) ->
2722    [_] = dets:lookup(T, N),
2723    read_4906(T, N-1).
2724
2725ins_small(_T, I, N) when I =:= N ->
2726    ok;
2727ins_small(T, I, N) ->
2728    ok = dets:insert(T, {I}),
2729    ins_small(T, I+1, N).
2730
2731%% Unwritable ramfile caused crash.
2732otp_5402(Config) when is_list(Config) ->
2733    Tab = otp_5402,
2734    File = filename:join(["cannot", "write", "this", "file"]),
2735
2736    %% close
2737    {ok, T} = dets:open_file(Tab, [{ram_file,true},
2738                                   {file, File}]),
2739    ok = dets:insert(T, {1,a}),
2740    {error,{file_error,_,_}} = dets:close(T),
2741
2742    %% sync
2743    {ok, T} = dets:open_file(Tab, [{ram_file,true},
2744                                   {file, File}]),
2745    ok = dets:insert(T, {1,a}),
2746    {error,{file_error,_,_}} = dets:sync(T),
2747    {error,{file_error,_,_}} = dets:close(T),
2748
2749    %% auto_save
2750    {ok, T} = dets:open_file(Tab, [{ram_file,true},
2751                                   {auto_save, 2000},
2752                                   {file, File}]),
2753    ok = dets:insert(T, {1,a}),
2754    timer:sleep(5000),
2755    {error,{file_error,_,_}} = dets:close(T),
2756    ok.
2757
2758%% Several clients open and close tables simultaneously.
2759simultaneous_open(Config) ->
2760    Tab = sim_open,
2761    File = filename(Tab, Config),
2762
2763    ok = monit(Tab, File),
2764    case feasible() of
2765        false -> {comment, "OK, but did not run all of the test"};
2766        true ->
2767            ok = kill_while_repairing(Tab, File),
2768            ok = kill_while_init(Tab, File),
2769            ok = open_ro(Tab, File),
2770            ok = open_w(Tab, File, 0, Config),
2771            ok = open_w(Tab, File, 100, Config)
2772    end.
2773
2774feasible() ->
2775    LP = erlang:system_info(logical_processors),
2776    (is_integer(LP)
2777     andalso LP >= erlang:system_info(schedulers_online)
2778     andalso not erlang:system_info(debug_compiled)
2779     andalso not erlang:system_info(lock_checking)).
2780
2781%% One process logs and another process closes the log. Before
2782%% monitors were used, this would make the client never return.
2783monit(Tab, File) ->
2784    file:delete(File),
2785    {ok, Tab} = dets:open_file(Tab, [{file,File}]),
2786    F1 = fun() -> dets:close(Tab) end,
2787    F2 = fun() -> {'EXIT', {badarg, _}} = do_log(Tab) end,
2788    spawn(F2),
2789    timer:sleep(100),
2790    spawn(F1),
2791    dets:close(Tab),
2792    ok = file:delete(File).
2793
2794do_log(Tab) ->
2795    case catch dets:insert(Tab, {hej,san,sa}) of
2796        ok -> do_log(Tab);
2797        Else -> Else
2798    end.
2799
2800%% Kill the Dets process while repair is in progress.
2801kill_while_repairing(Tab, File) ->
2802    create_opened_log(File),
2803    Delay = 1000,
2804    dets:start(),
2805    Parent = self(),
2806    F = fun() ->
2807                R = (catch dets:open_file(Tab, [{file,File}])),
2808                timer:sleep(Delay),
2809                Parent ! {self(), R}
2810        end,
2811    %% One of these will open the file, the other will be pending
2812    %% until the file has been repaired:
2813    P1 = spawn(F),
2814    P2 = spawn(F),
2815    P3 = spawn(F),
2816    DetsPid = find_dets_pid(),
2817    exit(DetsPid, kill),
2818
2819    receive {P1,R1} -> R1 end,
2820    receive {P2,R2} -> R2 end,
2821    receive {P3,R3} -> R3 end,
2822    io:format("Killed pid: ~p~n", [DetsPid]),
2823    io:format("Remaining Dets-pids (should be nil): ~p~n",
2824              [find_dets_pids()]),
2825    {replies,[{'EXIT', {dets_process_died, _}}, {ok,_}, {ok, _}]} =
2826         {replies,lists:sort([R1, R2, R3])},
2827
2828    timer:sleep(200),
2829    case dets:info(Tab) of
2830        undefined ->
2831            ok;
2832        _Info ->
2833            timer:sleep(5000),
2834            undefined = dets:info(Tab)
2835    end,
2836
2837    file:delete(File),
2838    ok.
2839
2840find_dets_pid() ->
2841    case find_dets_pids() of
2842        [] ->
2843            timer:sleep(100),
2844            find_dets_pid();
2845        [Pid] ->
2846            Pid
2847    end.
2848
2849find_dets_pids() ->
2850    lists:filter(fun(P) -> dets:pid2name(P) =/= undefined end,
2851                 erlang:processes()).
2852
2853%% Kill the Dets process when there are users and an on-going
2854%% initiailization.
2855kill_while_init(Tab, File) ->
2856    file:delete(File),
2857    Parent = self(),
2858    F = fun() ->
2859                R = dets:open_file(Tab, [{file,File}]),
2860                Parent ! {self(), R},
2861                receive {Parent, die} -> ok end,
2862                {error, not_owner} = dets:close(Tab)
2863        end,
2864    P1 = spawn(F),
2865    P2 = spawn(F),
2866    P3 = spawn(F),
2867    IF = fun() ->
2868                 R = dets:open_file(Tab, [{file,File}]),
2869                 Parent ! {self(), R},
2870                 Fun = fun(_) -> timer:sleep(100000) end,
2871                 {'EXIT', {badarg, _}} = (catch dets:init_table(Tab, Fun)),
2872                 receive {Parent, die} -> ok end
2873          end,
2874    P4 = spawn(IF),
2875    receive {P1,R1} -> {ok, _} = R1 end,
2876    receive {P2,R2} -> {ok, _} = R2 end,
2877    receive {P3,R3} -> {ok, _} = R3 end,
2878    receive {P4,R4} -> {ok, _} = R4 end,
2879    DetsPid = find_dets_pid(),
2880    exit(DetsPid, kill),
2881
2882    timer:sleep(1000),
2883    undefined = dets:info(Tab),
2884    P1 ! {Parent, die},
2885    P2 ! {Parent, die},
2886    P3 ! {Parent, die},
2887    P4 ! {Parent, die},
2888
2889    file:delete(File),
2890    timer:sleep(100),
2891    ok.
2892
2893open_ro(Tab, File) ->
2894    create_opened_log(File),
2895    Delay = 1000,
2896    Parent = self(),
2897    F = fun() ->
2898                R = dets:open_file(Tab, [{file,File},{access,read}]),
2899                timer:sleep(Delay),
2900                Parent ! {self(), R}
2901        end,
2902    P1 = spawn(F),
2903    P2 = spawn(F),
2904    P3 = spawn(F),
2905
2906    receive {P1,R1} -> {error,{not_closed,_}} = R1 end,
2907    receive {P2,R2} -> {error,{not_closed,_}} = R2 end,
2908    receive {P3,R3} -> {error,{not_closed,_}} = R3 end,
2909    ok.
2910
2911open_w(Tab, File, Delay, Config) ->
2912    create_opened_log(File),
2913
2914    Tab2 = t2,
2915    File2 = filename(Tab2, Config),
2916    file:delete(File2),
2917    {ok,Tab2} = dets:open_file(Tab2, [{file,File2}]),
2918    ok = dets:close(Tab2),
2919
2920    Parent = self(),
2921    F = fun() ->
2922                R = dets:open_file(Tab, [{file,File}]),
2923                timer:sleep(Delay),
2924                Parent ! {self(), R}
2925        end,
2926    Pid1 = spawn(F),
2927    Pid2 = spawn(F),
2928    Pid3 = spawn(F),
2929
2930    ok = wait_for_repair_to_start(Tab),
2931
2932    %% It is assumed that it takes some time to repair the file.
2933    {ok,Tab2} = dets:open_file(Tab2, [{file,File2}]),
2934    %% The Dets server managed to handle to open_file request.
2935    0 = qlen(), % still repairing
2936
2937    ok = dets:close(Tab2),
2938    file:delete(File2),
2939
2940    receive {Pid1,R1} -> {ok, Tab} = R1 end,
2941    receive {Pid2,R2} -> {ok, Tab} = R2 end,
2942    receive {Pid3,R3} -> {ok, Tab} = R3 end,
2943    timer:sleep(200),
2944    case dets:info(Tab) of
2945        undefined ->
2946            ok;
2947        _Info ->
2948            timer:sleep(5000),
2949            undefined = dets:info(Tab)
2950    end,
2951
2952    file:delete(File),
2953    ok.
2954
2955wait_for_repair_to_start(Tab) ->
2956    case catch dets_server:get_pid(Tab) of
2957        {'EXIT', _} ->
2958            timer:sleep(1),
2959            wait_for_repair_to_start(Tab);
2960        Pid when is_pid(Pid) ->
2961            ok
2962    end.
2963
2964qlen() ->
2965    {_, {_, N}} = lists:keysearch(message_queue_len, 1, process_info(self())),
2966    N.
2967
2968create_opened_log(File) ->
2969    Tab = t,
2970    file:delete(File),
2971    {ok, Tab} = dets:open_file(Tab, [{file,File}]),
2972    ok = ins(Tab, 60000),
2973    ok = dets:close(Tab),
2974    crash(File, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED),
2975    ok.
2976
2977%% OTP-5075. insert_new/2
2978insert_new(Config) ->
2979    Tab = insert_new,
2980    File = filename(Tab, Config),
2981    file:delete(File),
2982    {ok, T} = dets:open_file(Tab, [{file,File}]),
2983    {'EXIT', {badarg, _}} = (catch dets:insert_new(Tab, 14)),
2984    {'EXIT', {badarg, _}} = (catch dets:insert_new(Tab, {})),
2985    true = dets:insert_new(Tab, {1,a}),
2986    false = dets:insert_new(Tab, {1,a}),
2987    true = dets:insert_new(Tab, [{2,b}, {3,c}]),
2988    false = dets:insert_new(Tab, [{2,b}, {3,c}]),
2989    false = dets:insert_new(Tab, [{1,a}, {4,d}]),
2990    ok = dets:close(Tab),
2991
2992    file:delete(File),
2993    {ok, T} = dets:open_file(Tab, [{file,File},{type,bag}]),
2994    true = dets:insert_new(Tab, {1,a}),
2995    false = dets:insert_new(Tab, {1,b}),
2996    true = dets:insert_new(Tab, [{2,b}, {3,c}]),
2997    false = dets:insert_new(Tab, [{2,a}, {3,d}]),
2998    false = dets:insert_new(Tab, [{1,a}, {4,d}]),
2999    ok = dets:close(Tab),
3000
3001
3002    file:delete(File),
3003    ok.
3004
3005%% OTP-5126. repair_continuation/2
3006repair_continuation(Config) ->
3007    Tab = repair_continuation_table,
3008    Fname = filename(repair_cont, Config),
3009    file:delete(Fname),
3010    {ok, _} = dets:open_file(Tab, [{file,Fname}]),
3011    ok = dets:insert(Tab, [{1,a},{2,b},{3,c}]),
3012
3013    MS = [{'_',[],[true]}],
3014
3015    SRes = term_to_binary(dets:select(Tab, MS, 1)),
3016    %% Get rid of compiled match spec
3017    lists:foreach(fun (P) ->
3018                          garbage_collect(P)
3019                  end, processes()),
3020    {[true], C2} = binary_to_term(SRes),
3021
3022    {'EXIT', {badarg, _}} = (catch dets:select(C2)),
3023    C3 = dets:repair_continuation(C2, MS),
3024    {[true], C4} = dets:select(C3),
3025    C5 = dets:repair_continuation(C4, MS),
3026    {[true], _} = dets:select(C5),
3027    {'EXIT', {badarg, _}} = (catch dets:repair_continuation(Tab, bu)),
3028
3029    ok = dets:close(Tab),
3030    file:delete(Fname),
3031    ok.
3032
3033%% OTP-5487. Growth of read-only table (again).
3034otp_5487(Config) ->
3035    Tab = otp_5487,
3036    Fname = filename(otp_5487, Config),
3037    file:delete(Fname),
3038    Ets = ets:new(otp_5487, [public, set]),
3039    lists:foreach(fun(I) -> ets:insert(Ets, {I,I+1}) end,
3040                  lists:seq(0,1000)),
3041    {ok, _} = dets:open_file(Tab, [{file,Fname}]),
3042    ok = dets:from_ets(Tab, Ets),
3043    ok = dets:sync(Tab),
3044    ok = dets:close(Tab),
3045    {ok, _} = dets:open_file(Tab, [{file,Fname},{access,read}]),
3046    [{1,2}] = dets:lookup(Tab, 1),
3047    ok = dets:close(Tab),
3048    ets:delete(Ets),
3049    file:delete(Fname).
3050
3051%% OTP-6206. Badly formed free lists.
3052otp_6206(Config) ->
3053    Tab = otp_6206,
3054    File = filename(Tab, Config),
3055
3056    file:delete(File),
3057    Options = [{file,File}],
3058    {ok, Tab} = dets:open_file(Tab, Options),
3059    NObjs = 13006,
3060    ok = ins(Tab, NObjs),
3061    ok = del(Tab, NObjs, 2),
3062    ok = dets:close(Tab),
3063
3064    %% Used to return {badmatch,{error,{bad_freelists,File}}.
3065    {ok, Tab} = dets:open_file(Tab, [{repair,false}|Options]),
3066    ok = dets:close(Tab),
3067    file:delete(File),
3068    ok.
3069
3070%% OTP-6359. select and match never return the empty list.
3071otp_6359(Config) ->
3072    Tab = otp_6359,
3073    File = filename(Tab, Config),
3074
3075    file:delete(File),
3076    {ok, _} = dets:open_file(Tab, [{file, File}]),
3077    %% Used to return {[], Cont}:
3078    '$end_of_table' = dets:match(Tab, '_', 100),
3079    ok = dets:close(Tab),
3080    file:delete(File),
3081    ok.
3082
3083%% OTP-4738. ==/2 and =:=/2.
3084otp_4738(Config) ->
3085    otp_4738_set(Config),
3086    otp_4738_bag(Config),
3087    otp_4738_dupbag(Config),
3088    ok.
3089
3090otp_4738_dupbag(Config) ->
3091    Tab = otp_4738,
3092    File = filename(Tab, Config),
3093    file:delete(File),
3094    I = -12857447,
3095    F = float(I),
3096    One = 1,
3097    FOne = float(One),
3098    Args = [{file,File},{type,duplicate_bag}],
3099    {ok, Tab} = dets:open_file(Tab, Args),
3100    ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]),
3101    ok = dets:sync(Tab),
3102    [{F,One},{F,FOne}] = dets:lookup(Tab, F),
3103    [{I,One},{I,FOne}] = dets:lookup(Tab, I),
3104    ok = dets:insert(Tab, [{F,One},{F,FOne}]),
3105    [{I,One},{I,FOne},{F,One},{F,FOne},{F,One},{F,FOne}] =
3106        dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3107    ok = dets:insert(Tab, [{F,FOne},{F,One}]),
3108    [{I,One},{I,FOne},{F,One},{F,FOne},{F,One},
3109           {F,FOne},{F,FOne},{F,One}] =
3110        dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3111    ok = dets:delete_object(Tab, {I,FOne}),
3112    [{I,One},{F,One},{F,FOne},{F,One},{F,FOne},{F,FOne},{F,One}] =
3113        dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3114    ok = dets:insert(Tab, {I,FOne}),
3115    [{I,One},{I,FOne},{F,One},{F,FOne},{F,One},
3116           {F,FOne},{F,FOne},{F,One}] =
3117        dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3118    ok = dets:delete_object(Tab, {F,FOne}),
3119    [{I,One},{I,FOne},{F,One},{F,One},{F,One}] =
3120        dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3121    ok = dets:delete(Tab, F),
3122    [{I,One},{I,FOne}] = dets:match_object(Tab, '_'),
3123    ok = dets:close(Tab),
3124    file:delete(File),
3125
3126    Zero = 0,
3127    FZero = float(Zero),
3128    {ok, Tab} = dets:open_file(Tab, Args),
3129    ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]),
3130    ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]),
3131    ok = dets:insert(Tab, [{I,Zero},{F,Zero},{I,FZero},{I,FZero}]),
3132    Objs0 = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3133    ok = dets:close(Tab),
3134    crash(File, ?CLOSED_PROPERLY_POS+3, ?NOT_PROPERLY_CLOSED),
3135    io:format("Expect repair:~n"),
3136    {ok, Tab} = dets:open_file(Tab, Args),
3137    Objs1 = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3138    ok = dets:close(Tab),
3139    Objs1 = Objs0,
3140    file:delete(File),
3141    ok.
3142
3143otp_4738_bag(Config) ->
3144    Tab = otp_4738,
3145    File = filename(Tab, Config),
3146    file:delete(File),
3147    I = -12857447,
3148    F = float(I),
3149    One = 1,
3150    FOne = float(One),
3151    Args = [{file,File},{type,bag}],
3152    {ok, Tab} = dets:open_file(Tab, Args),
3153    ok = dets:insert(Tab, [{I,One},{F,One},{I,FOne},{F,FOne}]),
3154    ok = dets:sync(Tab),
3155    [{F,One},{F,FOne}] = dets:lookup(Tab, F),
3156    [{I,One},{I,FOne}] = dets:lookup(Tab, I),
3157    ok = dets:insert(Tab, [{F,One},{F,FOne}]),
3158    [{I,One},{I,FOne},{F,One},{F,FOne}] =
3159        dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3160    ok = dets:insert(Tab, [{F,FOne},{F,One}]),
3161    [{I,One},{I,FOne},{F,FOne},{F,One}] =
3162        dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3163    ok = dets:delete_object(Tab, {I,FOne}),
3164    [{I,One},{F,FOne},{F,One}] =
3165        dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3166    ok = dets:insert(Tab, {I,FOne}),
3167    [{I,One},{I,FOne},{F,FOne},{F,One}] =
3168        dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3169    ok = dets:delete(Tab, F),
3170    [{I,One},{I,FOne}] = dets:match_object(Tab, '_'),
3171    ok = dets:close(Tab),
3172    file:delete(File).
3173
3174otp_4738_set(Config) ->
3175    Tab = otp_4738,
3176    File = filename(Tab, Config),
3177    file:delete(File),
3178    Args = [{file,File},{type,set}],
3179
3180    %% I and F share the same slot.
3181    I = -12857447,
3182    F = float(I),
3183    {ok, Tab} = dets:open_file(Tab, Args),
3184    ok = dets:insert(Tab, [{I},{F}]),
3185    ok = dets:sync(Tab),
3186    [{F}] = dets:lookup(Tab, F),
3187    [{I}] = dets:lookup(Tab, I),
3188    ok = dets:insert(Tab, [{F}]),
3189    [{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3190    ok = dets:close(Tab),
3191    file:delete(File),
3192
3193    {ok, Tab} = dets:open_file(Tab, Args),
3194    ok = dets:insert(Tab, [{I}]),
3195    ok = dets:sync(Tab),
3196    [] = dets:lookup(Tab, F),
3197    [{I}] = dets:lookup(Tab, I),
3198    ok = dets:insert(Tab, [{F}]),
3199    [{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3200    ok = dets:close(Tab),
3201    file:delete(File),
3202
3203    {ok, Tab} = dets:open_file(Tab, Args),
3204    ok = dets:insert(Tab, [{I},{F}]),
3205    %% {insert, ...} in the cache, try lookup:
3206    [{F}] = dets:lookup(Tab, F),
3207    [{I}] = dets:lookup(Tab, I),
3208    %% Both were found, but that cannot be verified.
3209    [{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3210    ok = dets:close(Tab),
3211    file:delete(File),
3212
3213    {ok, Tab} = dets:open_file(Tab, Args),
3214    ok = dets:insert(Tab, [{I}]),
3215    ok = dets:sync(Tab),
3216    ok = dets:insert(Tab, [{F}]),
3217    %% {insert, ...} in the cache, try lookup:
3218    [{F}] = dets:lookup(Tab, F),
3219    [{I}] = dets:lookup(Tab, I),
3220    [{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3221    ok = dets:close(Tab),
3222    file:delete(File),
3223
3224    {ok, Tab} = dets:open_file(Tab, Args),
3225    %% Both operations in the cache:
3226    ok = dets:insert(Tab, [{I}]),
3227    ok = dets:insert(Tab, [{F}]),
3228    [{I},{F}] = dets_utils:mkeysort(1, dets:match_object(Tab, '_')),
3229    ok = dets:close(Tab),
3230    file:delete(File),
3231    ok.
3232
3233%% OTP-7146. Bugfix: missing test when re-hashing.
3234otp_7146(Config) ->
3235    Tab = otp_7146,
3236    File = filename(Tab, Config),
3237    file:delete(File),
3238
3239    Max = 2048,
3240    {ok, Tab} = dets:open_file(Tab, [{max_no_slots,Max}, {file,File}]),
3241    write_dets(Tab, Max),
3242    ok = dets:close(Tab),
3243
3244    file:delete(File),
3245    ok.
3246
3247write_dets(Tab, Max) ->
3248    write_dets(Tab, 0, Max).
3249
3250write_dets(_Tab, N, Max) when N > Max ->
3251    ok;
3252write_dets(Tab, N, Max) ->
3253    ok = dets:insert(Tab,{ N, {entry,N}}),
3254    write_dets(Tab, N+1, Max).
3255
3256%% OTP-8070. Duplicated objects with insert_new() and duplicate_bag.
3257otp_8070(Config) when is_list(Config) ->
3258    Tab = otp_8070,
3259    File = filename(Tab, Config),
3260    file:delete(File),
3261    {ok, _} = dets:open_file(Tab, [{file,File},{type, duplicate_bag}]),
3262    ok = dets:insert(Tab, [{3,0}]),
3263    false = dets:insert_new(Tab, [{3,1},{3,1}]),
3264    [{3,0}] = dets:lookup(Tab, 3),
3265    ok = dets:close(Tab),
3266    file:delete(File),
3267    ok.
3268
3269%% OTP-8856. insert_new() bug.
3270otp_8856(Config) when is_list(Config) ->
3271    Tab = otp_8856,
3272    File = filename(Tab, Config),
3273    file:delete(File),
3274    Me = self(),
3275    {ok, _} = dets:open_file(Tab, [{type, bag}, {file, File}]),
3276    spawn(fun()-> Me ! {1, dets:insert(Tab, [])} end),
3277    spawn(fun()-> Me ! {2, dets:insert_new(Tab, [])} end),
3278    receive {1, ok} -> ok end,
3279    receive {2, true} -> ok end,
3280    ok = dets:close(Tab),
3281    file:delete(File),
3282
3283    {ok, _} = dets:open_file(Tab, [{type, set}, {file, File}]),
3284    spawn(fun() -> dets:delete(Tab, 0) end),
3285    spawn(fun() -> Me ! {3, dets:insert_new(Tab, {0,0})} end),
3286    receive {3, true} -> ok end,
3287    ok = dets:close(Tab),
3288    file:delete(File),
3289    ok.
3290
3291%% OTP-8903. bchunk/match/select bug.
3292otp_8903(Config) when is_list(Config) ->
3293    Tab = otp_8903,
3294    File = filename(Tab, Config),
3295    {ok,T} = dets:open_file(bug, [{file,File}]),
3296    ok = dets:insert(T, [{1,a},{2,b},{3,c}]),
3297    dets:safe_fixtable(T, true),
3298    {[_],C1} = dets:match_object(T, '_', 1),
3299    {BC1,_D} = dets:bchunk(T, start),
3300    ok = dets:close(T),
3301    {'EXIT', {badarg, _}} = (catch {foo,dets:match_object(C1)}),
3302    {'EXIT', {badarg, _}} = (catch {foo,dets:bchunk(T, BC1)}),
3303    {ok,T} = dets:open_file(bug, [{file,File}]),
3304    false = dets:info(T, safe_fixed),
3305    {'EXIT', {badarg, _}} = (catch {foo,dets:match_object(C1)}),
3306    {'EXIT', {badarg, _}} = (catch {foo,dets:bchunk(T, BC1)}),
3307    ok = dets:close(T),
3308    file:delete(File),
3309    ok.
3310
3311%% OTP-8923. rehash due to lookup after initialization.
3312otp_8923(Config) when is_list(Config) ->
3313    Tab = otp_8923,
3314    File = filename(Tab, Config),
3315    %% Create a file with more than 256 keys:
3316    file:delete(File),
3317    Bin = list_to_binary([ 0 || _ <- lists:seq(1, 400) ]),
3318    BigBin = list_to_binary([ 0 ||_ <- lists:seq(1, 4000)]),
3319    Ets = ets:new(temp, [{keypos,1}]),
3320    [ true = ets:insert(Ets, {C,Bin}) || C <- lists:seq(1, 700) ],
3321    true = ets:insert(Ets, {helper_data,BigBin}),
3322    true = ets:insert(Ets, {prim_btree,BigBin}),
3323    true = ets:insert(Ets, {sec_btree,BigBin}),
3324    %% Note: too few slots; re-hash will take place
3325    {ok, Tab} = dets:open_file(Tab, [{file,File}]),
3326    Tab = ets:to_dets(Ets, Tab),
3327    ok = dets:close(Tab),
3328    true = ets:delete(Ets),
3329
3330    {ok,Ref} = dets:open_file(File),
3331    [{1,_}] = dets:lookup(Ref, 1),
3332    ok = dets:close(Ref),
3333
3334    {ok,Ref2} = dets:open_file(File),
3335    [{helper_data,_}] = dets:lookup(Ref2, helper_data),
3336    ok = dets:close(Ref2),
3337
3338    file:delete(File),
3339    ok.
3340
3341%% OTP-9282. The name of a table can be an arbitrary term.
3342otp_9282(Config) when is_list(Config) ->
3343    some_calls(make_ref(), Config),
3344    some_calls({a,typical,name}, Config),
3345    some_calls(fun() -> a_funny_name end, Config),
3346    ok.
3347
3348some_calls(Tab, Config) ->
3349    File = filename(ref, Config),
3350    {ok,T} = dets:open_file(Tab, [{file,File}]),
3351    T = Tab,
3352    false = dets:info(T, safe_fixed),
3353    File = dets:info(T, filename),
3354    ok = dets:insert(Tab, [{3,0}]),
3355    [{3,0}] = dets:lookup(Tab, 3),
3356    [{3,0}] = dets:traverse(Tab, fun(X) -> {continue, X} end),
3357    ok = dets:close(T),
3358    file:delete(File).
3359
3360
3361%% OTP-11245. Tables remained fixed after traversal.
3362otp_11245(Config) when is_list(Config) ->
3363    Tab = otp_11245,
3364    File = filename(Tab, Config),
3365    {ok, Tab} = dets:open_file(Tab, [{file,File}]),
3366    N = 1024,
3367    ins(Tab, N),
3368    N = length(dets:match(Tab, '_')),
3369    false = dets:info(Tab, safe_fixed),
3370    dets:traverse(Tab, fun(_) -> continue end),
3371    false = dets:info(Tab, safe_fixed),
3372    N = dets:foldl(fun(_, N2) -> N2+1 end, 0, Tab),
3373    false = dets:info(Tab, safe_fixed),
3374    N = dets:foldr(fun(_, N2) -> N2+1 end, 0, Tab),
3375    false = dets:info(Tab, safe_fixed),
3376    ok = dets:close(Tab),
3377    file:delete(File),
3378    ok.
3379
3380%% OTP-11709. Bugfixes.
3381otp_11709(Config) when is_list(Config) ->
3382    Short = <<"foo">>,
3383    Long = <<"a sufficiently long text">>,
3384
3385    %% Bug: leaking file descriptor
3386    P0 = pps(),
3387    File = filename(otp_11709, Config),
3388    ok = file:write_file(File, Long),
3389    false = dets:is_dets_file(File),
3390    check_pps(P0),
3391
3392    %% Bug: deleting file
3393    Args = [[{access, A}, {repair, R}] ||
3394               A <- [read, read_write],
3395               R <- [true, false, force]],
3396    Fun1 = fun(S, As) ->
3397                   P1 = pps(),
3398                   ok = file:write_file(File, S),
3399                   {error,{not_a_dets_file,File}} = dets:open_file(File, As),
3400                   {ok, S} = file:read_file(File),
3401                   check_pps(P1)
3402           end,
3403    Fun2 = fun(S) ->
3404                   _ = [Fun1(S, As) || As <- Args],
3405                   ok
3406           end,
3407    ok = Fun2(Long),  % no change here
3408    ok = Fun2(Short), % mimic the behaviour for longer files
3409
3410    %% open_file/1
3411    ok = file:write_file(File, Long),
3412    {error,{not_a_dets_file,File}} = dets:open_file(File), % no change
3413    ok = file:write_file(File, Short),
3414    {error,{not_a_dets_file,File}} = dets:open_file(File), % mimic
3415
3416    _ = file:delete(File),
3417    ok.
3418
3419%% OTP-13229. open_file() exits with badarg when given binary file name.
3420%% Also OTP-15253.
3421otp_13229(_Config) ->
3422    F = <<"binfile.tab">>,
3423    try dets:open_file(name, [{file, F}]) of
3424        R ->
3425            exit({open_succeeded, R})
3426    catch
3427        error:badarg ->
3428            ok
3429    end,
3430    try dets:open_file(F, []) of % OTP-15253
3431        R2 ->
3432            exit({open_succeeded, R2})
3433    catch
3434        error:badarg ->
3435            ok
3436    end,
3437    try dets:open_file(F) of
3438        R3 ->
3439            exit({open_succeeded, R3})
3440    catch
3441        error:badarg ->
3442            ok
3443    end.
3444
3445%% OTP-13260. Race when opening a table.
3446otp_13260(Config) ->
3447    [ok] = lists:usort([otp_13260_1(Config) || _ <- lists:seq(1, 3)]),
3448    ok.
3449
3450otp_13260_1(Config) ->
3451    Tab = otp_13260,
3452    File = filename(Tab, Config),
3453    N = 20,
3454    P = self(),
3455    Pids = [spawn_link(fun() -> counter(P, Tab, File) end) ||
3456               _ <- lists:seq(1, N)],
3457    Rs = rec(Pids),
3458    true = lists:all(fun(R) -> is_integer(R) end, Rs),
3459    wait_for_close(Tab).
3460
3461rec([]) ->
3462    [];
3463rec([Pid | Pids]) ->
3464    receive {Pid, R} ->
3465            [R | rec(Pids)]
3466    end.
3467
3468%% One may have to run the test several times to trigger the bug.
3469counter(P, Tab, File) ->
3470    Key = key,
3471    N = case catch dets:update_counter(Tab, Key, 1) of
3472            {'EXIT', _} ->
3473                {ok, Tab} = dets:open_file(Tab, [{file, File}]),
3474                ok = dets:insert(Tab, {Key, 1}),
3475                dets:update_counter(Tab, Key, 1);
3476            N1 when is_integer(N1) ->
3477                N1;
3478            DetsBug ->
3479                DetsBug
3480        end,
3481    P ! {self(), N}.
3482
3483wait_for_close(Tab) ->
3484    case dets:info(Tab, owner) of
3485        undefined ->
3486            ok;
3487        _ ->
3488            timer:sleep(100),
3489            wait_for_close(Tab)
3490    end.
3491
3492%% OTP-13830. Format 8 is no longer supported.
3493otp_13830(Config) ->
3494    Tab = otp_13830,
3495    File8 = filename:join(?datadir(Config), "version_8.dets"),
3496    {error,{format_8_no_longer_supported,_}} =
3497        dets:open_file(Tab, [{file, File8}]),
3498    File = filename(Tab, Config),
3499    %% Check the 'version' option, for backwards compatibility:
3500    {ok, Tab} = dets:open_file(Tab, [{file, File}, {version, 9}]),
3501    ok = dets:close(Tab),
3502    {ok, Tab} = dets:open_file(Tab, [{file, File}, {version, default}]),
3503    ok = dets:close(Tab).
3504
3505%%
3506%% Parts common to several test cases
3507%%
3508
3509crash(File, Where) ->
3510    crash(File, Where, 10).
3511
3512crash(File, Where, What) when is_integer(What) ->
3513    {ok, Fd} = file:open(File, [read,write]),
3514    file:position(Fd, Where),
3515    ok = file:write(Fd, [What]),
3516    ok = file:close(Fd).
3517
3518args(Config) ->
3519    {Sets, Bags, Dups} =
3520	{[
3521	  [],
3522	  [{type, set}, {estimated_no_objects, 300},
3523	   {ram_file, true}],
3524	  [{type, set}, {estimated_no_objects, 300}],
3525	  [{type, set}, {estimated_no_objects, 300}],
3526	  [{auto_save,20}, {type, set},
3527	   {estimated_no_objects, 300}]
3528	 ],
3529
3530	 [
3531	  [{type, bag}, {estimated_no_objects, 300}, {ram_file, true}],
3532	  [{type, bag}],
3533	  [{type, bag}, {estimated_no_objects, 300}],
3534	  [{type, bag}, {estimated_no_objects, 300}],
3535	  [{type, bag},
3536	   {auto_save,20}, {estimated_no_objects, 300}],
3537	  [{type, bag}, {estimated_no_objects, 300},   {ram_file, true}]
3538	 ],
3539
3540	 [
3541	  [{type, duplicate_bag}, {estimated_no_objects, 300},
3542	   {ram_file, true}],
3543	  [{type, duplicate_bag}],
3544	  [{type, duplicate_bag}, {estimated_no_objects, 300}],
3545	  [{type, duplicate_bag}, {estimated_no_objects, 300}],
3546	  [{type, duplicate_bag},
3547	   {auto_save,20}, {estimated_no_objects, 300}],
3548	  [{type, duplicate_bag}, {estimated_no_objects, 300},
3549	   {ram_file, true}]
3550	 ]
3551	},
3552    zip_filename(Sets, Bags, Dups, Config).
3553
3554zip_filename(S, B, D, Conf) ->
3555    zip_filename(S, B, D, [], [], [], 1, Conf).
3556
3557zip_filename([H|T], B, D, S1, B1, D1, I, Conf) ->
3558    zip_filename(T, B, D, [[{file, new_filename(I, Conf)} | H] | S1],
3559		 B1, D1, I+1, Conf);
3560zip_filename([], [H|B], D, S1, B1, D1, I, Conf) ->
3561    zip_filename([], B, D, S1, [[{file, new_filename(I, Conf)} | H] | B1],
3562		 D1, I+1, Conf);
3563zip_filename([], [], [H|T], S1, B1, D1, I, Conf) ->
3564    zip_filename([], [], T, S1, B1, [[{file, new_filename(I, Conf)} | H] | D1],
3565		 I+1, Conf);
3566zip_filename([], [], [], S1, B1, D1, _, _Conf) ->
3567    {reverse(S1), reverse(B1), reverse(D1)}.
3568
3569del_test(Tab) ->
3570    ?format("Deltest on ~p~n", [Tab]),
3571    Objs = safe_get_all_objects(Tab),
3572    Keys = map(fun(X) -> element(1, X) end, Objs),
3573    foreach(fun(Key) -> dets:delete(Tab, Key) end, Keys),
3574    0 = length(get_all_objects(Tab)),
3575    [] = get_all_objects_fast(Tab),
3576    0 = dets:info(Tab, size).
3577
3578del_obj_test(Tab) ->
3579    ?format("Delobjtest on ~p~n", [Tab]),
3580    Objs = safe_get_all_objects(Tab),
3581    LL = length(Objs),
3582    LL = dets:info(Tab, size),
3583    foreach(fun(Obj) -> dets:delete_object(Tab, Obj) end, Objs),
3584    0 = length(get_all_objects(Tab)),
3585    [] = get_all_objects_fast(Tab),
3586    0 = dets:info(Tab, size).
3587
3588match_del_test(Tab) ->
3589    ?format("Match delete test on ~p~n", [Tab]),
3590    ok = dets:match_delete(Tab, {'_','_','_'}),
3591    Sz = dets:info(Tab, size),
3592    true = Sz =:= length(dets:match_object(Tab, '_')),
3593    ok = dets:match_delete(Tab, '_'),
3594    0 = dets:info(Tab, size),
3595    0 = length(get_all_objects(Tab)),
3596    [] = get_all_objects_fast(Tab).
3597
3598trav_test(_Data, Len, Tab) ->
3599    ?format("Travtest on ~p~n", [Tab]),
3600    _X0 = dets:traverse(Tab, fun(_X) -> continue end),
3601    XX = dets:traverse(Tab, fun(X) -> {continue, X} end),
3602    case Len =:= length(XX) of
3603	      false -> ?format("DIFF ~p~n", [XX -- _Data]);
3604	      true -> ok
3605	  end,
3606    1 = length(dets:traverse(Tab, fun(X) -> {done, X} end)).
3607
3608match_test(Data, Tab) ->
3609    ?format("Match test on ~p~n", [Tab]),
3610    Data1 = sort(filter(fun(X) when tuple_size(X) =:= 3 -> true;
3611                           (_X) -> false
3612                        end, Data)),
3613    Data1 = sort(dets:match_object(Tab, {'$1', '$2', '$3'})),
3614
3615    Len = length(Data),
3616    Len = length(dets:match(Tab, '_')),
3617    Len2 = length(Data1),
3618    Len2 = length(dets:match(Tab, {'$1', '_', '_'})),
3619
3620    Data3 =
3621	filter(fun(X) ->
3622		       K = element(1, X),
3623		       if
3624			   tuple_size(X) =:= 3, tuple_size(K) =:= 2 -> true;
3625			   true -> false
3626		       end
3627	       end, Data),
3628    Len3 = length(Data3),
3629    Len3 = length(dets:match(Tab, {{'$1', '$2'}, '_', '_'})),
3630    Len3 = length(dets:match_object(Tab, {{'$1', '$2'}, '_', '_'})),
3631
3632    R = make_ref(),
3633    dets:insert(Tab, {{R, R}, 33 ,44}),
3634    1 = length(dets:match(Tab, {{R, R}, '_', '_'})),
3635    1 = length(dets:match_object(Tab, {{R, R}, '_', '_'})).
3636
3637%%
3638%% Utilities
3639%%
3640
3641headsz() ->
3642    ?HEADSZ_v9.
3643
3644unwritable(Fname) ->
3645    {ok, Info} = file:read_file_info(Fname),
3646    Mode = Info#file_info.mode - 8#00200,
3647    file:write_file_info(Fname, Info#file_info{mode = Mode}).
3648
3649writable(Fname) ->
3650    {ok, Info} = file:read_file_info(Fname),
3651    Mode = Info#file_info.mode bor 8#00200,
3652    file:write_file_info(Fname, Info#file_info{mode = Mode}).
3653
3654truncate(File, Where) ->
3655    {ok, Fd} = file:open(File, [read,write]),
3656    file:position(Fd, Where),
3657    ok = file:truncate(Fd),
3658    ok = file:close(Fd).
3659
3660new_filename(Name, _Config) when is_integer(Name) ->
3661    filename:join(?privdir(_Config),
3662		  integer_to_list(Name) ++ ".DETS").
3663
3664filename(Name, Config) when is_atom(Name) ->
3665    filename(atom_to_list(Name), Config);
3666filename(Name, _Config) ->
3667    filename:join(?privdir(_Config), Name).
3668
3669open_files(_Name, []) ->
3670    [];
3671open_files(Name0, [Args | Tail]) ->
3672    ?format("init ~p~n", [Args]),
3673    Name = list_to_atom(integer_to_list(Name0)),
3674    {ok, Name} = dets:open_file(Name, Args),
3675    [Name | open_files(Name0+1, Tail)].
3676
3677close_all(Tabs) -> foreach(fun(Tab) -> ok = dets:close(Tab) end, Tabs).
3678
3679delete_files(Args) ->
3680    Fun = fun(F) ->
3681		  {value, {file, File}} = keysearch(file, 1, F),
3682		  file:delete(File),
3683		  File
3684	  end,
3685    map(Fun, Args).
3686
3687%% Initialize all tables
3688initialize(Tabs, Data) ->
3689    foreach(fun(Tab) ->
3690                    Fun = fun(Obj) -> ok =  dets:insert(Tab, Obj) end,
3691                    foreach(Fun, Data),
3692                    dets:sync(Tab)
3693            end, Tabs).
3694
3695%% need more than 512 objects to really trig overflow
3696make_data(Kp) ->
3697    make_data(Kp, set).
3698
3699make_data(Kp, Type) ->
3700    dup(Type, make_data(Kp, Type, 520)).
3701
3702dup(duplicate_bag, [H1, H2 |T]) ->
3703    [H1,H2, H1, H2 | dup(duplicate_bag, T)];
3704dup(_, Other) ->
3705    Other.
3706
3707make_data(_Kp, Type, 0) ->
3708    odd_keys(Type);
3709make_data(1, set, I) ->
3710    [{I, q,w} | make_data(1, set, I-1)];
3711make_data(2, set, I) ->
3712    [{hh, I, q,w} | make_data(2, set, I-1)];
3713make_data(1, bag, I) ->
3714    [{I, q,w} , {I, hah, 77} | make_data(1, bag, I-1)];
3715make_data(2, bag, I) ->
3716    [{hh, I, q,w} , {hh, I, lalal, 900} | make_data(2, bag, I-1)];
3717make_data(1, duplicate_bag, I) ->
3718    [{I, q,w} , {I, hah, 77} | make_data(1, duplicate_bag, I-1)];
3719make_data(2, duplicate_bag, I) ->
3720    [{hh, I, q,w} , {hh, I, lalal, 900} | make_data(2, duplicate_bag, I-1)].
3721
3722odd_keys(_) ->
3723    [{foo, 1 ,2},
3724     {{foo, foo}, 2,3},
3725     {"kakaka", {{{}}}, jj},
3726     {{"kallll", "kkk", []}, 66.7777},
3727     {make_ref(), 99, 66},
3728     {{1},2,3,4,5,6,7,duplicate(50, 8)},
3729     {self(), 7,8,88},
3730     {[self()], 8, 11}].
3731
3732
3733ins(_T, 0) ->
3734    ok;
3735ins(T, N) ->
3736    case dets:insert(T, {N, item(N)}) of
3737	ok -> ins(T, N-1);
3738	Error -> Error
3739    end.
3740
3741item(N) when N rem 2 =:= 0 ->
3742    {item, number, N};
3743item(N) ->
3744    {item, number, N, a, much, bigger, one, i, think}.
3745
3746del(_T, N, _I) when N =< 0 ->
3747    ok;
3748del(T, N, I) ->
3749    ok = dets:delete(T, N),
3750    del(T, N-I, I).
3751
3752ensure_node(0, _Node) ->
3753    could_not_start_node;
3754ensure_node(N, Node) ->
3755    case net_adm:ping(Node) of
3756	pang ->
3757	    receive after 1000 ->
3758			     ok
3759		    end,
3760	    ensure_node(N-1,Node);
3761	pong ->
3762	    ok
3763    end.
3764
3765size_test(Len, Tabs) ->
3766    foreach(fun(Tab) ->
3767                    Len = dets:info(Tab, size)
3768            end, Tabs).
3769
3770no_keys_test([T | Ts]) ->
3771    no_keys_test(T),
3772    no_keys_test(Ts);
3773no_keys_test([]) ->
3774    ok;
3775no_keys_test(T) ->
3776    Kp = dets:info(T, keypos),
3777    All = dets:match_object(T, '_'),
3778    L = lists:map(fun(X) -> element(Kp, X) end, All),
3779    NoKeys = length(lists:usort(L)),
3780    case {dets:info(T, no_keys), NoKeys} of
3781        {N, N} ->
3782            ok;
3783        {N1, N2} ->
3784            exit({no_keys_test, N1, N2})
3785    end.
3786
3787safe_get_all_objects(Tab) ->
3788    dets:safe_fixtable(Tab, true),
3789    Objects = get_all_objects(Tab),
3790    dets:safe_fixtable(Tab, false),
3791    Objects.
3792
3793%% Caution: unless the table has been fixed, strange results can be returned.
3794get_all_objects(Tab) -> get_all_objects(dets:first(Tab), Tab, []).
3795
3796%% Assuming no key matches {error, Reason}...
3797get_all_objects('$end_of_table', _Tab, L) -> L;
3798get_all_objects({error, Reason}, _Tab, _L) ->
3799    exit({get_all_objects, {error, Reason}});
3800get_all_objects(Key, Tab, L) ->
3801    Objs = dets:lookup(Tab, Key),
3802    get_all_objects(dets:next(Tab, Key), Tab, Objs ++ L).
3803
3804count_objects_quite_fast(Tab) ->
3805    R1 = dets:match_object(Tab, '_', 1),
3806    count_objs_1(R1, 0).
3807
3808count_objs_1('$end_of_table', N) ->
3809    N;
3810count_objs_1({Ts,C}, N) when is_list(Ts) ->
3811    count_objs_1(dets:match_object(C), length(Ts) + N).
3812
3813get_all_objects_fast(Tab) ->
3814    dets:match_object(Tab, '_').
3815
3816histogram(Tab) ->
3817    OnePercent = case dets:info(Tab, no_slots) of
3818	undefined -> undefined;
3819	{_, NoSlots, _} -> NoSlots/100
3820    end,
3821    histogram(Tab, OnePercent).
3822
3823histogram(Tab, OnePercent) ->
3824    E = ets:new(histo, []),
3825    dets:safe_fixtable(Tab, true),
3826    Hist = histo(Tab, E, 0, OnePercent, OnePercent),
3827    dets:safe_fixtable(Tab, false),
3828    case Hist of
3829        ok ->
3830            H = ets:tab2list(E),
3831            true = ets:delete(E),
3832            sort(H);
3833        Error ->
3834            ets:delete(E),
3835            Error
3836    end.
3837
3838histo(T, E, I, One, Count) when is_number(Count), I > Count ->
3839    io:format("."),
3840    histo(T, E, I, One, Count+One);
3841histo(T, E, I, One, Count) ->
3842    case dets:slot(T, I) of
3843        '$end_of_table' when is_number(Count) ->
3844            io:format("~n"),
3845            ok;
3846        '$end_of_table' ->
3847            ok;
3848        Objs when is_list(Objs) ->
3849            L = length(Objs),
3850            case catch ets:update_counter(E, L, 1) of
3851                {'EXIT', _} ->
3852                    ets:insert(E, {L, 1});
3853                _ ->
3854                    ok
3855            end,
3856            histo(T, E, I+1, One, Count);
3857        Error ->
3858            Error
3859    end.
3860
3861sum_histogram(H) ->
3862    sum_histogram(H, 0).
3863
3864sum_histogram([{S,N1} | H], N) ->
3865    sum_histogram(H, N + S*N1);
3866sum_histogram([], N) ->
3867    N.
3868
3869ave_histogram(H) ->
3870    ave_histogram(H, 0)/sum_histogram(H).
3871
3872ave_histogram([{S,N1} | H], N) ->
3873    ave_histogram(H, N + S*S*N1);
3874ave_histogram([], N) ->
3875    N.
3876
3877bad_object({error,{{bad_object,_}, FileName}}, FileName) ->
3878    ok; % No debug.
3879bad_object({error,{{{bad_object,_,_},_,_,_}, FileName}}, FileName) ->
3880    ok. % Debug.
3881
3882check_badarg({'EXIT', {badarg, [{M,F,Args,_} | _]}}, M, F, Args) ->
3883    true;
3884check_badarg({'EXIT', {badarg, [{M,F,A,_} | _]}}, M, F, Args)  ->
3885    true = test_server:is_native(M) andalso length(Args) =:= A.
3886
3887check_pps({Ports0,Procs0} = P0) ->
3888    ok = check_dets_tables(),
3889    case pps() of
3890        P0 ->
3891            ok;
3892        _ ->
3893            %% On some (rare) occasions the dets process is still
3894            %% running although the call to close() has returned, as
3895            %% it seems...
3896            timer:sleep(500),
3897            case pps() of
3898                P0 ->
3899                    ok;
3900                {Ports1,Procs1} = P1 ->
3901		    case {Ports1 -- Ports0, Procs1 -- Procs0} of
3902			{[], []} -> ok;
3903			{PortsDiff,ProcsDiff} ->
3904			    io:format("failure, got ~p~n, expected ~p\n", [P1, P0]),
3905			    show("Old port", Ports0 -- Ports1),
3906			    show("New port", PortsDiff),
3907			    show("Old proc", Procs0 -- Procs1),
3908			    show("New proc", ProcsDiff),
3909			    ct:fail(failed)
3910		    end
3911	    end
3912    end.
3913
3914%% Copied from dets_server.erl:
3915-define(REGISTRY, dets_registry).
3916-define(OWNERS, dets_owners).
3917-define(STORE, dets).
3918
3919check_dets_tables() ->
3920    Store = [T ||
3921                T <- ets:all(),
3922                ets:info(T, name) =:= ?STORE,
3923                owner(T) =:= dets],
3924    S = case Store of
3925            [Tab] -> ets:tab2list(Tab);
3926            [] -> []
3927        end,
3928    case {ets:tab2list(?REGISTRY), ets:tab2list(?OWNERS), S} of
3929        {[], [], []} -> ok;
3930        {R, O, _} ->
3931            io:format("Registry: ~p~n", [R]),
3932            io:format("Owners: ~p~n", [O]),
3933            io:format("Store: ~p~n", [S]),
3934            not_ok
3935    end.
3936
3937owner(Tab) ->
3938    Owner = ets:info(Tab, owner),
3939    case process_info(Owner, registered_name) of
3940	{registered_name, Name} -> Name;
3941	_ -> Owner
3942    end.
3943
3944show(_S, []) ->
3945    ok;
3946show(S, [{Pid, Name, InitCall}|Pids]) when is_pid(Pid) ->
3947    io:format("~s: ~w (~w), ~w: ~p~n",
3948              [S, Pid, proc_reg_name(Name), InitCall,
3949               erlang:process_info(Pid)]),
3950    show(S, Pids);
3951show(S, [{Port, _}|Ports]) when is_port(Port)->
3952    io:format("~s: ~w: ~p~n", [S, Port, erlang:port_info(Port)]),
3953    show(S, Ports).
3954
3955pps() ->
3956    dets:start(),
3957    {port_list(), process_list()}.
3958
3959port_list() ->
3960    [{P,safe_second_element(erlang:port_info(P, name))} ||
3961        P <- erlang:ports()].
3962
3963process_list() ->
3964    [{P,process_info(P, registered_name),
3965      safe_second_element(process_info(P, initial_call))} ||
3966        P <- processes()].
3967
3968proc_reg_name({registered_name, Name}) -> Name;
3969proc_reg_name([]) -> no_reg_name.
3970
3971safe_second_element({_,Info}) -> Info;
3972safe_second_element(Other) -> Other.
3973