1%%
2%% %CopyrightBegin%
3%%
4%%
5%% Copyright Ericsson AB 2002-2018. All Rights Reserved.
6%%
7%% Licensed under the Apache License, Version 2.0 (the "License");
8%% you may not use this file except in compliance with the License.
9%% You may obtain a copy of the License at
10%%
11%%     http://www.apache.org/licenses/LICENSE-2.0
12%%
13%% Unless required by applicable law or agreed to in writing, software
14%% distributed under the License is distributed on an "AS IS" BASIS,
15%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16%% See the License for the specific language governing permissions and
17%% limitations under the License.
18%%
19%% %CopyrightEnd%
20%%
21
22-module(ttb_SUITE).
23
24-compile(export_all).
25%% Test functions
26-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
27	 init_per_group/2,end_per_group/2,
28	 file/1,file_no_pi/1,file_fetch/1,wrap/1,wrap_merge/1,
29	 wrap_merge_fetch_format/1,write_config1/1,write_config2/1,
30	 write_config3/1,history/1,write_trace_info/1,seq_trace/1,
31	 diskless/1,otp_4967_1/1,otp_4967_2/1]).
32-export([init_per_testcase/2, end_per_testcase/2]).
33-export([foo/0]).
34
35-include_lib("common_test/include/ct.hrl").
36
37-define(default_timeout, test_server:minutes(1)).
38-define(OUTPUT, "handler_output").
39-define(FNAME, "temptest").
40-define(DIRNAME, "ddtemp").
41
42init_per_testcase(Case, Config) ->
43    ?line Dog=test_server:timetrap(?default_timeout),
44    ttb:stop(),
45    rm(?OUTPUT),
46    [rm(Upload) || Upload<-filelib:wildcard("ttb_upload*")],
47    rm(?DIRNAME),
48    [rm(At) || At <- filelib:wildcard("*@*")],
49    rm("ttb_last_config"),
50    %% Workaround for bug(?) in test_server - if the test case fails
51    %% with a timetrap timeout, then end_per_testcase will run with
52    %% faulty group_leader - which in turn makes test_server:stop_node
53    %% hang (stop_node is called by most of the cleanup functions).
54    %% Therefore we do the cleanup before each testcase instead - this
55    %% is obviously not 100% correct, but it will at least make sure
56    %% that the nodes which are to be started in a test case at are
57    %% terminated.
58    try apply(?MODULE,Case,[cleanup,Config])
59    catch error:undef -> ok
60    end,
61    [{watchdog, Dog}|Config].
62end_per_testcase(_Case, Config) ->
63    %% try apply(?MODULE,Case,[cleanup,Config])
64    %% catch error:undef -> ok
65    %% end,
66    Dog=?config(watchdog, Config),
67    test_server:timetrap_cancel(Dog),
68    ok.
69
70suite() -> [{ct_hooks,[ts_install_cth]}].
71
72all() ->
73    [file, file_no_pi, file_fetch, wrap, wrap_merge,
74     wrap_merge_fetch_format, write_config1, write_config2,
75     write_config3, history, write_trace_info, seq_trace,
76     diskless, diskless_wrap, otp_4967_1, otp_4967_2,
77     fetch_when_no_option_given, basic_ttb_run_ip_port, basic_ttb_run_file_port,
78     return_fetch_dir_implies_fetch, logfile_name_in_fetch_dir, upload_to_my_logdir,
79     upload_to_my_existing_logdir, fetch_with_options_not_as_list,
80     error_when_formatting_multiple_files_4393, format_on_trace_stop,
81     trace_to_remote_files_on_localhost_with_different_pwd,
82     trace_to_local_files_on_localhost_with_different_pwd,
83     trace_to_remote_files_on_localhost_with_different_pwd_abs,
84     changing_cwd_on_control_node, changing_cwd_on_remote_node,
85     changing_cwd_on_control_node_with_local_trace,
86     one_command_trace_setup, dbg_style_fetch, shell_tracing_init,
87     only_one_state_for_format_handler, only_one_state_with_default_format_handler,
88     only_one_state_with_initial_format_handler, run_trace_with_shortcut1,
89     run_trace_with_shortcut2, run_trace_with_shortcut3, run_trace_with_shortcut4,
90     cant_specify_local_and_flush, trace_sorted_by_default,disable_sorting,
91     trace_resumed_after_node_restart, trace_resumed_after_node_restart_ip,
92     trace_resumed_after_node_restart_wrap,
93     trace_resumed_after_node_restart_wrap_mult
94].
95
96groups() ->
97    [].
98
99init_per_suite(Config) ->
100    clean_priv_dir(Config),
101    Config.
102
103end_per_suite(_Config) ->
104    ok.
105
106init_per_group(_GroupName, Config) ->
107    Config.
108
109end_per_group(_GroupName, Config) ->
110    Config.
111
112
113file(suite) ->
114    [];
115file(doc) ->
116    ["Start tracing on multiple nodes, single file"];
117file(Config) when is_list(Config) ->
118    ?line Node = node(),
119    ?line {ok,OtherNode} = test_server:start_node(node2,slave,[]),
120    ?line c:nl(?MODULE),
121    ?line S = self(),
122    ?line Privdir=priv_dir(Config),
123    ?line File = filename:join(Privdir,"file"),
124    ?line {ok,[Node]} =
125	ttb:tracer(Node,[{file, File},
126			 {handler,{fun myhandler/4, S}}]),
127    ?line {ok,[{S,[{matched,Node,_}]}]} = ttb:p(S,call),
128    ?line {ok,[OtherNode]} =
129	ttb:tracer([Node,OtherNode],[{file, File},
130				     {handler,{fun myhandler/4, S}}]),
131    ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
132    ?line {ok,[]} = ttb:tracer([Node,OtherNode],
133				  [{file, File},
134				   {handler,{fun myhandler/4, S}}]),
135    ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
136    ?line ?MODULE:foo(),
137    ?line rpc:call(OtherNode,?MODULE,foo,[]),
138    ?line ttb:stop([nofetch]),
139    ?line ok = ttb:format(filename:join(Privdir,atom_to_list(Node)++"-file")),
140    ?line ok = ttb:format(filename:join(Privdir,
141					atom_to_list(OtherNode)++"-file")),
142
143    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
144	   end_of_trace,
145	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
146	   end_of_trace] = flush(),
147    ok.
148
149file(cleanup,_Config) ->
150    ?line test_server:stop_node(ttb_helper:get_node(node2)).
151
152file_no_pi(suite) ->
153    [];
154file_no_pi(doc) ->
155    ["Start tracing on multiple nodes, single file, no process information"];
156file_no_pi(Config) when is_list(Config) ->
157    ?line Node = node(),
158    ?line {ok,OtherNode} = test_server:start_node(node2,slave,[]),
159    ?line c:nl(?MODULE),
160    ?line S = self(),
161    ?line Privdir=priv_dir(Config),
162    ?line File = filename:join(Privdir,"file"),
163    ?line {ok,[_,_]} =
164	ttb:tracer([Node,OtherNode],[{file, File},
165				     {handler,{fun myhandler/4, S}},
166				     {process_info,false}]),
167    ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
168    ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
169    ?line ?MODULE:foo(),
170    ?line rpc:call(OtherNode,?MODULE,foo,[]),
171    ?line ttb:stop([nofetch]),
172    ?line ok = ttb:format(filename:join(Privdir,atom_to_list(Node)++"-file")),
173    ?line ok = ttb:format(filename:join(Privdir,
174					atom_to_list(OtherNode)++"-file")),
175
176    ?line [{trace_ts,LocalProc,call,{?MODULE,foo,[]}, {_,_,_}},
177	   end_of_trace,
178	   {trace_ts,RemoteProc,call,{?MODULE,foo,[]},{_,_,_}},
179	   end_of_trace] = flush(),
180    ?line true = is_pid(LocalProc),
181    ?line true = is_pid(RemoteProc),
182    ok.
183
184file_no_pi(cleanup,_Config) ->
185    ?line test_server:stop_node(ttb_helper:get_node(node2)).
186
187
188file_fetch(suite) ->
189    [];
190file_fetch(doc) ->
191    ["stop with the fetch option, i.e. collect all files when ttb is stopped"];
192file_fetch(Config) when is_list(Config) ->
193    ?line Node = node(),
194    ?line {ok,OtherNode} = test_server:start_node(node2,slave,[]),
195    ?line c:nl(?MODULE),
196    ?line S = self(),
197    ?line Privdir=priv_dir(Config),
198    ?line ThisDir = filename:join(Privdir,this),
199    ?line ok = file:make_dir(ThisDir),
200    ?line OtherDir = filename:join(Privdir,other),
201    ?line ok = file:make_dir(OtherDir),
202    ?line ThisFile = filename:join(ThisDir,"file_fetch"),
203    ?line OtherFile = filename:join(OtherDir,"file_fetch"),
204
205    %% I'm setting priv_dir as cwd, so ttb_upload directory is created there
206    %% and not in any other strange place!
207    ?line {ok,Cwd} = file:get_cwd(),
208    ?line ok = file:set_cwd(Privdir),
209
210    ?line {ok,[Node]} =
211	ttb:tracer(Node,[{file, ThisFile},
212			 {handler,{fun myhandler/4, S}}]),
213    ?line {ok,[OtherNode]} =
214	ttb:tracer([OtherNode],[{file, OtherFile},
215				     {handler,{fun myhandler/4, S}}]),
216    ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
217    ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
218    ?line ?MODULE:foo(),
219    ?line rpc:call(OtherNode,?MODULE,foo,[]),
220    ?line test_server:capture_start(),
221    ?line ttb:stop([return_fetch_dir]),
222    ?line test_server:capture_stop(),
223    ?line [StoreString] = test_server:capture_get(),
224    ?line UploadDir =
225	lists:last(string:lexemes(lists:flatten(StoreString),"$ \n")),
226
227    %% check that files are no longer in original directories...
228    ?line ok = check_gone(ThisDir,atom_to_list(Node)++"-file_fetch"),
229    ?line ok = check_gone(ThisDir,atom_to_list(Node)++"-file_fetch.ti"),
230%    ?line false = lists:member(TrcLog,ThisList),
231%    ?line false = lists:member(TIFile,ThisList),
232
233    ?line {ok,OtherList} = file:list_dir(OtherDir),
234    ?line false = lists:member(atom_to_list(OtherNode)++"-file_fetch",OtherList),
235    ?line false = lists:member(atom_to_list(OtherNode)++"-file_fetch.ti",
236			       OtherList),
237
238    %% but instead in ttb_upload directory, where they can be formatted
239    ?line ok = ttb:format(filename:join(UploadDir,
240					atom_to_list(Node)++"-file_fetch")),
241    ?line ok = ttb:format(filename:join(UploadDir,
242					atom_to_list(OtherNode)++"-file_fetch")),
243
244    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
245	   end_of_trace,
246	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
247	   end_of_trace] = flush(),
248
249    ?line ok = file:set_cwd(Cwd),
250    ok.
251
252file_fetch(cleanup,_Config) ->
253    ?line test_server:stop_node(ttb_helper:get_node(node2)).
254
255
256wrap(suite) ->
257    [];
258wrap(doc) ->
259    ["Start tracing on multiple nodes, wrap files"];
260wrap(Config) when is_list(Config) ->
261    ?line Node = node(),
262    ?line {ok,OtherNode} = test_server:start_node(node2,slave,[]),
263    ?line c:nl(?MODULE),
264    ?line S = self(),
265    ?line Privdir=priv_dir(Config),
266    ?line File = filename:join(Privdir,"wrap"),
267    ?line {ok,[_,_]} =
268	ttb:tracer([Node,OtherNode],[{file, {wrap,File,200,3}},
269				     {handler,{fun myhandler/4, S}}]),
270    ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
271    ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
272    ?line ?MODULE:foo(),
273    ?line rpc:call(OtherNode,?MODULE,foo,[]),
274    ?line ?MODULE:foo(),
275    ?line rpc:call(OtherNode,?MODULE,foo,[]),
276    ?line ?MODULE:foo(),
277    ?line rpc:call(OtherNode,?MODULE,foo,[]),
278    ?line ttb:stop([nofetch]),
279    ?line ok = ttb:format(filename:join(Privdir,
280					atom_to_list(Node)++"-wrap.*.wrp")),
281    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
282	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
283	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
284	   end_of_trace] = flush(),
285    ?line ok = ttb:format(filename:join(Privdir,
286					atom_to_list(OtherNode)++"-wrap.*.wrp")),
287    ?line [{trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
288	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
289	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
290	   end_of_trace] = flush(),
291
292    %% Check that merge does not crash even if the timestamp flag is not on.
293    ?line ok = ttb:format(
294		 [filename:join(Privdir,
295				atom_to_list(Node)++"-wrap.*.wrp"),
296		  filename:join(Privdir,
297				atom_to_list(OtherNode)++"-wrap.*.wrp")],[{disable_sort,true}]),
298    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
299	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
300	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
301	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
302	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
303	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
304	   end_of_trace] = flush(),
305    ok.
306
307wrap(cleanup,_Config) ->
308    ?line test_server:stop_node(ttb_helper:get_node(node2)).
309
310wrap_merge(suite) ->
311    [];
312wrap_merge(doc) ->
313    ["Start tracing on multiple nodes, wrap files, merge logs from both nodes"];
314wrap_merge(Config) when is_list(Config) ->
315    ?line Node = node(),
316    ?line {ok,OtherNode} = test_server:start_node(node2,slave,[]),
317    ?line c:nl(?MODULE),
318    ?line S = self(),
319    ?line Privdir=priv_dir(Config),
320    ?line File = filename:join(Privdir,"wrap_merge"),
321    ?line {ok,[_,_]} =
322	ttb:tracer([Node,OtherNode],[{file, {wrap,File,200,3}},
323				     {handler,{fun myhandler/4, S}}]),
324    ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]}=ttb:p(all,[call,timestamp]),
325    ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
326    ?line ?MODULE:foo(),
327    ?line rpc:call(OtherNode,?MODULE,foo,[]),
328    ?line ?MODULE:foo(),
329    ?line rpc:call(OtherNode,?MODULE,foo,[]),
330    ?line ?MODULE:foo(),
331    ?line rpc:call(OtherNode,?MODULE,foo,[]),
332    ?line ttb:stop([nofetch]),
333    ?line ok = ttb:format(
334		 [filename:join(Privdir,
335				atom_to_list(Node)++"-wrap_merge.*.wrp"),
336		  filename:join(Privdir,
337				atom_to_list(OtherNode)++"-wrap_merge.*.wrp")]),
338    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
339	   {trace_ts,_,call,{?MODULE,foo,[]},_},
340	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
341	   {trace_ts,_,call,{?MODULE,foo,[]},_},
342	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
343	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},_},
344	   end_of_trace] = flush(),
345    ok.
346
347wrap_merge(cleanup,_Config) ->
348    ?line test_server:stop_node(ttb_helper:get_node(node2)).
349
350
351wrap_merge_fetch_format(suite) ->
352    [];
353wrap_merge_fetch_format(doc) ->
354    ["Start tracing on multiple nodes, wrap files, fetch and format at stop"];
355wrap_merge_fetch_format(Config) when is_list(Config) ->
356    ?line Node = node(),
357    ?line {ok,OtherNode} = test_server:start_node(node2,slave,[]),
358    ?line c:nl(?MODULE),
359    ?line S = self(),
360    ?line Privdir=priv_dir(Config),
361    ?line File = filename:join(Privdir,"wrap_merge_fetch_format"),
362
363    %% I'm setting priv_dir as cwd, so ttb_upload directory is created there
364    %% and not in any other strange place!
365    ?line {ok,Cwd} = file:get_cwd(),
366    ?line ok = file:set_cwd(Privdir),
367
368    ?line {ok,[_,_]} =
369	ttb:tracer([Node,OtherNode],[{file, {wrap,File,200,3}},
370				     {handler,{fun myhandler/4, S}}]),
371    ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]}=ttb:p(all,[call,timestamp]),
372    ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
373    ?line ?MODULE:foo(),
374    ?line rpc:call(OtherNode,?MODULE,foo,[]),
375    ?line ?MODULE:foo(),
376    ?line rpc:call(OtherNode,?MODULE,foo,[]),
377    ?line ?MODULE:foo(),
378    ?line rpc:call(OtherNode,?MODULE,foo,[]),
379    ?line ttb:stop([format]),
380    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
381	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},_},
382	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
383	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},_},
384	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
385	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},_},
386	   end_of_trace] = flush(),
387
388    ?line ok = file:set_cwd(Cwd),
389    ok.
390
391wrap_merge_fetch_format(cleanup,_Config) ->
392    ?line test_server:stop_node(ttb_helper:get_node(node2)).
393
394write_config1(suite) ->
395    [];
396write_config1(doc) ->
397    ["Write config given commands"];
398write_config1(Config) when is_list(Config) ->
399    ?line Node = node(),
400    ?line {ok,OtherNode} = test_server:start_node(node2,slave,[]),
401    ?line c:nl(?MODULE),
402    ?line S = self(),
403
404    ?line Privdir=priv_dir(Config),
405    ?line File = filename:join(Privdir,"write_config1"),
406    ?line ok = ttb:write_config(File,
407				[{ttb,tracer,[[Node,OtherNode],
408					      [{file, File},
409					       {handler,{fun myhandler/4,S}}]]},
410				 {ttb,p,[all,call]},
411				 {ttb,tp,[?MODULE,foo,[]]}]),
412    ?line [_,_,_] = ttb:list_config(File),
413    ?line ok = ttb:run_config(File),
414    ?line ?MODULE:foo(),
415    ?line rpc:call(OtherNode,?MODULE,foo,[]),
416    ?line ttb:stop([nofetch]),
417    ?line ok = ttb:format(
418		 [filename:join(Privdir,
419				atom_to_list(Node)++"-write_config1"),
420		  filename:join(Privdir,
421				atom_to_list(OtherNode)++"-write_config1")]),
422    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
423	   {trace_ts,Other,call,{?MODULE,foo,[]},{_,_,_}},
424	   end_of_trace] = flush(),
425
426    case metatest(Other,OtherNode,Privdir,"-write_config1.ti") of
427	{error,Reason} ->
428	    timer:sleep(5000),
429	    ?line ok = ttb:format(
430			 [filename:join(Privdir,
431					atom_to_list(Node)++"-write_config1"),
432			  filename:join(Privdir,
433					atom_to_list(OtherNode)++
434					"-write_config1")]),
435	    ?line io:format("\nTrying again: ~p\n",[flush()]),
436	    ?line ct:fail(Reason);
437	ok ->
438	    ok
439    end,
440    ok.
441
442
443write_config1(cleanup,_Config) ->
444    ?line test_server:stop_node(ttb_helper:get_node(node2)).
445
446write_config2(suite) ->
447    [];
448write_config2(doc) ->
449    ["Write config from history (all)"];
450write_config2(Config) when is_list(Config) ->
451    ?line Node = node(),
452    ?line {ok,OtherNode} = test_server:start_node(node2,slave,[]),
453    ?line c:nl(?MODULE),
454    ?line S = self(),
455    ?line Privdir=priv_dir(Config),
456    ?line File = filename:join(Privdir,"write_config2"),
457    ?line {ok,[_,_]} =
458	ttb:tracer([Node,OtherNode],[{file, File},
459				     {handler,{fun myhandler/4, S}}]),
460    ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
461    ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
462    ?line ok = ttb:write_config(File,all),
463    ?line ttb:stop(),
464    ?line [_,_,_] = ttb:list_config(File),
465    ?line ok = ttb:run_config(File),
466    ?line ?MODULE:foo(),
467    ?line rpc:call(OtherNode,?MODULE,foo,[]),
468    ?line ttb:stop([nofetch]),
469    ?line ok = ttb:format(
470		 [filename:join(Privdir,
471				atom_to_list(Node)++"-write_config2"),
472		  filename:join(Privdir,
473				atom_to_list(OtherNode)++"-write_config2")]),
474    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
475	   {trace_ts,Other,call,{?MODULE,foo,[]},{_,_,_}},
476	   end_of_trace] = flush(),
477
478    case metatest(Other,OtherNode,Privdir,"-write_config2.ti") of
479	{error,Reason} ->
480	    timer:sleep(5000),
481	    ?line ok = ttb:format(
482			 [filename:join(Privdir,
483					atom_to_list(Node)++"-write_config2"),
484			  filename:join(Privdir,
485					atom_to_list(OtherNode)++
486					"-write_config2")]),
487	    ?line io:format("\nTrying again: ~p\n",[flush()]),
488	    ?line ct:fail(Reason);
489	ok ->
490	    ok
491    end,
492    ok.
493
494write_config2(cleanup,_Config) ->
495    ?line test_server:stop_node(ttb_helper:get_node(node2)).
496
497
498write_config3(suite) ->
499    [];
500write_config3(doc) ->
501    ["Write config from history (selected and append)"];
502write_config3(Config) when is_list(Config) ->
503    ?line Node = node(),
504    ?line {ok,OtherNode} = test_server:start_node(node2,slave,[]),
505    ?line c:nl(?MODULE),
506    ?line S = self(),
507    ?line Privdir=priv_dir(Config),
508    ?line File = filename:join(Privdir,"write_config3"),
509    ?line {ok,[_,_]} =
510	ttb:tracer([Node,OtherNode],[{file, File},
511				     {handler,{fun myhandler/4, S}}]),
512    ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
513    ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
514    ?line ok = ttb:write_config(File,[1,2]),
515    ?line ttb:stop([nofetch]),
516    ?line [_,_] = ttb:list_config(File),
517    ?line ok = ttb:run_config(File),
518    ?line ?MODULE:foo(),
519    ?line rpc:call(OtherNode,?MODULE,foo,[]),
520    ?line ttb:stop([nofetch]),
521    ?line ok = ttb:format(
522		 [filename:join(Privdir,
523				atom_to_list(Node)++"-write_config3"),
524		  filename:join(Privdir,
525				atom_to_list(OtherNode)++"-write_config3")]),
526    ?line [end_of_trace] = flush(), %foo is not traced
527
528    ?line ok = ttb:write_config(File,[{ttb,tp,[?MODULE,foo,[]]}],
529			      [append]),
530    ?line [_,_,_] = ttb:list_config(File),
531    ?line ok = ttb:run_config(File),
532    ?line ?MODULE:foo(),
533    ?line rpc:call(OtherNode,?MODULE,foo,[]),
534    ?line ttb:stop([nofetch]),
535    ?line ok = ttb:format(
536		 [filename:join(Privdir,
537				atom_to_list(Node)++"-write_config3"),
538		  filename:join(Privdir,
539				atom_to_list(OtherNode)++"-write_config3")]),
540    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
541	   {trace_ts,Other,call,{?MODULE,foo,[]},{_,_,_}},
542	   end_of_trace] = flush(),
543
544    case metatest(Other,OtherNode,Privdir,"-write_config3.ti") of
545	{error,Reason} ->
546	    timer:sleep(5000),
547	    ?line ok = ttb:format(
548			 [filename:join(Privdir,
549					atom_to_list(Node)++"-write_config3"),
550			  filename:join(Privdir,
551					atom_to_list(OtherNode)++
552					"-write_config3")]),
553	    ?line io:format("\nTrying again: ~p\n",[flush()]),
554	    ?line ct:fail(Reason);
555	ok ->
556	    ok
557    end,
558    ok.
559
560write_config3(cleanup,_Config) ->
561    ?line test_server:stop_node(ttb_helper:get_node(node2)).
562
563
564
565history(suite) ->
566    [];
567history(doc) ->
568    ["List history and execute entry from history"];
569history(Config) when is_list(Config) ->
570    ?line Node = node(),
571    ?line {ok,OtherNode} = test_server:start_node(node2,slave,[]),
572    ?line c:nl(?MODULE),
573    ?line S = self(),
574
575    ?line Nodes = [Node,OtherNode],
576    ?line Privdir=priv_dir(Config),
577    ?line File = filename:join(Privdir,"history"),
578    ?line StartOpts = [{file, File},
579		       {handler,{fun myhandler/4, S}}],
580    ?line {ok,[_,_]} = ttb:tracer(Nodes,StartOpts),
581    ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
582    ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
583    ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:ctp(?MODULE,foo),
584    ?line [{1,{ttb,tracer,[Nodes,StartOpts]}},
585	   {2,{ttb,p,[all,call]}},
586	   {3,{ttb,tp,[?MODULE,foo,[]]}},
587	   {4,{ttb,ctp,[?MODULE,foo]}}] = ttb:list_history(),
588    ?line rpc:call(OtherNode,?MODULE,foo,[]),
589    ?line ok = ttb:run_history(3),
590    ?line ?MODULE:foo(),
591    ?line ok = ttb:run_history([3,4]),
592    ?line ?MODULE:foo(),
593    ?line ttb:stop([nofetch]),
594    ?line ok = ttb:format(
595		 [filename:join(Privdir,atom_to_list(Node)++"-history"),
596		  filename:join(Privdir,atom_to_list(OtherNode)++"-history")]),
597    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
598	   end_of_trace] = flush(),
599    ok.
600
601history(cleanup,_Config) ->
602    ?line test_server:stop_node(ttb_helper:get_node(node2)).
603
604
605write_trace_info(suite) ->
606    [];
607write_trace_info(doc) ->
608    ["Write trace info and give handler explicitly in format command"];
609write_trace_info(Config) when is_list(Config) ->
610    ?line Node = node(),
611    ?line {ok,OtherNode} = test_server:start_node(node2,slave,[]),
612    ?line c:nl(?MODULE),
613    ?line S = self(),
614    ?line Privdir=priv_dir(Config),
615    ?line File = filename:join(Privdir,"write_trace_info"),
616    ?line {ok,[_,_]} =
617	ttb:tracer([Node,OtherNode],[{file, File},
618				     {handler,{fun myhandler/4, S}}]),
619    ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
620    ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
621    ?line ok = ttb:write_trace_info(mytraceinfo,fun() -> node() end),
622    ?line ?MODULE:foo(),
623    ?line rpc:call(OtherNode,?MODULE,foo,[]),
624    ?line ttb:stop([nofetch]),
625    ?line ok = ttb:format(
626		 [filename:join(Privdir,atom_to_list(Node)++"-write_trace_info"),
627		  filename:join(Privdir,
628				atom_to_list(OtherNode)++"-write_trace_info")],
629		 [{handler,{fun otherhandler/4,S}}]),
630    ?line [{{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},[Node]},
631	   {{trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},[OtherNode]},
632	   end_of_trace] = flush(),
633
634    ok.
635
636write_trace_info(cleanup,_Config) ->
637    ?line test_server:stop_node(ttb_helper:get_node(node2)).
638
639
640seq_trace(suite) ->
641    [];
642seq_trace(doc) ->
643    ["Test sequential tracing"];
644seq_trace(Config) when is_list(Config) ->
645    ?line S = self(),
646
647    ?line Privdir=priv_dir(Config),
648    ?line File = filename:join(Privdir,"seq_trace"),
649    ?line {ok,[Node]} = ttb:tracer(node(),[{file,File},
650				     {handler,{fun myhandler/4, S}}]),
651    ?line {ok,[{new,[{matched,Node,0}]}]} = ttb:p(new,call),
652    ?line {ok,[{matched,Node,1},{saved,1}]} =
653	   ttb:tpl(?MODULE,seq,0,ttb:seq_trigger_ms(send)),
654
655    ?line Start = spawn(fun() -> seq() end),
656    ?line timer:sleep(300),
657    ?line ttb:stop([nofetch]),
658    ?line ok = ttb:format(
659		 [filename:join(Privdir,atom_to_list(Node)++"-seq_trace")]),
660    ?line [{trace_ts,StartProc,call,{?MODULE,seq,[]},{_,_,_}},
661           {seq_trace,0,{send,{First, Seq0},StartProc,P1Proc,SpawnRequest1}},
662           {seq_trace,0,{send,{Seq0, Seq1},P1Proc,StartProc,SpawnReply1}},
663           {seq_trace,0,{send,{Seq2, Seq3},StartProc,P2Proc,SpawnRequest2}},
664           {seq_trace,0,{send,{Seq3, Seq4},P2Proc,StartProc,SpawnReply2}},
665	   {seq_trace,0,{send,{Seq5, Seq6},StartProc,P1Proc,{Start,P2}}},
666	   {seq_trace,0,{send,{Seq6,  Seq7},P1Proc,P2Proc,{P1,Start}}},
667	   {seq_trace,0,{send,{Seq7,  Last},P2Proc,StartProc,{P2,P1}}},
668	   end_of_trace] = flush(),
669    spawn_request = element(1, SpawnRequest1),
670    SReq1 = element(2, SpawnRequest1),
671    spawn_reply = element(1, SpawnReply1),
672    SReq1 = element(2, SpawnReply1),
673    spawn_request = element(1, SpawnRequest2),
674    SReq2 = element(2, SpawnRequest2),
675    spawn_reply = element(1, SpawnReply2),
676    SReq2 = element(2, SpawnReply2),
677    true = First < Seq0,
678    true = Seq0 < Seq1,
679    true = Seq1 < Seq2,
680    true = Seq2 < Seq3,
681    true = Seq4 < Seq5,
682    true = Seq6 < Seq7,
683    true = Seq7 < Last,
684   %% Additional test for metatrace
685    case StartProc of
686	{Start,_,_} -> ok;
687	Pid when is_pid(Pid) ->
688	    io:format("\n\nProcinfo was pid: ~p.\n"
689		      "Should have been {Pid,Name,Node}\n",
690		      [Pid]),
691	    io:format("Trace information file:\n~p\n",
692		      [ttb:dump_ti(
693			 filename:join(Privdir,
694				       atom_to_list(Node)++"-seq_trace.ti"))]),
695	    ct:fail("metatrace failed for startproc")
696    end,
697    case P1Proc of
698	{P1,_,_} -> ok;
699	P1 when is_pid(P1) ->
700	    io:format("\n\nProcinfo was pid: ~p.\n"
701		      "Should have been {Pid,Name,Node}\n",
702		      [P1]),
703	    io:format("Trace information file:\n~p\n",
704		      [ttb:dump_ti(
705			 filename:join(Privdir,
706				       atom_to_list(Node)++"-seq_trace.ti"))]),
707	    ct:fail("metatrace failed for P1")
708    end,
709    case P2Proc of
710	{P2,_,_} -> ok;
711	P2 when is_pid(P2) ->
712	    io:format("\n\nProcinfo was pid: ~p.\n"
713		      "Should have been {Pid,Name,Node}\n",
714		      [P2]),
715	    io:format("Trace information file:\n~p\n",
716		      [ttb:dump_ti(
717			 filename:join(Privdir,
718				       atom_to_list(Node)++"-seq_trace.ti"))]),
719	    ct:fail("metatrace failed for P2")
720    end,
721    ok.
722
723
724diskless(suite) ->
725    [];
726diskless(doc) ->
727    ["Start tracing on diskless remote node"];
728diskless(Config) when is_list(Config) ->
729    ?line {ok,RemoteNode} = test_server:start_node(node2,slave,[]),
730    ?line c:nl(?MODULE),
731    ?line S = self(),
732    ?line Privdir=priv_dir(Config),
733    ?line File = filename:join(Privdir,"diskless"),
734    ?line {ok,[RemoteNode]} =
735	ttb:tracer([RemoteNode],[{file, {local, File}},
736				 {handler,{fun myhandler/4, S}}]),
737    ?line {ok,[{all,[{matched,RemoteNode,_}]}]} = ttb:p(all,call),
738    ?line {ok,[{matched,RemoteNode,1}]} = ttb:tp(?MODULE,foo,[]),
739
740    ?line rpc:call(RemoteNode,?MODULE,foo,[]),
741    ?line timer:sleep(5000), % needed for the IP port to flush
742    ?line ttb:stop([nofetch]),
743    ?line ok = ttb:format(filename:join(Privdir,
744					atom_to_list(RemoteNode)++"-diskless")),
745
746    ?line [{trace_ts,{_,_,RemoteNode},call,{?MODULE,foo,[]},{_,_,_}},
747	   end_of_trace] = flush(),
748    ok.
749
750diskless(cleanup,_Config) ->
751    ?line test_server:stop_node(ttb_helper:get_node(node2)).
752
753diskless_wrap(suite) ->
754    [];
755diskless_wrap(doc) ->
756    ["Start tracing on diskless remote node, save to local wrapped file"];
757diskless_wrap(Config) when is_list(Config) ->
758    ?line {ok,RemoteNode} = test_server:start_node(node2,slave,[]),
759    ?line c:nl(?MODULE),
760    ?line S = self(),
761    ?line Privdir=priv_dir(Config),
762    ?line File = filename:join(Privdir,"diskless"),
763    ?line {ok,[RemoteNode]} =
764	ttb:tracer([RemoteNode],[{file, {local, {wrap,File,200,3}}},
765				 {handler,{fun myhandler/4, S}}]),
766    ?line {ok,[{all,[{matched,RemoteNode,_}]}]} = ttb:p(all,call),
767    ?line {ok,[{matched,RemoteNode,1}]} = ttb:tp(?MODULE,foo,[]),
768
769    ?line rpc:call(RemoteNode,?MODULE,foo,[]),
770    ?line timer:sleep(5000), % needed for the IP port to flush
771    ?line ttb:stop([nofetch]),
772    ?line ok = ttb:format(filename:join(Privdir,
773					atom_to_list(RemoteNode)++"-diskless.*.wrp")),
774
775    ?line [{trace_ts,{_,_,RemoteNode},call,{?MODULE,foo,[]},{_,_,_}},
776	   end_of_trace] = flush(),
777    ok.
778
779diskless_wrap(cleanup,_Config) ->
780    ?line test_server:stop_node(ttb_helper:get_node(node2)).
781
782otp_4967_1(suite) ->
783    [];
784otp_4967_1(doc) ->
785    ["OTP-4967: clear flag"];
786otp_4967_1(Config) when is_list(Config) ->
787    ?line {ok,[Node]} = ttb:tracer(),
788    ?line {ok,[{all,[{matched,Node,_}]}]} =  ttb:p(all,call),
789    ?line {ok,[{all,[{matched,Node,_}]}]} =  ttb:p(all,clear),
790    ?line stopped = ttb:stop(),
791    ok.
792
793
794otp_4967_2(suite) ->
795    [];
796otp_4967_2(doc) ->
797    ["OTP-4967: Trace message sent to {Name, Node}"];
798otp_4967_2(Config) when is_list(Config) ->
799    io:format("1: ~p",[erlang:timestamp()]),
800    ?line Privdir = priv_dir(Config),
801    io:format("2: ~p",[erlang:timestamp()]),
802    ?line File = filename:join(Privdir,"otp_4967"),
803    io:format("3: ~p",[erlang:timestamp()]),
804    ?line S = self(),
805    io:format("4: ~p",[erlang:timestamp()]),
806    ?line {ok,[Node]} =
807	ttb:tracer(node(),[{file, File},
808			   {handler,{fun myhandler/4, S}}]),
809
810    io:format("5: ~p",[erlang:timestamp()]),
811    %% Test that delayed registration of a process works.
812    receive after 200 -> ok end,
813    ?line register(otp_4967,self()),
814    io:format("6: ~p",[erlang:timestamp()]),
815    ?line {ok,[{S,[{matched,Node,1}]}]} =  ttb:p(self(),s),
816    io:format("7: ~p",[erlang:timestamp()]),
817    ?line {otp_4967,node()} ! heihopp,
818    io:format("8: ~p",[erlang:timestamp()]),
819    ?line stopped = ttb:stop([format]),
820    io:format("9: ~p",[erlang:timestamp()]),
821    ?line Msgs = flush(),
822    io:format("10: ~p",[erlang:timestamp()]),
823    ?line io:format("Messages received: \n~p\n",[Msgs]),
824    io:format("11: ~p",[erlang:timestamp()]),
825    ?line true = lists:member(heihopp,Msgs), % the heihopp message itself
826    io:format("13: ~p",[erlang:timestamp()]),
827    ?line {value,{trace_ts,_,send,heihopp,{_,otp_4967,Node},{_,_,_}}} =
828	lists:keysearch(heihopp,4,Msgs), % trace trace of the heihopp message
829    io:format("14: ~p",[erlang:timestamp()]),
830    ?line end_of_trace = lists:last(Msgs), % end of the trace
831    ok.
832
833
834myhandler(_Fd,Trace,_,Relay) ->
835    Relay ! Trace,
836    Relay.
837
838simple_call_handler() ->
839    {fun(A, {trace_ts, _, call, _, _} ,_,_) -> io:format(A, "ok.~n", []);
840	(A, {drop, N}, _, _) -> io:format(A, "{drop, ~p}.", [N]);
841	(_, end_of_trace, _, _) -> ok end, []}.
842
843marking_call_handler() ->
844    {fun(_, _, _, initial) -> file:write_file("HANDLER_OK", []);
845	(_,_,_,_) -> ok end, initial}.
846
847counter_call_handler() ->
848    {fun(_, {trace_ts, _, call, _, _} ,_,State) -> State + 1;
849	(A, end_of_trace, _, State) -> io:format(A,"~p.~n", [State]) end, 0}.
850
851ret_caller_call_handler() ->
852    {fun(A, {trace_ts, _, call, _, _, _} ,_,_) -> io:format(A, "ok.~n", []);
853	(A, {trace_ts, _, return_from, _, _, _}, _, _) -> io:format(A, "ok.~n", []);
854	(_, _, _, _) -> ok end, []}.
855
856node_call_handler() ->
857    {fun(A, {trace_ts, {_,_,Node}, call, _, _} ,_,_) -> io:format(A, "~p.~n", [Node]);
858	(_, end_of_trace, _, _) -> ok end, []}.
859
860otherhandler(_Fd,_,end_of_trace,Relay) ->
861    Relay ! end_of_trace,
862    Relay;
863otherhandler(_Fd,Trace,TI,Relay) ->
864    {value,{mytraceinfo,I}} = lists:keysearch(mytraceinfo,1,TI),
865    Relay ! {Trace,I},
866    Relay.
867
868flush() ->
869    flush([]).
870flush(Acc) ->
871    receive
872	X ->
873	    flush(Acc ++ [X])
874    after 1000 ->
875	    Acc
876    end.
877
878foo() ->
879    %% Sync between nodes is not always exact, so here is a litle timeout to
880    %% make sure traces come i correct sequence when merging.
881    %% In the real world there is no way to avoid this kind of trouble
882    timer:sleep(100),
883    foo_called.
884
885
886seq() ->
887    Fun = fun() -> timer:sleep(100),
888		   receive {From,To} -> To ! {self(),From} end
889	  end,
890    P1 = spawn(Fun),
891    P2 = spawn(Fun),
892    P1 ! {self(),P2},
893    receive {P2,P1} -> ok end,
894    {P1,P2}.
895
896%% Additional test for metatrace which might fail on OtherNode
897metatest(Proc,Node,Privdir,Filename) ->
898    case Proc of
899	{_,_,Node} -> ok;
900	Pid when is_pid(Pid) ->
901	    io:format("\n\nProcinfo was pid: ~p.\n"
902		      "Should have been {Pid,Name,Node}\n",
903		      [Pid]),
904	    io:format("Trace information file:\n~p\n",
905		      [ttb:dump_ti(
906			 filename:join(Privdir,atom_to_list(Node)++Filename))]),
907%	    ct:fail("metatrace failed on "++atom_to_list(Node))
908	    {error,"metatrace failed on "++atom_to_list(Node)}
909    end.
910
911check_gone(Dir,File) ->
912    ?line {ok,List} = file:list_dir(Dir),
913    ?line case lists:member(File,List) of
914	      true ->
915		  timer:sleep(2000),
916		  {ok,NewList} = file:list_dir(Dir),
917		  case lists:member(File,NewList) of
918		      true ->
919			  io:format("~p: ~p~n",
920				    [Dir,NewList]),
921			  ct:fail(File ++ " not removed from original place");
922		      false ->
923			  io:format("gone after 2 sec....~n",[]),
924			  ok
925		  end;
926	      false ->
927		  ok
928	  end.
929
930start_client_and_server() ->
931    ?line {ok,ClientNode} = test_server:start_node(client,slave,[]),
932    ?line ok = ttb_helper:c(code, add_paths, [code:get_path()]),
933    ?line {ok,ServerNode} = test_server:start_node(server,slave,[]),
934    ?line ok = ttb_helper:s(code, add_paths, [code:get_path()]),
935    ?line ttb_helper:clear(),
936    {ServerNode, ClientNode}.
937
938stop_client_and_server() ->
939    ClientNode = ttb_helper:get_node(client),
940    ServerNode = ttb_helper:get_node(server),
941    erlang:monitor_node(ClientNode,true),
942    erlang:monitor_node(ServerNode,true),
943    ?line test_server:stop_node(ClientNode),
944    ?line test_server:stop_node(ServerNode),
945    wait_for_client_and_server_stop(ClientNode,ServerNode).
946
947wait_for_client_and_server_stop(undefined,undefined) ->
948    ok;
949wait_for_client_and_server_stop(ClientNode,ServerNode) ->
950    receive
951	{nodedown,ClientNode} ->
952	    erlang:monitor_node(ClientNode,false),
953	    wait_for_client_and_server_stop(undefined,ServerNode);
954	{nodedown,ServerNode} ->
955	    erlang:monitor_node(ServerNode,false),
956	    wait_for_client_and_server_stop(ClientNode,undefined)
957    end.
958
959begin_trace(ServerNode, ClientNode, Dest) ->
960    ?line {ok, _} =
961	ttb:tracer([ServerNode,ClientNode],[{file, Dest}]),
962    ?line ttb:p(all, call),
963    ?line ttb:tp(server, received, []),
964    ?line ttb:tp(client, put, []),
965    ?line ttb:tp(client, get, []).
966
967begin_trace_local(ServerNode, ClientNode, Dest) ->
968    ?line {ok, _} =
969	ttb:tracer([ServerNode,ClientNode],[{file, Dest}]),
970    ?line ttb:p(all, call),
971    ?line ttb:tpl(server, received, []),
972    ?line ttb:tpl(client, put, []),
973    ?line ttb:tpl(client, get, []).
974
975check_size(N, Dest, Output, ServerNode, ClientNode) ->
976    begin_trace(ServerNode, ClientNode, Dest),
977    case Dest of
978        {local, _} ->
979            ttb_helper:msgs_ip(N);
980        _ ->
981	    ttb_helper:msgs(N)
982    end,
983    {_, D} = ttb:stop([fetch, return_fetch_dir]),
984    ttb:format(D, [{out, Output}, {handler, simple_call_handler()}]),
985    {ok, Ret} = file:consult(Output),
986    check_output(N+1, Ret).
987
988check_output(Expected, Ret)
989  when length(Ret) =:= Expected -> ok;
990check_output(Expected, Ret) ->
991    io:format("~p~n",[Ret]),
992    io:format("Expected ~p got ~p ~n",[Expected, length(Ret)]),
993    Expected = length(Ret).
994
995fetch_when_no_option_given(suite) ->
996    [];
997fetch_when_no_option_given(doc) ->
998    ["Fetch when no option given"];
999fetch_when_no_option_given(Config) when is_list(Config) ->
1000    ?line {ServerNode, ClientNode} = start_client_and_server(),
1001    ?line {ok, Privdir} = file:get_cwd(),
1002    ?line [] = filelib:wildcard(filename:join(Privdir,"ttb_upload_temptest*")),
1003    begin_trace(ServerNode, ClientNode, ?FNAME),
1004    ?line ttb_helper:msgs(4),
1005    ?line stopped = ttb:stop(),
1006    ?line [_] = filelib:wildcard(filename:join(Privdir,"ttb_upload_temptest*")).
1007
1008fetch_when_no_option_given(cleanup,_Config) ->
1009    ?line stop_client_and_server().
1010
1011
1012basic_ttb_run_ip_port(suite) ->
1013    [];
1014basic_ttb_run_ip_port(doc) ->
1015    ["Basic ttb run ip port"];
1016basic_ttb_run_ip_port(Config) when is_list(Config) ->
1017    ?line {ServerNode, ClientNode} = start_client_and_server(),
1018    ?line check_size(1, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode),
1019    ?line check_size(2, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode),
1020    ?line check_size(10, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode).
1021basic_ttb_run_ip_port(cleanup,_Config) ->
1022    ?line stop_client_and_server().
1023
1024basic_ttb_run_file_port(suite) ->
1025    [];
1026basic_ttb_run_file_port(doc) ->
1027    ["Basic ttb run file port"];
1028basic_ttb_run_file_port(Config) when is_list(Config) ->
1029    ?line {ServerNode, ClientNode} = start_client_and_server(),
1030    ?line check_size(1, ?FNAME, ?OUTPUT, ServerNode, ClientNode),
1031    ?line check_size(2, ?FNAME, ?OUTPUT, ServerNode, ClientNode),
1032    ?line check_size(10, ?FNAME, ?OUTPUT, ServerNode, ClientNode).
1033basic_ttb_run_file_port(cleanup,_Config) ->
1034    ?line stop_client_and_server().
1035
1036return_fetch_dir_implies_fetch(suite) ->
1037    [];
1038return_fetch_dir_implies_fetch(doc) ->
1039    ["Return_fetch_dir implies fetch"];
1040return_fetch_dir_implies_fetch(Config) when is_list(Config) ->
1041    ?line {ServerNode, ClientNode} = start_client_and_server(),
1042    ?line begin_trace(ServerNode, ClientNode, ?FNAME),
1043    ?line ttb_helper:msgs(2),
1044    ?line {_,_} = ttb:stop([return_fetch_dir]).
1045return_fetch_dir_implies_fetch(cleanup,_Config) ->
1046    ?line stop_client_and_server().
1047
1048logfile_name_in_fetch_dir(suite) ->
1049    [];
1050logfile_name_in_fetch_dir(doc) ->
1051    ["Logfile name in fetch dir"];
1052logfile_name_in_fetch_dir(Config) when is_list(Config) ->
1053    ?line {ServerNode, ClientNode} = start_client_and_server(),
1054    ?line begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
1055    ?line {_,Dir} = ttb:stop([return_fetch_dir]),
1056    ?line P1 = lists:nth(3, string:lexemes(filename:basename(Dir), "_")),
1057    ?line P2 = hd(string:lexemes(P1, "-")),
1058    ?line _File = P2.
1059logfile_name_in_fetch_dir(cleanup,_Config) ->
1060    ?line stop_client_and_server().
1061
1062upload_to_my_logdir(suite) ->
1063    [];
1064upload_to_my_logdir(doc) ->
1065    ["Upload to my logdir"];
1066upload_to_my_logdir(Config) when is_list(Config) ->
1067    ?line {ServerNode, ClientNode} = start_client_and_server(),
1068    ?line {ok, _} =
1069	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
1070    ?line {stopped,_} = ttb:stop([return_fetch_dir, {fetch_dir, ?DIRNAME}]),
1071    ?line true = filelib:is_file(?DIRNAME),
1072    ?line [] = filelib:wildcard("ttb_upload_"++?FNAME).
1073upload_to_my_logdir(cleanup,_Config) ->
1074    ?line stop_client_and_server().
1075
1076upload_to_my_existing_logdir(suite) ->
1077    [];
1078upload_to_my_existing_logdir(doc) ->
1079    ["Upload to my existing logdir"];
1080upload_to_my_existing_logdir(Config) when is_list(Config) ->
1081    ?line {ServerNode, ClientNode} = start_client_and_server(),
1082    ?line ok = file:make_dir(?DIRNAME),
1083    ?line {ok, _} =
1084	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
1085    ?line {error,_,_} = (catch ttb:stop([return_fetch_dir, {fetch_dir, ?DIRNAME}])),
1086    ?line {stopped,_} = ttb:stop(return_fetch_dir).
1087upload_to_my_existing_logdir(cleanup,_Config) ->
1088    ?line stop_client_and_server().
1089
1090fetch_with_options_not_as_list(suite) ->
1091    [];
1092fetch_with_options_not_as_list(doc) ->
1093    ["Fetch with options not as list"];
1094fetch_with_options_not_as_list(Config) when is_list(Config) ->
1095    ?line {ServerNode, ClientNode} = start_client_and_server(),
1096    ?line {ok, _} =
1097	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
1098    ?line {stopped, D} = ttb:stop(return_fetch_dir),
1099    ?line false = filelib:is_file(?OUTPUT),
1100    ?line ttb:format(D, {out, ?OUTPUT}),
1101    ?line true = filelib:is_file(?OUTPUT).
1102fetch_with_options_not_as_list(cleanup,_Config) ->
1103    ?line stop_client_and_server().
1104
1105error_when_formatting_multiple_files_4393(suite) ->
1106    [];
1107error_when_formatting_multiple_files_4393(doc) ->
1108    ["Error when formatting multiple files"];
1109error_when_formatting_multiple_files_4393(Config) when is_list(Config) ->
1110    ?line {ServerNode, ClientNode} = start_client_and_server(),
1111    ?line begin_trace(ServerNode, ClientNode, ?FNAME),
1112    ?line ttb_helper:msgs(2),
1113    ?line {_, Dir} = ttb:stop(return_fetch_dir),
1114    ?line Files = [filename:join(Dir, atom_to_list(ServerNode) ++ "-" ++ ?FNAME),
1115		   filename:join(Dir, atom_to_list(ClientNode) ++ "-" ++ ?FNAME)],
1116    ?line ok = ttb:format(Files).
1117error_when_formatting_multiple_files_4393(cleanup,_Config) ->
1118    ?line stop_client_and_server().
1119
1120format_on_trace_stop(suite) ->
1121    [];
1122format_on_trace_stop(doc) ->
1123    ["Format on trace stop"];
1124format_on_trace_stop(Config) when is_list(Config) ->
1125    ?line {ServerNode, ClientNode} = start_client_and_server(),
1126    ?line begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
1127    ?line ttb_helper:msgs_ip(2),
1128    ?line file:delete("HANDLER_OK"),
1129    ?line {_,_} = ttb:stop([fetch, return_fetch_dir, {format, {handler, marking_call_handler()}}]),
1130    ?line true = filelib:is_file("HANDLER_OK"),
1131    ?line ok = file:delete("HANDLER_OK").
1132format_on_trace_stop(cleanup,_Config) ->
1133    ?line stop_client_and_server().
1134
1135%% The following three tests are for the issue "fixes fetch fail when nodes on the same host
1136%% have different cwd"
1137trace_to_remote_files_on_localhost_with_different_pwd(suite) ->
1138    [];
1139trace_to_remote_files_on_localhost_with_different_pwd(doc) ->
1140    ["Trace to remote files on localhost with different pwd"];
1141trace_to_remote_files_on_localhost_with_different_pwd(Config) when is_list(Config) ->
1142    ?line {ok, OldDir} = file:get_cwd(),
1143    ?line ok = file:set_cwd(".."),
1144    ?line {ServerNode, ClientNode} = start_client_and_server(),
1145    ?line check_size(2, ?FNAME, ?OUTPUT, ServerNode, ClientNode),
1146    ?line ok = file:set_cwd(OldDir).
1147trace_to_remote_files_on_localhost_with_different_pwd(cleanup,_Config) ->
1148    ?line stop_client_and_server().
1149
1150trace_to_local_files_on_localhost_with_different_pwd(suite) ->
1151    [];
1152trace_to_local_files_on_localhost_with_different_pwd(doc) ->
1153    ["Trace to local files on localhost with different pwd"];
1154trace_to_local_files_on_localhost_with_different_pwd(Config) when is_list(Config) ->
1155    ?line {ok, OldDir} = file:get_cwd(),
1156    ?line ok = file:set_cwd(".."),
1157    ?line {ServerNode, ClientNode} = start_client_and_server(),
1158    ?line check_size(2, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode),
1159    ?line ok = file:set_cwd(OldDir).
1160trace_to_local_files_on_localhost_with_different_pwd(cleanup,_Config) ->
1161    ?line stop_client_and_server().
1162
1163trace_to_remote_files_on_localhost_with_different_pwd_abs(suite) ->
1164    [];
1165trace_to_remote_files_on_localhost_with_different_pwd_abs(doc) ->
1166    ["Trace to remote files on localhost with different pwd abs"];
1167trace_to_remote_files_on_localhost_with_different_pwd_abs(Config) when is_list(Config) ->
1168    ?line {ok, OldDir} = file:get_cwd(),
1169    ?line ok = file:set_cwd(".."),
1170    ?line {ok, Path} = file:get_cwd(),
1171    ?line {ServerNode, ClientNode} = start_client_and_server(),
1172    ?line File = filename:join(Path, ?FNAME),
1173    ?line check_size(2, File, ?OUTPUT, ServerNode, ClientNode),
1174    ?line ok = file:set_cwd(OldDir).
1175trace_to_remote_files_on_localhost_with_different_pwd_abs(cleanup,_Config) ->
1176    ?line stop_client_and_server().
1177
1178%% Trace is not affected by changes of cwd on control node or remote nodes during tracing
1179%% (three tests)
1180changing_cwd_on_control_node(suite) ->
1181    [];
1182changing_cwd_on_control_node(doc) ->
1183    ["Changing cwd on control node during tracing is safe"];
1184changing_cwd_on_control_node(Config) when is_list(Config) ->
1185    ?line {ok, OldDir} = file:get_cwd(),
1186    ?line {ServerNode, ClientNode} = start_client_and_server(),
1187    ?line begin_trace(ServerNode, ClientNode, ?FNAME),
1188    ?line NumMsgs = 3,
1189    ?line ttb_helper:msgs(NumMsgs),
1190    ?line ok = file:set_cwd(".."),
1191    ?line ttb_helper:msgs(NumMsgs),
1192    ?line {_, D} = ttb:stop([fetch, return_fetch_dir]),
1193    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
1194    ?line {ok, Ret} = file:consult(?OUTPUT),
1195    check_output(2*(NumMsgs + 1),Ret),
1196    ok = file:set_cwd(OldDir).
1197changing_cwd_on_control_node(cleanup,_Config) ->
1198    ?line stop_client_and_server().
1199
1200changing_cwd_on_control_node_with_local_trace(suite) ->
1201    [];
1202changing_cwd_on_control_node_with_local_trace(doc) ->
1203    ["Changing cwd on control node during local tracing is safe"];
1204changing_cwd_on_control_node_with_local_trace(Config) when is_list(Config) ->
1205    {ok, OldDir} = file:get_cwd(),
1206    {ServerNode, ClientNode} = start_client_and_server(),
1207    begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
1208    NumMsgs = 3,
1209    ttb_helper:msgs_ip(NumMsgs),
1210    ok = file:set_cwd(".."),
1211    ttb_helper:msgs_ip(NumMsgs),
1212    {_, D} = ttb:stop([fetch, return_fetch_dir]),
1213    ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
1214    {ok, Ret} = file:consult(?OUTPUT),
1215    Expected = 2*(NumMsgs + 1),
1216    check_output(Expected, Ret),
1217    ok = file:set_cwd(OldDir).
1218changing_cwd_on_control_node_with_local_trace(cleanup,_Config) ->
1219    ?line stop_client_and_server().
1220
1221changing_cwd_on_remote_node(suite) ->
1222    [];
1223changing_cwd_on_remote_node(doc) ->
1224    ["Changing cwd on remote node during tracing is safe"];
1225changing_cwd_on_remote_node(Config) when is_list(Config) ->
1226    ?line {ServerNode, ClientNode} = start_client_and_server(),
1227    ?line begin_trace(ServerNode, ClientNode, ?FNAME),
1228    ?line NumMsgs = 2,
1229    ?line ttb_helper:msgs(NumMsgs),
1230    ?line ok = rpc:call(ClientNode, file, set_cwd, [".."]),
1231    ?line ttb_helper:msgs(NumMsgs),
1232    ?line {_, D} = ttb:stop([fetch, return_fetch_dir]),
1233    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
1234    ?line {ok, Ret} = file:consult(?OUTPUT),
1235    check_output(2*(NumMsgs + 1),Ret).
1236changing_cwd_on_remote_node(cleanup,_Config) ->
1237    ?line stop_client_and_server().
1238
1239one_command_trace_setup(suite) ->
1240    [];
1241one_command_trace_setup(doc) ->
1242    ["One command trace setup"];
1243one_command_trace_setup(Config) when is_list(Config) ->
1244    ?line {ServerNode, ClientNode} = start_client_and_server(),
1245    ?line ttb:start_trace([ClientNode, ServerNode],
1246			  [{server, received, '_', []},
1247			   {client, put, 1, []},
1248			   {client, get, '_', []}],
1249			  {all, call},
1250			  [{file, ?FNAME}]),
1251    ?line ttb_helper:msgs(2),
1252    ?line {_, D} = ttb:stop(return_fetch_dir),
1253    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
1254    ?line {ok, Ret} = file:consult(?OUTPUT),
1255    ?line 5 = length(Ret).
1256one_command_trace_setup(cleanup,_Config) ->
1257    ?line stop_client_and_server().
1258
1259dbg_style_fetch(suite) ->
1260    [];
1261dbg_style_fetch(doc) ->
1262    ["Dbg style fetch"];
1263dbg_style_fetch(Config) when is_list(Config) ->
1264    ?line {ServerNode, ClientNode} = start_client_and_server(),
1265    ?line DirSize = length(element(2, file:list_dir("."))),
1266    ?line ttb:start_trace([ClientNode, ServerNode],
1267			  [{server, received, '_', []},
1268			   {client, put, 1, []},
1269			   {client, get, '_', []}],
1270			  {all, call},
1271			  [{shell, only}]),
1272    ?line DirSize = length(element(2, file:list_dir("."))),
1273    ?line ttb_helper:msgs(2),
1274    ?line DirSize = length(element(2, file:list_dir("."))),
1275    ?line stopped, ttb:stop(format),
1276    %%+1 -> ttb_last_trace
1277    ?line true = (DirSize + 1 == length(element(2, file:list_dir(".")))),
1278    ?line {ok,[{all, [{matched,_,_}, {matched,_,_}]}]} =
1279	ttb:start_trace([ClientNode, ServerNode],
1280			[{server, received, '_', []},
1281			 {client, put, 1, []},
1282			 {client, get, '_', []}],
1283			{all, call},
1284			[{shell, only}]),
1285    ?line ttb:stop().
1286dbg_style_fetch(cleanup,_Config) ->
1287    ?line stop_client_and_server().
1288
1289shell_tracing_init(suite) ->
1290    [];
1291shell_tracing_init(doc) ->
1292    ["Shell tracing init"];
1293shell_tracing_init(Config) when is_list(Config) ->
1294    ?line {ServerNode, ClientNode} = start_client_and_server(),
1295    ?line ttb:tracer([ClientNode, ServerNode], shell),
1296    ?line ttb:stop(),
1297    ?line ttb:tracer([ClientNode, ServerNode],
1298		     [{file, {local, ?FNAME}}, shell]),
1299    ?line ttb:stop(),
1300    ?line local_client_required_on_shell_tracing =
1301	try  ttb:tracer([ClientNode, ServerNode],[{file, ?FNAME}, shell])
1302	catch
1303	    exit:local_client_required_on_shell_tracing ->
1304		local_client_required_on_shell_tracing
1305	end.
1306shell_tracing_init(cleanup,_Config) ->
1307    ?line stop_client_and_server().
1308
1309only_one_state_for_format_handler(suite) ->
1310    [];
1311only_one_state_for_format_handler(doc) ->
1312    ["Only one state for format handler"];
1313only_one_state_for_format_handler(Config) when is_list(Config) ->
1314    ?line {ServerNode, ClientNode} = start_client_and_server(),
1315    ?line begin_trace_local(ServerNode, ClientNode, ?FNAME),
1316    ?line ttb_helper:msgs(2),
1317    ?line {_, D} = ttb:stop([return_fetch_dir]),
1318    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, counter_call_handler()}]),
1319    ?line {ok, Ret} = file:consult(?OUTPUT),
1320    ?line [5] = Ret.
1321only_one_state_for_format_handler(cleanup,_Config) ->
1322    ?line stop_client_and_server().
1323
1324only_one_state_with_default_format_handler(suite) ->
1325    [];
1326only_one_state_with_default_format_handler(doc) ->
1327    ["Only one state with default format handler"];
1328only_one_state_with_default_format_handler(Config) when is_list(Config) ->
1329    ?line {ServerNode, ClientNode} = start_client_and_server(),
1330    ?line begin_trace_local(ServerNode, ClientNode, ?FNAME),
1331    ?line ttb_helper:msgs(2),
1332    ?line {_, D} = ttb:stop([return_fetch_dir]),
1333    ?line ttb:format(D, [{out, ?OUTPUT}]),
1334    ?line true = filelib:is_file(?OUTPUT).
1335only_one_state_with_default_format_handler(cleanup,_Config) ->
1336    ?line stop_client_and_server().
1337
1338only_one_state_with_initial_format_handler(suite) ->
1339    [];
1340only_one_state_with_initial_format_handler(doc) ->
1341    ["Only one state with initial format handler"];
1342only_one_state_with_initial_format_handler(Config) when is_list(Config) ->
1343    ?line {ServerNode, ClientNode} = start_client_and_server(),
1344    ?line {ok, _} =
1345	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}, {handler, counter_call_handler()}]),
1346    ?line ttb:p(all, call),
1347    ?line ttb:tpl(server, received, []),
1348    ?line ttb:tpl(client, put, []),
1349    ?line ttb:tpl(client, get, []),
1350    ?line ttb_helper:msgs(2),
1351    ?line {_, D} = ttb:stop([return_fetch_dir]),
1352    ?line ttb:format(D, [{out, ?OUTPUT}]),
1353    ?line {ok, Ret} = file:consult(?OUTPUT),
1354    ?line [5] = Ret.
1355only_one_state_with_initial_format_handler(cleanup,_Config) ->
1356    ?line stop_client_and_server().
1357
1358run_trace_with_shortcut(Shortcut, Ret, F) ->
1359    ?line {ServerNode, ClientNode} = start_client_and_server(),
1360    ?line {ok, _} =
1361	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
1362    ?line ttb:p(all, call),
1363    ?line ttb:F(client, put, Shortcut),
1364    ?line ttb_helper:msgs(2),
1365    ?line {_, D} = ttb:stop([return_fetch_dir]),
1366    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, ret_caller_call_handler()}]),
1367    ?line {ok, Ret} =file:consult(?OUTPUT),
1368    ?line stop_client_and_server().
1369
1370fun_for(return) ->
1371    {codestr, "fun(_) -> return_trace() end"};
1372fun_for(msg_false) ->
1373    {codestr, "fun(_) -> message(false) end"}.
1374
1375run_trace_with_shortcut1(suite) ->
1376    [];
1377run_trace_with_shortcut1(doc) ->
1378    ["Run trace with shortcut 1"];
1379run_trace_with_shortcut1(Config) when is_list(Config) ->
1380    ?line run_trace_with_shortcut(caller, [ok,ok], tp),
1381    ?line run_trace_with_shortcut(caller, [ok,ok], tpl).
1382run_trace_with_shortcut1(cleanup,_Config) ->
1383    ?line stop_client_and_server().
1384
1385run_trace_with_shortcut2(suite) ->
1386    [];
1387run_trace_with_shortcut2(doc) ->
1388    ["Run trace with shortcut 2"];
1389run_trace_with_shortcut2(Config) when is_list(Config) ->
1390    ?line run_trace_with_shortcut(return, [ok,ok], tp),
1391    ?line run_trace_with_shortcut(return, [ok,ok], tpl).
1392run_trace_with_shortcut2(cleanup,_Config) ->
1393    ?line stop_client_and_server().
1394
1395run_trace_with_shortcut3(suite) ->
1396    [];
1397run_trace_with_shortcut3(doc) ->
1398    ["Run trace with shortcut 3"];
1399run_trace_with_shortcut3(Config) when is_list(Config) ->
1400    ?line run_trace_with_shortcut(fun_for(return), [ok,ok], tp),
1401    ?line run_trace_with_shortcut(fun_for(return), [ok,ok], tpl).
1402run_trace_with_shortcut3(cleanup,_Config) ->
1403    ?line stop_client_and_server().
1404
1405run_trace_with_shortcut4(suite) ->
1406    [];
1407run_trace_with_shortcut4(doc) ->
1408    ["Run trace with shortcut 4"];
1409run_trace_with_shortcut4(Config) when is_list(Config) ->
1410    ?line run_trace_with_shortcut(fun_for(msg_false), [], tp),
1411    ?line run_trace_with_shortcut(fun_for(msg_false), [], tpl).
1412run_trace_with_shortcut4(cleanup,_Config) ->
1413    ?line stop_client_and_server().
1414
1415cant_specify_local_and_flush(suite) ->
1416    [];
1417cant_specify_local_and_flush(doc) ->
1418    ["Can't specify local and flush"];
1419cant_specify_local_and_flush(Config) when is_list(Config) ->
1420    ?line {ServerNode, ClientNode} = start_client_and_server(),
1421    ?line flush_unsupported_with_ip_trace_port =
1422	try ttb:tracer([ServerNode, ClientNode],
1423		       [{flush, 1000}, {file, {local, ?FNAME}}])
1424	catch
1425	    exit:flush_unsupported_with_ip_trace_port ->
1426		flush_unsupported_with_ip_trace_port
1427	end.
1428cant_specify_local_and_flush(cleanup,_Config) ->
1429    ?line stop_client_and_server().
1430
1431trace_sorted_by_default(suite) ->
1432    [];
1433trace_sorted_by_default(doc) ->
1434    ["Trace sorted by default"];
1435trace_sorted_by_default(Config) when is_list(Config) ->
1436    ?line {ServerNode, ClientNode} = start_client_and_server(),
1437    ?line begin_trace_local(ServerNode, ClientNode, ?FILE),
1438    ?line ttb_helper:msgs(2),
1439    ?line {_, D} = ttb:stop([return_fetch_dir]),
1440    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, node_call_handler()}, {disable_sort, false}]),
1441    {ok, Ret} = file:consult(?OUTPUT),
1442    ?line [ClientNode,ServerNode,ClientNode,ServerNode,ServerNode] = Ret.
1443trace_sorted_by_default(cleanup,_Config) ->
1444    ?line stop_client_and_server().
1445
1446disable_sorting(suite) ->
1447    [];
1448disable_sorting(doc) ->
1449    ["Disable sorting"];
1450disable_sorting(Config) when is_list(Config) ->
1451    ?line {ServerNode, ClientNode} = start_client_and_server(),
1452    ?line begin_trace_local(ServerNode, ClientNode, ?FILE),
1453    ?line ttb_helper:msgs(2),
1454    ?line {_, D} = ttb:stop([return_fetch_dir]),
1455    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, node_call_handler()}, {disable_sort, true}]),
1456    {ok, Ret} = file:consult(?OUTPUT),
1457    ?line [ClientNode,ClientNode,ServerNode,ServerNode,ServerNode] = Ret.
1458disable_sorting(cleanup,_Config) ->
1459    ?line stop_client_and_server().
1460
1461%% -----------------------------------------------------------------------------
1462%% tests for autoresume of tracing
1463%% -----------------------------------------------------------------------------
1464
1465trace_resumed_after_node_restart(suite) ->
1466    [];
1467trace_resumed_after_node_restart(doc) ->
1468    ["Test trace resumed after node restart, trace to files on remote node."];
1469trace_resumed_after_node_restart(Config) when is_list(Config) ->
1470    ?line {ServerNode, ClientNode} = start_client_and_server(),
1471    ?line begin_trace_with_resume(ServerNode, ClientNode, ?FNAME),
1472    ?line logic(2,6,file).
1473trace_resumed_after_node_restart(cleanup,_Config) ->
1474    ?line stop_client_and_server().
1475
1476trace_resumed_after_node_restart_ip(suite) ->
1477    [];
1478trace_resumed_after_node_restart_ip(doc) ->
1479    ["Test trace resumed after node restart, trace via tcp/ip to local node."];
1480trace_resumed_after_node_restart_ip(Config) when is_list(Config) ->
1481    ?line {ServerNode, ClientNode} = start_client_and_server(),
1482    ?line begin_trace_with_resume(ServerNode, ClientNode, {local, ?FNAME}),
1483    ?line logic(2,6,local).
1484trace_resumed_after_node_restart_ip(cleanup,_Config) ->
1485    ?line stop_client_and_server().
1486
1487trace_resumed_after_node_restart_wrap(suite) ->
1488    [];
1489trace_resumed_after_node_restart_wrap(doc) ->
1490    ["Test trace resumed after node restart, wrap option."];
1491trace_resumed_after_node_restart_wrap(Config) when is_list(Config) ->
1492    ?line {ServerNode, ClientNode} = start_client_and_server(),
1493    ?line begin_trace_with_resume(ServerNode, ClientNode, {wrap, ?FNAME, 10, 4}),
1494    ?line logic(1,4,file).
1495trace_resumed_after_node_restart_wrap(cleanup,_Config) ->
1496    ?line stop_client_and_server().
1497
1498trace_resumed_after_node_restart_wrap_mult(suite) ->
1499    [];
1500trace_resumed_after_node_restart_wrap_mult(doc) ->
1501    ["Test trace resumed after node restart, wrap option, multiple files."];
1502trace_resumed_after_node_restart_wrap_mult(Config) when is_list(Config) ->
1503    ?line {ServerNode, ClientNode} = start_client_and_server(),
1504    ?line begin_trace_with_resume(ServerNode, ClientNode, {wrap, ?FNAME, 10, 4}),
1505    ?line logic(20,8,file).
1506trace_resumed_after_node_restart_wrap_mult(cleanup,_Config) ->
1507    ?line stop_client_and_server().
1508
1509logic(N, M, TracingType) ->
1510    helper_msgs(N, TracingType),
1511    test_server:stop_node(ttb_helper:get_node(client)),
1512    timer:sleep(2500),
1513    ?line {ok,_ClientNode} = test_server:start_node(client,slave,[]),
1514    ct:log("client started",[]),
1515    ?line ok = ttb_helper:c(code, add_paths, [code:get_path()]),
1516    ct:log("paths added",[]),
1517    ?line ttb_helper:c(client, init, []),
1518    ct:log("client initiated",[]),
1519    ?line helper_msgs(N, TracingType),
1520    ct:log("helper msgs sent and flushed",[]),
1521    ?line {_, D} = ttb:stop([return_fetch_dir]),
1522    ct:log("stopped ~p",[D]),
1523    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, ret_caller_call_handler2()}]),
1524    ct:log("formatted ~p",[{D,?OUTPUT}]),
1525    ?line {ok, Ret} = file:consult(?OUTPUT),
1526    ct:log("consulted: ~p",[Ret]),
1527    check_output(M,Ret).
1528
1529begin_trace_with_resume(ServerNode, ClientNode, Dest) ->
1530    ?line {ok, _} = ttb:tracer([ServerNode,ClientNode], [{file, Dest}, resume]),
1531    ?line ttb:p(all, [call, timestamp]),
1532    ?line ttb:tp(server, received, []),
1533    ?line ttb:tp(client, put, []),
1534    ?line ttb:tp(client, get, []).
1535
1536ret_caller_call_handler2() ->
1537    {fun(A, {trace_ts, _, call, _, _} ,_,_) -> io:format(A, "ok.~n", []);
1538	(_, _, _, _) -> ok end, []}.
1539
1540helper_msgs(N, TracingType) ->
1541    case TracingType of
1542        local ->
1543            ttb_helper:msgs_ip(N);
1544        _ ->
1545            ttb_helper:msgs(N)
1546    end.
1547
1548priv_dir(Conf) ->
1549    %% Due to problem with long paths on windows => creating a new
1550    %% priv_dir under data_dir
1551    Dir = filename:absname(filename:join(?config(data_dir, Conf),priv_dir)),
1552    filelib:ensure_dir(filename:join(Dir,"*")),
1553    Dir.
1554
1555clean_priv_dir(Config) ->
1556    PrivDir = priv_dir(Config),
1557    case filelib:is_dir(PrivDir) of
1558        true -> rm(PrivDir);
1559        false -> ok
1560    end.
1561
1562rm(This) ->
1563    case filelib:is_dir(This) of
1564        true ->
1565            {ok,Files} = file:list_dir(This),
1566            [rm(filename:join(This,F)) || F <- Files],
1567	    file:del_dir(This);
1568        false ->
1569	    file:delete(This)
1570    end.
1571