1%%%-------------------------------------------------------------------
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2017-2019. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20-module(ssl_dist_bench_SUITE).
21
22-behaviour(ct_suite).
23
24-include_lib("common_test/include/ct_event.hrl").
25-include_lib("public_key/include/public_key.hrl").
26
27%% CT meta
28-export([suite/0, all/0, groups/0,
29         init_per_suite/1, end_per_suite/1,
30         init_per_group/2, end_per_group/2,
31         init_per_testcase/2, end_per_testcase/2]).
32
33%% Test cases
34-export(
35   [setup/1,
36    roundtrip/1,
37    sched_utilization/1,
38    throughput_0/1,
39    throughput_64/1,
40    throughput_1024/1,
41    throughput_4096/1,
42    throughput_16384/1,
43    throughput_65536/1,
44    throughput_262144/1,
45    throughput_1048576/1]).
46
47%% Debug
48-export([payload/1, roundtrip_runner/3, setup_runner/3, throughput_runner/4]).
49
50%%%-------------------------------------------------------------------
51
52suite() -> [{ct_hooks, [{ts_install_cth, [{nodenames, 2}]}]}].
53
54all() ->
55    [{group, ssl},
56     {group, crypto},
57     {group, plain}].
58
59groups() ->
60    [{ssl, all_groups()},
61     {crypto, all_groups()},
62     {plain, all_groups()},
63     %%
64     {setup, [{repeat, 1}], [setup]},
65     {roundtrip, [{repeat, 1}], [roundtrip]},
66     {sched_utilization,[{repeat, 1}], [sched_utilization]},
67     {throughput, [{repeat, 1}],
68      [throughput_0,
69       throughput_64,
70       throughput_1024,
71       throughput_4096,
72       throughput_16384,
73       throughput_65536,
74       throughput_262144,
75       throughput_1048576]}].
76
77all_groups() ->
78    [{group, setup},
79     {group, roundtrip},
80     {group, throughput},
81     {group, sched_utilization}
82    ].
83
84init_per_suite(Config) ->
85    Digest = sha1,
86    ECCurve = secp521r1,
87    TLSVersion = 'tlsv1.2',
88    TLSCipher = {ecdhe_ecdsa,aes_128_cbc,sha256,sha256},
89    %%
90    Node = node(),
91    try
92        Node =/= nonode@nohost orelse
93            throw({skipped,"Node not distributed"}),
94        verify_node_src_addr(),
95        {supported, SSLVersions} =
96            lists:keyfind(supported, 1, ssl:versions()),
97        lists:member(TLSVersion, SSLVersions) orelse
98            throw(
99              {skipped,
100               "SSL does not support " ++ term_to_string(TLSVersion)}),
101        lists:member(ECCurve, ssl:eccs(TLSVersion)) orelse
102            throw(
103              {skipped,
104               "SSL does not support " ++ term_to_string(ECCurve)}),
105        lists:member(TLSCipher, ssl:cipher_suites()) orelse
106            throw(
107              {skipped,
108               "SSL does not support " ++ term_to_string(TLSCipher)})
109    of
110        _ ->
111            PrivDir = proplists:get_value(priv_dir, Config),
112            %%
113            [_, HostA] = split_node(Node),
114            NodeAName = ?MODULE_STRING ++ "_node_a",
115            NodeAString = NodeAName ++ "@" ++ HostA,
116            NodeAConfFile = filename:join(PrivDir, NodeAString ++ ".conf"),
117            NodeA = list_to_atom(NodeAString),
118            %%
119            ServerNode = ssl_bench_test_lib:setup(dist_server),
120            [_, HostB] = split_node(ServerNode),
121            NodeBName = ?MODULE_STRING ++ "_node_b",
122            NodeBString = NodeBName ++ "@" ++ HostB,
123            NodeBConfFile = filename:join(PrivDir, NodeBString ++ ".conf"),
124            NodeB = list_to_atom(NodeBString),
125            %%
126            CertOptions =
127                [{digest, Digest},
128                 {key, {namedCurve, ECCurve}}],
129            RootCert =
130                public_key:pkix_test_root_cert(
131                  ?MODULE_STRING ++ " ROOT CA", CertOptions),
132            SSLConf =
133                [{verify, verify_peer},
134                 {versions, [TLSVersion]},
135                 {ciphers, [TLSCipher]}],
136            ServerConf =
137                [{fail_if_no_peer_cert, true},
138                 {verify_fun,
139                  {fun inet_tls_dist:verify_client/3,[]}}
140                 | SSLConf],
141            ClientConf = SSLConf,
142            %%
143            write_node_conf(
144              NodeAConfFile, NodeA, ServerConf, ClientConf,
145              CertOptions, RootCert),
146            write_node_conf(
147              NodeBConfFile, NodeB, ServerConf, ClientConf,
148              CertOptions, RootCert),
149            %%
150            [{node_a_name, NodeAName},
151             {node_a, NodeA},
152             {node_a_dist_args,
153              "-proto_dist inet_tls "
154              "-ssl_dist_optfile " ++ NodeAConfFile ++ " "},
155             {node_b_name, NodeBName},
156             {node_b, NodeB},
157             {node_b_dist_args,
158              "-proto_dist inet_tls "
159              "-ssl_dist_optfile " ++ NodeBConfFile ++ " "},
160             {server_node, ServerNode}
161             |Config]
162    catch
163        throw:Result ->
164            Result
165    end.
166
167end_per_suite(Config) ->
168    ServerNode = proplists:get_value(server_node, Config),
169    slave:stop(ServerNode).
170
171init_per_group(ssl, Config) ->
172    [{ssl_dist, true}, {ssl_dist_prefix, "SSL"}|Config];
173init_per_group(crypto, Config) ->
174    [{ssl_dist, false}, {ssl_dist_prefix, "Crypto"},
175     {ssl_dist_args,
176      "-proto_dist inet_crypto"}
177     |Config];
178init_per_group(plain, Config) ->
179    [{ssl_dist, false}, {ssl_dist_prefix, "Plain"}|Config];
180init_per_group(_GroupName, Config) ->
181    Config.
182
183end_per_group(_GroupName, _Config) ->
184    ok.
185
186init_per_testcase(_Func, Conf) ->
187    Conf.
188
189end_per_testcase(_Func, _Conf) ->
190    ok.
191
192-define(COUNT, 400).
193
194%%%-------------------------------------------------------------------
195%%% CommonTest API helpers
196
197verify_node_src_addr() ->
198    Msg = "Hello, world!",
199    {ok,Host} = inet:gethostname(),
200    {ok,DstAddr} = inet:getaddr(Host, inet),
201    {ok,Socket} = gen_udp:open(0, [{active,false}]),
202    {ok,Port} = inet:port(Socket),
203    ok = gen_udp:send(Socket, DstAddr, Port, Msg),
204    case gen_udp:recv(Socket, length(Msg) + 1, 1000) of
205        {ok,{DstAddr,Port,Msg}} ->
206            ok;
207        {ok,{SrcAddr,Port,Msg}} ->
208            throw({skipped,
209                   "Src and dst address mismatch: " ++
210                       term_to_string(SrcAddr) ++ " =:= " ++
211                       term_to_string(DstAddr)});
212        Weird ->
213            error(Weird)
214    end.
215
216write_node_conf(
217  ConfFile, Node, ServerConf, ClientConf, CertOptions, RootCert) ->
218    [Name,Host] = split_node(Node),
219    Conf =
220        public_key:pkix_test_data(
221          #{root => RootCert,
222            peer =>
223                [{extensions,
224                  [
225                   #'Extension'{
226                      extnID = ?'id-ce-subjectAltName',
227                      extnValue = [{dNSName, Host}],
228                      critical = true},
229                   #'Extension'{
230                      extnID = ?'id-ce-subjectAltName',
231                      extnValue =
232                          [{directoryName,
233                            {rdnSequence,
234                             [[#'AttributeTypeAndValue'{
235                                  type = ?'id-at-commonName',
236                                  value =
237                                      {utf8String,
238                                       unicode:characters_to_binary(
239                                         Name, utf8)
240                                      }
241                                 }]]}}],
242                      critical = true}
243                  ]} | CertOptions]}),
244    NodeConf =
245        [{server, ServerConf ++ Conf}, {client, ClientConf ++ Conf}],
246    {ok, Fd} = file:open(ConfFile, [write]),
247    ok = file:change_mode(ConfFile, 8#400),
248    io:format(Fd, "~p.~n", [NodeConf]),
249    ok = file:close(Fd).
250
251split_node(Node) ->
252    string:split(atom_to_list(Node), "@").
253
254%%%-------------------------------------------------------------------
255%%% Test cases
256
257%%-----------------------
258%% Connection setup speed
259
260setup(Config) ->
261    run_nodepair_test(fun setup/5, Config).
262
263setup(A, B, Prefix, HA, HB) ->
264    Rounds = 50,
265    [] = ssl_apply(HA, erlang, nodes, []),
266    [] = ssl_apply(HB, erlang, nodes, []),
267    {SetupTime, CycleTime} =
268        ssl_apply(HA, fun () -> setup_runner(A, B, Rounds) end),
269    ok = ssl_apply(HB, fun () -> setup_wait_nodedown(A, 10000) end),
270    %% [] = ssl_apply(HA, erlang, nodes, []),
271    %% [] = ssl_apply(HB, erlang, nodes, []),
272    SetupSpeed = round((Rounds*1000000*1000) / SetupTime),
273    CycleSpeed = round((Rounds*1000000*1000) / CycleTime),
274    _ = report(Prefix++" Setup", SetupSpeed, "setups/1000s"),
275    report(Prefix++" Setup Cycle", CycleSpeed, "cycles/1000s").
276
277%% Runs on node A against rex in node B
278setup_runner(A, B, Rounds) ->
279    StartTime = start_time(),
280    SetupTime = setup_loop(A, B, 0, Rounds),
281    {microseconds(SetupTime), microseconds(elapsed_time(StartTime))}.
282
283setup_loop(_A, _B, T, 0) ->
284    T;
285setup_loop(A, B, T, N) ->
286    StartTime = start_time(),
287    [N,A] = [N|rpc:block_call(B, erlang, nodes, [])],
288    Time = elapsed_time(StartTime),
289    [N,B] = [N|erlang:nodes()],
290    Mref = erlang:monitor(process, {rex,B}),
291    true = net_kernel:disconnect(B),
292    receive
293        {'DOWN',Mref,process,_,_} ->
294            [] = erlang:nodes(),
295            setup_loop(A, B, Time + T, N - 1)
296    end.
297
298setup_wait_nodedown(A, Time) ->
299    ok = net_kernel:monitor_nodes(true),
300    case nodes() of
301        [] ->
302            ok;
303        [A] ->
304            receive
305                {nodedown,A} ->
306                    ok;
307                Unexpected ->
308                    {error,{unexpected,Unexpected}}
309            after Time ->
310                    {error,timeout}
311            end
312    end.
313
314
315%%----------------
316%% Roundtrip speed
317
318roundtrip(Config) ->
319    run_nodepair_test(fun roundtrip/5, Config).
320
321roundtrip(A, B, Prefix, HA, HB) ->
322    Rounds = 40000,
323    [] = ssl_apply(HA, erlang, nodes, []),
324    [] = ssl_apply(HB, erlang, nodes, []),
325    ok = ssl_apply(HA, net_kernel, allow, [[B]]),
326    ok = ssl_apply(HB, net_kernel, allow, [[A]]),
327    Time = ssl_apply(HA, fun () -> roundtrip_runner(A, B, Rounds) end),
328    [B] = ssl_apply(HA, erlang, nodes, []),
329    [A] = ssl_apply(HB, erlang, nodes, []),
330    Speed = round((Rounds*1000000) / Time),
331    report(Prefix++" Roundtrip", Speed, "pings/s").
332
333%% Runs on node A and spawns a server on node B
334roundtrip_runner(A, B, Rounds) ->
335    ClientPid = self(),
336    [A] = rpc:call(B, erlang, nodes, []),
337    ServerPid =
338        erlang:spawn(
339          B,
340          fun () -> roundtrip_server(ClientPid, Rounds) end),
341    ServerMon = erlang:monitor(process, ServerPid),
342    microseconds(
343      roundtrip_client(ServerPid, ServerMon, start_time(), Rounds)).
344
345roundtrip_server(_Pid, 0) ->
346    ok;
347roundtrip_server(Pid, N) ->
348    receive
349        N ->
350            Pid ! N,
351            roundtrip_server(Pid, N-1)
352    end.
353
354roundtrip_client(_Pid, Mon, StartTime, 0) ->
355    Time = elapsed_time(StartTime),
356    receive
357        {'DOWN', Mon, _, _, normal} ->
358            Time;
359        {'DOWN', Mon, _, _, Other} ->
360            exit(Other)
361    end;
362roundtrip_client(Pid, Mon, StartTime, N) ->
363    Pid ! N,
364    receive
365        N ->
366            roundtrip_client(Pid, Mon, StartTime, N - 1)
367    end.
368
369%%---------------------------------------
370%% Scheduler utilization at constant load
371
372
373sched_utilization(Config) ->
374    run_nodepair_test(
375      fun(A, B, Prefix, HA, HB) ->
376              sched_utilization(A, B, Prefix, HA, HB, proplists:get_value(ssl_dist, Config))
377      end, Config).
378
379sched_utilization(A, B, Prefix, HA, HB, SSL) ->
380    [] = ssl_apply(HA, erlang, nodes, []),
381    [] = ssl_apply(HB, erlang, nodes, []),
382    {ClientMsacc, ServerMsacc, Msgs} =
383        ssl_apply(HA, fun () -> sched_util_runner(A, B, SSL) end),
384    [B] = ssl_apply(HA, erlang, nodes, []),
385    [A] = ssl_apply(HB, erlang, nodes, []),
386    msacc:print(ClientMsacc),
387    msacc:print(ServerMsacc),
388    ct:pal("Got ~p busy_dist_port msgs",[length(Msgs)]),
389    ct:log("Stats of B from A: ~p",
390           [ssl_apply(HA, net_kernel, node_info, [B])]),
391    ct:log("Stats of A from B: ~p",
392           [ssl_apply(HB, net_kernel, node_info, [A])]),
393    SchedUtilClient =
394        round(10000 * msacc:stats(system_runtime,ClientMsacc) /
395                  msacc:stats(system_realtime,ClientMsacc)),
396    SchedUtilServer =
397        round(10000 * msacc:stats(system_runtime,ServerMsacc) /
398                  msacc:stats(system_realtime,ServerMsacc)),
399    Verdict =
400        case Msgs of
401            [] ->
402                "";
403            _ ->
404                " ???"
405        end,
406    {comment, ClientComment} =
407        report(Prefix ++ " Sched Utilization Client" ++ Verdict,
408               SchedUtilClient, "/100 %" ++ Verdict),
409    {comment, ServerComment} =
410        report(Prefix++" Sched Utilization Server" ++ Verdict,
411               SchedUtilServer, "/100 %" ++ Verdict),
412    {comment, "Client " ++ ClientComment ++ ", Server " ++ ServerComment}.
413
414%% Runs on node A and spawns a server on node B
415%% We want to avoid getting busy_dist_port as it hides the true SU usage
416%% of the receiver and sender.
417sched_util_runner(A, B, true) ->
418    sched_util_runner(A, B, 250);
419sched_util_runner(A, B, false) ->
420    sched_util_runner(A, B, 250);
421sched_util_runner(A, B, Senders) ->
422    Payload = payload(5),
423    [A] = rpc:call(B, erlang, nodes, []),
424    ServerPids =
425        [erlang:spawn_link(
426           B, fun () -> throughput_server() end)
427         || _ <- lists:seq(1, Senders)],
428    ServerMsacc =
429        erlang:spawn(
430          B,
431          fun() ->
432                  receive
433                      {start,Pid} ->
434                          msacc:start(10000),
435                          receive
436                              {done,Pid} ->
437                                  Pid ! {self(),msacc:stats()}
438                          end
439                  end
440          end),
441    erlang:system_monitor(self(),[busy_dist_port]),
442    %% We spawn 250 senders which should mean that we
443    %% have a load of 250 msgs/msec
444    [spawn_link(
445       fun() ->
446               throughput_client(Pid, Payload)
447       end) || Pid <- ServerPids],
448    %%
449    receive after 1000 -> ok end,
450    ServerMsacc ! {start,self()},
451    msacc:start(10000),
452    ClientMsaccStats = msacc:stats(),
453    receive after 1000 -> ok end,
454    ServerMsacc ! {done,self()},
455    ServerMsaccStats = receive {ServerMsacc,Stats} -> Stats end,
456    %%
457    {ClientMsaccStats,ServerMsaccStats, flush()}.
458
459flush() ->
460    receive
461        M ->
462            [M | flush()]
463    after 0 ->
464            []
465    end.
466
467throughput_server() ->
468    receive _ -> ok end,
469    receive _ -> ok end,
470    receive _ -> ok end,
471    receive _ -> ok end,
472    receive _ -> ok end,
473    receive _ -> ok end,
474    receive _ -> ok end,
475    receive _ -> ok end,
476    receive _ -> ok end,
477    receive _ -> ok end,
478    throughput_server().
479
480throughput_client(Pid, Payload) ->
481    Pid ! Payload,
482    receive after 1 -> throughput_client(Pid, Payload) end.
483
484%%-----------------
485%% Throughput speed
486
487throughput_0(Config) ->
488    run_nodepair_test(
489      fun (A, B, Prefix, HA, HB) ->
490              throughput(A, B, Prefix, HA, HB, 500000, 0)
491      end, Config).
492
493throughput_64(Config) ->
494    run_nodepair_test(
495      fun (A, B, Prefix, HA, HB) ->
496              throughput(A, B, Prefix, HA, HB, 500000, 64)
497      end, Config).
498
499throughput_1024(Config) ->
500    run_nodepair_test(
501      fun (A, B, Prefix, HA, HB) ->
502              throughput(A, B, Prefix, HA, HB, 100000, 1024)
503      end, Config).
504
505throughput_4096(Config) ->
506    run_nodepair_test(
507      fun (A, B, Prefix, HA, HB) ->
508              throughput(A, B, Prefix, HA, HB, 50000, 4096)
509      end, Config).
510
511throughput_16384(Config) ->
512    run_nodepair_test(
513      fun (A, B, Prefix, HA, HB) ->
514              throughput(A, B, Prefix, HA, HB, 10000, 16384)
515      end, Config).
516
517throughput_65536(Config) ->
518    run_nodepair_test(
519      fun (A, B, Prefix, HA, HB) ->
520              throughput(A, B, Prefix, HA, HB, 2000, 65536)
521      end, Config).
522
523throughput_262144(Config) ->
524    run_nodepair_test(
525      fun (A, B, Prefix, HA, HB) ->
526              throughput(A, B, Prefix, HA, HB, 500, 262144)
527      end, Config).
528
529throughput_1048576(Config) ->
530    run_nodepair_test(
531      fun (A, B, Prefix, HA, HB) ->
532              throughput(A, B, Prefix, HA, HB, 200, 1048576)
533      end, Config).
534
535throughput(A, B, Prefix, HA, HB, Packets, Size) ->
536    [] = ssl_apply(HA, erlang, nodes, []),
537    [] = ssl_apply(HB, erlang, nodes, []),
538    #{time := Time,
539      client_dist_stats := ClientDistStats,
540      client_msacc_stats := ClientMsaccStats,
541      client_prof := ClientProf,
542      server_msacc_stats := ServerMsaccStats,
543      server_prof := ServerProf,
544      server_gc_before := Server_GC_Before,
545      server_gc_after := Server_GC_After} =
546        ssl_apply(HA, fun () -> throughput_runner(A, B, Packets, Size) end),
547    [B] = ssl_apply(HA, erlang, nodes, []),
548    [A] = ssl_apply(HB, erlang, nodes, []),
549    ClientMsaccStats =:= undefined orelse
550        msacc:print(ClientMsaccStats),
551    io:format("ClientDistStats: ~p~n", [ClientDistStats]),
552    Overhead =
553        50 % Distribution protocol headers (empirical) (TLS+=54)
554        + byte_size(erlang:term_to_binary([0|<<>>])), % Benchmark overhead
555    Bytes = Packets * (Size + Overhead),
556    io:format("~w bytes, ~.4g s~n", [Bytes,Time/1000000]),
557    SizeString = integer_to_list(Size),
558    ClientMsaccStats =:= undefined orelse
559        report(
560          Prefix ++ " Sender_RelativeCoreLoad_" ++ SizeString,
561          round(msacc:stats(system_runtime, ClientMsaccStats)
562                * 1000000 / Bytes),
563          "ps/byte"),
564    ServerMsaccStats =:= undefined orelse
565        begin
566            report(
567              Prefix ++ " Receiver_RelativeCoreLoad_" ++ SizeString,
568              round(msacc:stats(system_runtime, ServerMsaccStats)
569                    * 1000000 / Bytes),
570              "ps/byte"),
571            msacc:print(ServerMsaccStats)
572        end,
573    io:format("******* ClientProf:~n", []), prof_print(ClientProf),
574    io:format("******* ServerProf:~n", []), prof_print(ServerProf),
575    io:format("******* Server GC Before:~n~p~n", [Server_GC_Before]),
576    io:format("******* Server GC After:~n~p~n", [Server_GC_After]),
577    Speed = round((Bytes * 1000000) / (1024 * Time)),
578    report(Prefix ++ " Throughput_" ++ SizeString, Speed, "kB/s").
579
580%% Runs on node A and spawns a server on node B
581throughput_runner(A, B, Rounds, Size) ->
582    Payload = payload(Size),
583    [A] = rpc:call(B, erlang, nodes, []),
584    ClientPid = self(),
585    ServerPid =
586        erlang:spawn_opt(
587          B,
588          fun () -> throughput_server(ClientPid, Rounds) end,
589          [{message_queue_data, off_heap}]),
590    ServerMon = erlang:monitor(process, ServerPid),
591    msacc_available() andalso
592        begin
593            msacc:stop(),
594            msacc:reset(),
595            msacc:start(),
596            ok
597        end,
598    prof_start(),
599    #{time := Time} = Result =
600        throughput_client(ServerPid, ServerMon, Payload, Rounds),
601    prof_stop(),
602    MsaccStats =
603        case msacc_available() of
604            true ->
605                MStats = msacc:stats(),
606                msacc:stop(),
607                MStats;
608            false ->
609                undefined
610        end,
611    Prof = prof_end(),
612    [{_Node,Socket}] = dig_dist_node_sockets(),
613    DistStats = inet:getstat(Socket),
614    Result#{time := microseconds(Time),
615            client_dist_stats => DistStats,
616            client_msacc_stats => MsaccStats,
617            client_prof => Prof}.
618
619dig_dist_node_sockets() ->
620    [case DistCtrl of
621         {_Node,Socket} = NodeSocket when is_port(Socket) ->
622             NodeSocket;
623         {Node,DistCtrlPid} when is_pid(DistCtrlPid) ->
624             [{links,DistCtrlLinks}] = process_info(DistCtrlPid, [links]),
625             case [S || S <- DistCtrlLinks, is_port(S)] of
626                 [Socket] ->
627                     {Node,Socket};
628                 [] ->
629                     [{monitors,[{process,DistSenderPid}]}] =
630                         process_info(DistCtrlPid, [monitors]),
631                     [{links,DistSenderLinks}] =
632                         process_info(DistSenderPid, [links]),
633                     [Socket] = [S || S <- DistSenderLinks, is_port(S)],
634                     {Node,Socket}
635             end
636     end || DistCtrl <- erlang:system_info(dist_ctrl)].
637
638
639throughput_server(Pid, N) ->
640    GC_Before = get_server_gc_info(),
641    %% dbg:tracer(port, dbg:trace_port(file, "throughput_server_gc.log")),
642    %% dbg:p(TLSDistReceiver, garbage_collection),
643    msacc_available() andalso
644        begin
645            msacc:stop(),
646            msacc:reset(),
647            msacc:start(),
648            ok
649        end,
650    prof_start(),
651    throughput_server_loop(Pid, GC_Before, N).
652
653throughput_server_loop(_Pid, GC_Before, 0) ->
654    prof_stop(),
655    MsaccStats =
656        case msacc_available() of
657            true ->
658                msacc:stop(),
659                MStats = msacc:stats(),
660                msacc:reset(),
661                MStats;
662            false ->
663                undefined
664        end,
665    Prof = prof_end(),
666    %% dbg:flush_trace_port(),
667    exit(#{server_msacc_stats => MsaccStats,
668           server_prof => Prof,
669           server_gc_before => GC_Before,
670           server_gc_after => get_server_gc_info()});
671throughput_server_loop(Pid, GC_Before, N) ->
672    receive
673        Msg ->
674            case Msg of
675                {Pid, N, _} ->
676                    throughput_server_loop(Pid, GC_Before, N - 1);
677                Other ->
678                    erlang:error({self(),?FUNCTION_NAME,Other})
679            end
680    end.
681
682get_server_gc_info() ->
683    case whereis(ssl_connection_sup_dist) of
684        undefined ->
685            undefined;
686        SupPid ->
687            [{_Id,TLSDistReceiver,_Type,_Modules}|_] =
688                supervisor:which_children(SupPid),
689            erlang:process_info(
690              TLSDistReceiver, [garbage_collection,garbage_collection_info])
691    end.
692
693throughput_client(Pid, Mon, Payload, N) ->
694    throughput_client_loop(Pid, Mon, Payload, N, start_time()).
695
696throughput_client_loop(_Pid, Mon, _Payload, 0, StartTime) ->
697    receive
698        {'DOWN', Mon, _, _, #{} = Result} ->
699            Result#{time => elapsed_time(StartTime)};
700        {'DOWN', Mon, _, _, Other} ->
701            exit(Other)
702    end;
703throughput_client_loop(Pid, Mon, Payload, N, StartTime) ->
704    Pid ! {self(), N, Payload},
705    throughput_client_loop(Pid, Mon, Payload, N - 1, StartTime).
706
707
708-define(prof, none). % none | cprof | eprof
709
710-if(?prof =:= cprof).
711prof_start() ->
712    cprof:stop(),
713    cprof:start(),
714    ok.
715-elif(?prof =:= eprof).
716prof_start() ->
717    catch eprof:stop(),
718    {ok,_} = eprof:start(),
719    profiling = eprof:start_profiling(processes()),
720    ok.
721-elif(?prof =:= none).
722prof_start() ->
723    ok.
724-endif.
725
726-if(?prof =:= cprof).
727prof_stop() ->
728    cprof:pause(),
729    ok.
730-elif(?prof =:= eprof).
731prof_stop() ->
732    _ = eprof:stop_profiling(),
733    ok.
734-elif(?prof =:= none).
735prof_stop() ->
736    ok.
737-endif.
738
739-if(?prof =:= cprof).
740prof_end() ->
741    Prof = cprof:analyse(),
742    cprof:stop(),
743    Prof.
744-elif(?prof =:= eprof).
745prof_end() ->
746    eprof:dump_data().
747-elif(?prof =:= none).
748prof_end() ->
749    [].
750-endif.
751
752-if(?prof =:= cprof).
753prof_print(Prof) ->
754    io:format("~p.~n", [Prof]).
755-elif(?prof =:= eprof).
756prof_print(Dump) ->
757    eprof:analyze(undefined, total, [], Dump).
758-elif(?prof =:= none).
759prof_print([]) ->
760    ok.
761-endif.
762
763%%%-------------------------------------------------------------------
764%%% Test cases helpers
765
766run_nodepair_test(TestFun, Config) ->
767    A = proplists:get_value(node_a, Config),
768    B = proplists:get_value(node_b, Config),
769    Prefix = proplists:get_value(ssl_dist_prefix, Config),
770    HA = start_ssl_node_a(Config),
771    HB = start_ssl_node_b(Config),
772    try TestFun(A, B, Prefix, HA, HB)
773    after
774        stop_ssl_node_a(HA),
775        stop_ssl_node_b(HB, Config),
776        ok
777    end.
778
779ssl_apply(Handle, M, F, Args) ->
780    case ssl_dist_test_lib:apply_on_ssl_node(Handle, M, F, Args) of
781        {'EXIT',Reason} ->
782            error(Reason);
783        Result ->
784            Result
785    end.
786
787ssl_apply(Handle, Fun) ->
788    case ssl_dist_test_lib:apply_on_ssl_node(Handle, Fun) of
789        {'EXIT',Reason} ->
790            error(Reason);
791        Result ->
792            Result
793    end.
794
795start_ssl_node_a(Config) ->
796    Name = proplists:get_value(node_a_name, Config),
797    Args = get_node_args(node_a_dist_args, Config),
798    ssl_dist_test_lib:start_ssl_node(Name, Args).
799
800start_ssl_node_b(Config) ->
801    Name = proplists:get_value(node_b_name, Config),
802    Args = get_node_args(node_b_dist_args, Config),
803    ServerNode = proplists:get_value(server_node, Config),
804    rpc:call(
805      ServerNode, ssl_dist_test_lib, start_ssl_node, [Name, Args]).
806
807stop_ssl_node_a(HA) ->
808    ssl_dist_test_lib:stop_ssl_node(HA).
809
810stop_ssl_node_b(HB, Config) ->
811    ServerNode = proplists:get_value(server_node, Config),
812    rpc:call(ServerNode, ssl_dist_test_lib, stop_ssl_node, [HB]).
813
814get_node_args(Tag, Config) ->
815    case proplists:get_value(ssl_dist, Config) of
816        true ->
817            proplists:get_value(Tag, Config);
818        false ->
819            proplists:get_value(ssl_dist_args, Config, "")
820    end.
821
822
823
824payload(Size) ->
825    iolist_to_binary(
826      [case Size bsr 8 of
827           0 ->
828               [];
829           Blocks ->
830               payload(Blocks, create_binary(256))
831       end | create_binary(Size band 255)]).
832%%
833payload(0, _) ->
834    [];
835payload(Blocks, Block) ->
836    Half = payload(Blocks bsr 1, Block),
837    [Half, Half |
838     if
839         Blocks band 1 =:= 1 ->
840             Block;
841         true ->
842             []
843     end].
844
845create_binary(Size) ->
846    create_binary(Size, <<>>).
847%%
848create_binary(0, Bin) ->
849    Bin;
850create_binary(Size, Bin) ->
851    NextSize = Size - 1,
852    create_binary(NextSize, <<Bin/binary, NextSize>>).
853
854start_time() ->
855    erlang:system_time().
856
857elapsed_time(StartTime) ->
858    erlang:system_time() - StartTime.
859
860microseconds(Time) ->
861    erlang:convert_time_unit(Time, native, microsecond).
862
863report(Name, Value, Unit) ->
864    ct:pal("~s: ~w ~s", [Name, Value, Unit]),
865    ct_event:notify(
866      #event{
867         name = benchmark_data,
868         data = [{value, Value}, {suite, "ssl_dist"}, {name, Name}]}),
869    {comment, term_to_string(Value) ++ " " ++ Unit}.
870
871term_to_string(Term) ->
872    unicode:characters_to_list(
873      io_lib:write(Term, [{encoding, unicode}])).
874
875msacc_available() ->
876    msacc:available().
877