1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2012-2018. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20
21
22%%
23-module(httpd_bench_SUITE).
24-compile(export_all).
25
26-include_lib("common_test/include/ct.hrl").
27-include_lib("common_test/include/ct_event.hrl").
28-include_lib("public_key/include/public_key.hrl").
29-include_lib("kernel/include/file.hrl").
30
31-define(remote_host, "NETMARKS_REMOTE_HOST").
32-define(LF, [10]).
33-define(CR, [13]).
34-define(CRLF, ?CR ++ ?LF).
35
36%%--------------------------------------------------------------------
37%% Common Test interface functions -----------------------------------
38%%--------------------------------------------------------------------
39suite() ->
40    [{timetrap, {minutes, 1}},
41     {ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}].
42
43all() ->
44    [
45     {group, http_dummy},
46     {group, http_inets},
47     {group, http_nginx},
48     {group, https_inets},
49     {group, https_dummy},
50     {group, https_nginx},
51     {group, http_dummy_keep_alive},
52     {group, http_inets_keep_alive},
53     {group, http_nginx_keep_alive},
54     {group, https_inets_keep_alive},
55     {group, https_dummy_keep_alive},
56     {group, https_nginx_keep_alive}
57    ].
58
59groups() ->
60    [
61     {http_dummy, [],  client_tests()},
62     {http_inets, [],   client_tests()},
63     {http_nginx, [],   client_tests()},
64     {https_dummy, [],  client_tests()},
65     {https_inets, [],  client_tests()},
66     {https_nginx, [],  client_tests()},
67     {http_dummy_keep_alive, [],  client_tests()},
68     {http_inets_keep_alive, [],  client_tests()},
69     {http_nginx_keep_alive, [],  client_tests()},
70     {https_dummy_keep_alive, [], client_tests()},
71     {https_inets_keep_alive, [], client_tests()},
72     {https_nginx_keep_alive, [], client_tests()}
73    ].
74
75
76client_tests() ->
77    [wget_small,
78     erl_dummy_small,
79     httpc_small,
80     wget_big,
81     erl_dummy_big,
82     httpc_big
83    ].
84
85init_per_suite(Config) ->
86    try
87	{Node, Host} = setup(Config, node()),
88	init_ssl(Config),
89	[{iter, 10}, {server_node, Node}, {server_host, Host} | Config]
90    catch E:R:ST ->
91            ct:pal("~p:~p:~p",[E,R,ST]),
92	    {skipped, "Benchmark machines only"}
93    end.
94
95end_per_suite(_Config) ->
96    [application:stop(App) || App <- [asn1, crypto, public_key, ssl, inets]].
97
98init_per_group(Group, Config) when Group == http_dummy_keep_alive;
99				   Group == https_dummy_keep_alive;
100				   Group == http_inets_keep_alive;
101				   Group == https_inets_keep_alive;
102				   Group == http_nginx_keep_alive;
103				   Group == https_nginx_keep_alive ->
104    Version = http_version(Group),
105    start_web_server(Group,
106		     [{keep_alive, true},
107		      {reuse_sessions, false},
108		      {http_version, Version},
109		      {http_opts,[{version, Version}]},
110		      {http_headers, [{"connection", "keep-alive"}]},
111		      {httpc_opts, [{keep_alive_timeout, 1500},
112				    {max_keep_alive_length, ?config(iter, Config)}]}
113		      | Config]);
114init_per_group(Group, Config)  when Group == http_dummy;
115				    Group == https_dummy;
116				    Group == http_inets;
117				    Group == https_inets;
118				    Group == http_nginx;
119				    Group == https_nginx ->
120    Version = http_version(Group),
121    start_web_server(Group,
122		     [{keep_alive, false},
123		      {reuse_sessions, false},
124		      {http_version, Version},
125		      {http_headers, [{"connection", "close"}]},
126		      {http_opts,[{version, Version}]},
127		      {httpc_opts, [{keep_alive_timeout, 0}, {max_keep_alive_length, 0}]}
128		      | Config]);
129
130
131init_per_group(_, Config) ->
132    Config.
133
134end_per_group(Group, Config) ->
135    stop_web_server(Group, Config).
136
137init_per_testcase(TestCase, Config) when TestCase == httpc_small;
138					 TestCase == httpc_big
139					 ->
140    Opts = ?config(httpc_opts, Config),
141    inets:start(httpc, [{profile, TestCase}, {socket_opts, [{nodelay, true}]}]),
142    httpc:set_options(Opts, TestCase),
143    [{profile, TestCase} | proplists:delete(profile, Config)];
144
145init_per_testcase(_, Config) ->
146    Config.
147end_per_testcase(TestCase, _Config) when TestCase == httpc_small;
148					 TestCase == httpc_big ->
149    ok = inets:stop(httpc, TestCase);
150end_per_testcase(_TestCase, Config) ->
151    Config.
152%%--------------------------------------------------------------------
153%% Test Cases --------------------------------------------------------
154%%--------------------------------------------------------------------
155
156erl_dummy_small(Config) when is_list(Config) ->
157    {ok, Result} = run_test(httpd_lib_client, "1k_file", Config),
158    notify(Result, Config, "erl_1k_file").
159
160erl_dummy_big(Config)  when is_list(Config) ->
161    {ok, Result} = run_test(httpd_lib_client, "1M_file", Config),
162    notify(Result, Config, "erl_1M_file").
163
164wget_small(Config) when is_list(Config) ->
165    {ok, Result} = run_test(wget_client, "1k_file", Config),
166    notify(Result, Config, "wget_1k_file").
167
168wget_big(Config)  when is_list(Config) ->
169    {ok, Result} = run_test(wget_client, "1M_file", Config),
170    notify(Result, Config, "wget_1M_file").
171
172httpc_small(Config) when is_list(Config) ->
173    {ok, Result} = run_test(httpc_client, "1k_file", Config),
174    notify(Result, Config, "httpc_1k_file").
175
176httpc_big(Config)  when is_list(Config) ->
177    {ok, Result} = run_test(httpc_client, "1M_file", Config),
178    notify(Result, Config, "httpc_1M_file").
179
180%%--------------------------------------------------------------------
181%% Internal functions ------------------------------------------------
182%%--------------------------------------------------------------------
183
184%%--------------------------------------------------------------------
185%% Report benchmark results  ------------------------------------------------
186%%--------------------------------------------------------------------
187
188notify({TestPerSec, _MBps}, Config, Suffix) ->
189    Name = lists:concat([?config(protocol,Config), " ",
190			 server_name(Config, [dummy_pid, httpd_pid, nginx_port]),
191			 "", Suffix]),
192    ct:comment("~p tps", [TestPerSec]),
193    ct_event:notify(#event{name = benchmark_data,
194			   data=[{value, TestPerSec},
195				 {suite, ?MODULE},
196				 {name, Name}]}),
197			 ok.
198%%--------------------------------------------------------------------
199%% Setup erlang nodes  ------------------------------------------------
200%%--------------------------------------------------------------------
201
202server_name(Config, [Server | Rest]) ->
203    case proplists:get_value(Server, Config) of
204	undefined ->
205	    server_name(Config, Rest);
206	_ ->
207	    server_name(Server)
208    end.
209
210server_name(httpd_pid) ->
211   "inets";
212server_name(nginx_port) ->
213    "nginx";
214server_name(dummy_pid) ->
215    "erlang".
216
217setup(_Config, nonode@nohost) ->
218    exit(dist_not_enabled);
219setup(_Config, _LocalNode) ->
220    Host = case os:getenv(?remote_host) of
221	       false ->
222		   {ok, This} = inet:gethostname(),
223		   This;
224	       RemHost ->
225		   RemHost
226	   end,
227    Node = list_to_atom("inets_perf_server@" ++ Host),
228    SlaveArgs = case init:get_argument(pa) of
229	       {ok, PaPaths} ->
230		   lists:append([" -pa " ++ P || [P] <- PaPaths]);
231	       _ -> []
232	   end,
233    Prog =
234	case os:find_executable("erl") of
235	    false -> "erl";
236	    P -> P
237	end,
238    case net_adm:ping(Node) of
239	pong -> ok;
240	pang ->
241	    {ok, Node} = slave:start(Host, inets_perf_server, SlaveArgs, no_link, Prog)
242    end,
243    Path = code:get_path(),
244    true = rpc:call(Node, code, set_path, [Path]),
245    [ensure_started(Node, App) || App <- [asn1, crypto, public_key, ssl, inets]],
246    [ensure_started(node(), App) || App <- [asn1, crypto, public_key, ssl, inets]],
247    (Node =:= node()) andalso restrict_schedulers(client),
248    {Node, Host}.
249
250ensure_started(Node, App) ->
251     ok = rpc:call(Node, application, ensure_started, [App]).
252
253
254restrict_schedulers(Type) ->
255    %% We expect this to run on 8 core machine
256    Extra0 = 1,
257    Extra =  if (Type =:= server) -> -Extra0; true -> Extra0 end,
258    Scheds = erlang:system_info(schedulers),
259    erlang:system_flag(schedulers_online, (Scheds div 2) + Extra).
260
261%%--------------------------------------------------------------------
262%% Setup TLS input files  ------------------------------------------------
263%%--------------------------------------------------------------------
264
265init_ssl(Config) ->
266    DDir = ?config(data_dir, Config),
267    PDir = ?config(priv_dir, Config),
268    {ok, _} = make_certs:all(DDir,
269			     PDir).
270cert_opts(Config) ->
271    ClientCaCertFile = filename:join([?config(priv_dir, Config),
272				      "client", "cacerts.pem"]),
273    ClientCertFile = filename:join([?config(priv_dir, Config),
274				    "client", "cert.pem"]),
275    ServerCaCertFile = filename:join([?config(priv_dir, Config),
276				      "server", "cacerts.pem"]),
277    ServerCertFile = filename:join([?config(priv_dir, Config),
278				    "server", "cert.pem"]),
279    ServerKeyFile = filename:join([?config(priv_dir, Config),
280			     "server", "key.pem"]),
281    ClientKeyFile = filename:join([?config(priv_dir, Config),
282				   "client", "key.pem"]),
283    [{server_verification_opts, [{reuseaddr, true},
284				 {cacertfile, ServerCaCertFile},
285				 {ciphers, ["ECDHE-RSA-AES256-GCM-SHA384"]},
286				 {certfile, ServerCertFile}, {keyfile, ServerKeyFile}]},
287     {client_verification_opts, [
288				 %%{verify, verify_peer},
289				 {cacertfile, ClientCaCertFile},
290				 {certfile, ClientCertFile},
291				 {keyfile, ClientKeyFile}]}].
292
293%%--------------------------------------------------------------------
294%% Run clients  ------------------------------------------------
295%%--------------------------------------------------------------------
296
297run_test(Client, File, Config) ->
298    Parent = self(),
299    Pid = spawn(fun() ->
300			receive
301			    go ->
302				Parent ! {self(),
303					  do_runs(Client, [{file, File} | Config])}
304			end
305		end),
306    Pid ! go,
307    receive
308	{Pid,{{tps, Tps}, {mbps, MBps}}} ->
309	    ct:pal("Tps: ~p  Bps~p", [Tps, MBps]),
310	    {ok, {Tps, MBps}}
311    end.
312
313do_runs(Client, Config) ->
314    N = ?config(iter, Config),
315    DataDir = ?config(data_dir, Config),
316    File = ?config(file, Config),
317    Name = filename:join(DataDir, File),
318    Args = ?MODULE:Client(Config),
319    ?MODULE:Client({init, Args}),
320    Run =
321	fun() ->
322		ok = ?MODULE:Client(Args, N)
323	end,
324    {ok, Info} = file:read_file_info(Name, []),
325    Length = Info#file_info.size,
326    {TimeInMicro, _} = timer:tc(Run),
327    ReqPerSecond = (1000000 * N) div TimeInMicro,
328    BytesPerSecond = (1000000 * N * Length) div TimeInMicro,
329    {{tps, ReqPerSecond}, {mbps, BytesPerSecond}}.
330
331
332httpc_client({init, [_, Profile, URL, Headers, HTTPOpts]}) ->
333     %% Make sure pipelining feature will kick in when appropriate.
334    {ok, {{_ ,200, "OK"}, _,_}} = httpc:request(get,{URL, Headers}, HTTPOpts,
335						[{body_format, binary},
336						 {socket_opts, [{nodelay, true}]}], Profile),
337    ct:sleep(1000);
338httpc_client(Config) ->
339    File = ?config(file, Config),
340    Protocol = ?config(protocol, Config),
341    Profile = ?config(profile, Config),
342    URL = (?config(urlfun,Config))(File),
343    Headers =  ?config(http_headers, Config),
344    HTTPOpts = ?config(http_opts, Config),
345    [Protocol, Profile, URL, Headers, HTTPOpts].
346httpc_client(_,0) ->
347    ok;
348httpc_client([Protocol, Profile, URL, Headers, HTTPOpts], N) ->
349    {ok, {{_ ,200,"OK"}, _,_}} = httpc:request(get,{URL, Headers}, HTTPOpts, [{body_format, binary},
350									     {socket_opts, [{nodelay, true}]}], Profile),
351    httpc_client([Protocol, Profile, URL, Headers, HTTPOpts], N-1).
352
353httpd_lib_client({init, [_, Type, Version, Request, Host, Port, Opts]}) ->
354    ok = httpd_test_lib:verify_request(Type, Host,
355     				       Port,
356     				       Opts, node(),
357     				       Request,
358     				       [{statuscode, 200},
359     					{version, Version}], infinity),
360    ct:sleep(1000);
361httpd_lib_client(Config) ->
362    File = ?config(file, Config),
363    KeepAlive = ?config(keep_alive, Config),
364    Host = ?config(server_host, Config),
365    Port = ?config(port, Config),
366    ReuseSession = ?config(reuse_sessions, Config),
367    {Type, Opts} =
368	case ?config(protocol, Config) of
369	    "http" ->
370		{ip_comm, [{active, true}, {mode, binary},{nodelay, true}]};
371	    "https" ->
372		SSLOpts =  proplists:get_value(client_verification_opts, cert_opts(Config)),
373		{ssl, [{active, true}, {mode, binary}, {nodelay, true},
374		       {reuse_sessions, ReuseSession} | SSLOpts]}
375
376	end,
377    Version = ?config(http_version, Config),
378    Request = case KeepAlive of
379		  true ->
380		      http_request("GET /" ++ File ++ " ", Version, Host, {"connection:keep-alive\r\n", ""});
381		  false ->
382		      http_request("GET /" ++ File ++ " ", Version, Host)
383	      end,
384
385    Args = [KeepAlive, Type, Version, Request, Host, Port, Opts],
386    httpd_lib_client(Args, 1),
387    Args.
388
389httpd_lib_client(_, 0) ->
390    ok;
391httpd_lib_client([true, Type, Version, Request, Host, Port, Opts], N) ->
392    ok = httpd_test_lib:verify_request_N(Type, Host,
393					 Port,
394					 Opts, node(),
395					 Request,
396					 [{statuscode, 200},
397					  {version, Version}], infinity, N);
398httpd_lib_client([false, Type, Version, Request, Host, Port, Opts] = List, N) ->
399    ok = httpd_test_lib:verify_request(Type, Host,
400				       Port,
401				       Opts, node(),
402				       Request,
403				       [{statuscode, 200},
404					{version, Version}], infinity),
405    httpd_lib_client(List, N-1).
406
407wget_client({init,_}) ->
408    ok;
409wget_client(Config) ->
410    File = ?config(file, Config),
411    URL = (?config(urlfun,Config))(File),
412    KeepAlive = ?config(keep_alive, Config),
413    PrivDir = ?config(priv_dir, Config),
414    Protocol = ?config(protocol, Config),
415    Iter = ?config(iter, Config),
416    FileName = filename:join(PrivDir, "wget_req"),
417    ProtocolOpts = case Protocol of
418		    "http" ->
419			   [];
420		       "https" ->
421			   proplists:get_value(client_verification_opts, cert_opts(Config))
422		   end,
423    wget_req_file(FileName,URL,Iter),
424    [KeepAlive, FileName, URL, Protocol, ProtocolOpts, Iter].
425wget_client([KeepAlive, WgetFile, _URL, Protocol, ProtocolOpts, _], _) ->
426    process_flag(trap_exit, true),
427    Cmd = wget_N(KeepAlive, WgetFile, Protocol, ProtocolOpts),
428    %%ct:pal("Wget cmd: ~p", [Cmd]),
429    Port = open_port({spawn, Cmd}, [stderr_to_stdout]),
430    wait_for_wget(Port).
431
432
433%%--------------------------------------------------------------------
434%% Start/stop servers  ------------------------------------------------
435%%--------------------------------------------------------------------
436start_web_server(Group, Config) when Group == http_dummy;
437				     Group == http_dummy_keep_alive ->
438    start_dummy("http", Config);
439
440start_web_server(Group, Config) when Group == https_dummy;
441				     Group == https_dummy_keep_alive ->
442    start_dummy("https", Config);
443
444start_web_server(Group, Config) when Group == http_inets;
445				     Group == http_inets_keep_alive ->
446    start_inets("http", [], Config);
447
448start_web_server(Group, Config) when Group == https_inets;
449				     Group == https_inets_keep_alive ->
450    Opts = proplists:get_value(server_verification_opts, cert_opts(Config)),
451    ReuseSessions = ?config(reuse_sessions, Config),
452    SSLConfHttpd = [{socket_type, {essl,
453				   [{nodelay, true}, {reuse_sessions, ReuseSessions} | Opts]}}],
454    start_inets("https", SSLConfHttpd, Config);
455
456start_web_server(Group, Config)  when Group == http_nginx;
457				      Group == http_nginx_keep_alive ->
458    case os:find_executable("nginx") of
459	false ->
460	    {skip, "nginx not found"};
461	_ ->
462	    start_nginx("http",  Config)
463    end;
464
465start_web_server(Group, Config)  when Group == https_nginx;
466				      Group == https_nginx_keep_alive ->
467     case os:find_executable("nginx") of
468	false ->
469	    {skip, "nginx not found"};
470	 _ ->
471	     start_nginx("https",  cert_opts(Config) ++ Config)
472     end.
473
474start_inets(Protocol, ConfHttpd, Config) ->
475    PrivDir = ?config(priv_dir, Config),
476    DataDir = ?config(data_dir, Config),
477    Node = ?config(server_node, Config),
478    Host = ?config(server_host, Config),
479    HTTPVersion = ?config(http_version, Config),
480    Conf = [httpd, [{port,0},
481		    {http_version, HTTPVersion},
482		    {ipfamily, inet},
483		    {server_name, "inets_test"},
484		    {server_root, PrivDir},
485		    {document_root, DataDir},
486		    {keep_alive, ?config(keep_alive, Config)},
487		    {keep_alive_timeout, 360}
488		    | ConfHttpd]],
489    {ok, Pid} = rpc:call(Node, inets, start, Conf),
490    Port = proplists:get_value(port,  rpc:call(Node, httpd, info, [Pid])),
491    F = fun(File) ->
492		lists:concat([Protocol,"://",Host,":",Port,"/",File])
493	end,
494    [{httpd_pid,Pid},{urlfun,F},{protocol,Protocol},{port,Port} | Config].
495
496start_dummy("http"= Protocol, Config) ->
497    HTTPVersion = ?config(http_version, Config),
498    Node = ?config(server_node, Config),
499    %%DataDir= ?config(data_dir, Config),
500    Host = ?config(server_host, Config),
501    Conf = [
502	    %%{big, filename:join(DataDir, "1M_file")},
503	    %%{small, filename:join(DataDir, "1k_file")},
504	    {big, {gen,  crypto:strong_rand_bytes(1000000)}},
505	    {small, {gen,  crypto:strong_rand_bytes(1000)}},
506	    {http_version, HTTPVersion},
507	    {keep_alive,  ?config(keep_alive, Config)}
508	   ],
509    {Pid, Port} = rpc:call(Node, http_test_lib, dummy_server, [ip_comm, inet, [{content_cb, ?MODULE}, {conf, Conf}]]),
510    F = fun(File) ->
511		lists:concat([Protocol,"://",Host,":",Port,"/",File])
512	end,
513    [{dummy_pid,Pid},{urlfun,F},{protocol, Protocol},{port,Port} | Config];
514
515start_dummy("https" = Protocol, Config) ->
516    HTTPVersion = ?config(http_version, Config),
517    Node = ?config(server_node, Config),
518    %% DataDir= ?config(data_dir, Config),
519    Host = ?config(server_host, Config),
520    SSLOpts =  proplists:get_value(server_verification_opts, cert_opts(Config)),
521    Opts = [{active, true}, {nodelay, true}, {reuseaddr, true} | SSLOpts],
522    Conf = [%%{big, filename:join(DataDir, "1M_file")},
523	    %%{small, filename:join(DataDir, "1k_file")},
524	    {big, {gen, crypto:strong_rand_bytes(1000000)}},
525	    {small, {gen, crypto:strong_rand_bytes(1000)}},
526	    {http_version, HTTPVersion},
527	    {keep_alive, ?config(keep_alive, Config)}
528	   ],
529    {Pid, Port} = rpc:call(Node, http_test_lib, dummy_server,
530			   [ssl, inet, [{ssl, Opts}, {content_cb, ?MODULE}, {conf, Conf}]]),
531    F = fun(File) ->
532		lists:concat([Protocol,"://",Host,":",Port,"/",File])
533	end,
534    [{dummy_pid,Pid},{urlfun,F},{protocol,Protocol},{port,Port} | Config].
535
536start_nginx(Protocol, Config) ->
537    PrivDir = ?config(priv_dir, Config),
538    DataDir = ?config(data_dir, Config),
539    Host = ?config(server_host, Config),
540    Port = inet_port(node()),
541
542    ConfFile = filename:join(PrivDir, "nginx.conf"),
543    nginx_conf(ConfFile, [{port, Port}, {protocol, Protocol} | Config]),
544    Cmd = "nginx -c " ++ ConfFile,
545    NginxPort =  open_port({spawn, Cmd}, [{cd, DataDir}, stderr_to_stdout]),
546
547    F = fun(File) ->
548 		lists:concat([Protocol,"://",Host,":",Port,"/",File])
549	end,
550
551    wait_for_nginx_up(Host, Port),
552
553    [{port, Port},{nginx_port, NginxPort},{urlfun,F},{protocol, Protocol} | Config ].
554
555stop_nginx(Config)->
556    PrivDir = ?config(priv_dir, Config),
557    {ok, Bin} = file:read_file(filename:join(PrivDir, "nginx.pid")),
558    Pid = string:strip(binary_to_list(Bin), right, $\n),
559    Cmd = "kill " ++ Pid,
560    os:cmd(Cmd).
561
562stop_web_server(Group, Config) when  Group == http_inets;
563				     Group == http_inets_keep_alive;
564				     Group == https_inets;
565				     Group == https_inets_keep_alive ->
566    ServerNode = ?config(server_node, Config),
567    rpc:call(ServerNode, inets, stop, [httpd, ?config(httpd_pid, Config)]);
568stop_web_server(Group, Config) when  Group == http_dummy;
569				     Group == http_dummy_keep_alive;
570				     Group == https_dummy;
571				     Group == https_dummy_keep_alive ->
572    stop_dummy_server(Config);
573stop_web_server(Group, Config) when  Group == http_nginx;
574				     Group == http_nginx_keep_alive;
575				     Group == https_nginx;
576				     Group == https_nginx_keep_alive ->
577    stop_nginx(Config).
578
579stop_dummy_server(Config) ->
580      case ?config(dummy_pid, Config) of
581	  Pid when is_pid(Pid) ->
582	      exit(Pid, kill);
583	  _ ->
584	      ok
585      end.
586
587%%--------------------------------------------------------------------
588%% Misc  ------------------------------------------------
589%%--------------------------------------------------------------------
590http_request(Request, "HTTP/1.1" = Version, Host, {Headers, Body}) ->
591    Request ++ Version ++ "\r\nhost:" ++ Host ++ "\r\n" ++ Headers ++ "\r\n" ++ Body;
592http_request(Request, Version, _, {Headers, Body}) ->
593    Request ++ Version ++ "\r\n" ++ Headers  ++ "\r\n" ++ Body.
594
595http_request(Request, "HTTP/1.1" = Version, Host) ->
596    Request ++ Version ++ "\r\nhost:" ++ Host  ++ "\r\n\r\n";
597http_request(Request, Version, _) ->
598    Request ++ Version ++ "\r\n\r\n".
599
600http_version(_) ->
601    "HTTP/1.1".
602
603inet_port(Node) ->
604    {Port, Socket} = do_inet_port(Node),
605     rpc:call(Node, gen_tcp, close, [Socket]),
606     Port.
607
608do_inet_port(Node) ->
609    {ok, Socket} = rpc:call(Node, gen_tcp, listen, [0, [{reuseaddr, true}]]),
610    {ok, Port} = rpc:call(Node, inet, port, [Socket]),
611    {Port, Socket}.
612
613%%--------------------------------------------------------------------
614%% Dummy server callbacks  ------------------------------------------------
615%%--------------------------------------------------------------------
616
617handle_request(CB, S, "/1M_file" ++ _, Opts) ->
618    Name = proplists:get_value(big, Opts),
619    KeepAlive = proplists:get_value(keep_alive, Opts),
620    do_handle_request(CB, S, Name, Opts, KeepAlive);
621handle_request(CB, S, "/1k_file" ++ _, Opts) ->
622    Name = proplists:get_value(small, Opts),
623    KeepAlive = proplists:get_value(keep_alive, Opts),
624    do_handle_request(CB, S, Name, Opts, KeepAlive).
625
626do_handle_request(CB, S, Name, Opts, KeepAlive) when is_list(Name) ->
627    Version = proplists:get_value(http_version, Opts),
628    {ok, Fdesc} = file:open(Name, [read, binary]),
629    {ok, Info} = file:read_file_info(Name, []),
630    Length = Info#file_info.size,
631    Response = response_status_line_and_headers(Version, "Content-Length:"
632						++ integer_to_list(Length) ++ ?CRLF, keep_alive(KeepAlive)),
633    CB:send(S, Response),
634    send_file(CB, S, Fdesc);
635do_handle_request(CB, S, {gen, Data}, Opts, KeepAlive) ->
636    Version = proplists:get_value(http_version, Opts),
637    Length = size(Data),
638    Response = response_status_line_and_headers(Version, "Content-Length:"
639						++ integer_to_list(Length) ++ ?CRLF, keep_alive(KeepAlive)),
640    CB:send(S, Response),
641    send_file(CB, S, {gen, Data}).
642
643send_file(CB, S, {gen, Data})  ->
644    CB:send(S, Data);
645    %% ChunkSize = 64*1024,
646    %% case size(Data) of
647    %% 	N when N > ChunkSize ->
648    %% 	    <<Chunk:N/binary, Rest/binary>> = Data,
649    %% 	    %%{Chunk, Rest} = lists:split(N, Data),
650    %% 	    CB:send(S, Chunk),
651    %% 	    send_file(CB, S, {gen, Rest});
652    %% 	_ ->
653    %% 	    CB:send(S, Data)
654    %% end;
655
656send_file(CB, S, FileDesc) ->
657    case file:read(FileDesc, 64*1024) of
658	{ok, Chunk} ->
659	    CB:send(S, Chunk),
660	    send_file(CB, S, FileDesc);
661	eof ->
662	    file:close(FileDesc),
663	    ok
664    end.
665
666response_status_line_and_headers(Version, Headers,  ConnectionHeader) ->
667    StatusLine = [Version, " ", "200 OK", ?CRLF],
668    [StatusLine, Headers, ConnectionHeader, ?CRLF].
669
670keep_alive(true)->
671    "Connection:keep-alive\r\n";
672keep_alive(false) ->
673    "Connection:close\r\n".
674
675handle_http_msg({_Method, RelUri, _, {_, _Headers}, _Body}, Socket, Conf) ->
676    handle_request(connect_cb(Socket), Socket, RelUri, Conf),
677    case proplists:get_value(keep_alive, Conf) of
678	true ->
679	    <<>>;
680	false ->
681	    stop
682    end.
683
684connect_cb({sslsocket, _, _}) ->
685    ssl;
686connect_cb(_) ->
687    gen_tcp.
688
689%%--------------------------------------------------------------------
690%% Setup wget  ------------------------------------------------
691%%--------------------------------------------------------------------
692wget_req_file(FileName, Url, Iter) ->
693    {ok, File} = file:open(FileName, [write]),
694    write_urls(File, Url, Iter).
695
696write_urls(File, Url, 1) ->
697    file:write(File, Url),
698    file:close(File);
699write_urls(File, Url, N) ->
700    file:write(File, Url),
701    file:write(File, "\n"),
702    write_urls(File, Url, N-1).
703
704wait_for_wget(Port) ->
705    receive
706	{Port, {data, _Data}} when is_port(Port) ->
707	    wait_for_wget(Port);
708	{Port, closed} ->
709	    ok;
710	{'EXIT', Port, _Reason} ->
711	    ok
712    end.
713
714wget_N(KeepAlive, WegetFile, "http", _ProtocolOpts) ->
715    "wget -i " ++ WegetFile ++ " " ++ wget_keep_alive(KeepAlive) ++
716	" --no-cache --timeout=120" ;
717wget_N(KeepAlive, WegetFile, "https", ProtocolOpts) ->
718
719    "wget -i " ++ WegetFile ++ " " ++ wget_keep_alive(KeepAlive)
720	++ wget_cert(ProtocolOpts) ++ wget_key(ProtocolOpts)
721	++ wget_cacert(ProtocolOpts) ++
722	" --no-cache --timeout=120".
723
724wget(KeepAlive, URL, "http", _ProtocolOpts) ->
725    "wget " ++ URL ++ " " ++ wget_keep_alive(KeepAlive) ++
726	" --no-cache --timeout=120" ;
727wget(KeepAlive, URL, "https", ProtocolOpts) ->
728
729    "wget " ++ URL ++ " " ++ wget_keep_alive(KeepAlive)
730	++ wget_cert(ProtocolOpts) ++ wget_key(ProtocolOpts)
731	++ wget_cacert(ProtocolOpts) ++
732	" --no-cache --timeout=120".
733
734wget_keep_alive(true)->
735    "";
736wget_keep_alive(false) ->
737   "--no-http-keep-alive ".
738
739wget_cacert(ProtocolOpts) ->
740    "--ca-certificate=" ++ proplists:get_value(cacertfile, ProtocolOpts) ++ " ".
741
742wget_cert(ProtocolOpts) ->
743    "--certificate=" ++ proplists:get_value(certfile, ProtocolOpts) ++ " ".
744
745wget_key(ProtocolOpts) ->
746    "--private-key=" ++ proplists:get_value(keyfile, ProtocolOpts) ++ " ".
747
748%%--------------------------------------------------------------------
749%% Setup nginx  ------------------------------------------------
750%%--------------------------------------------------------------------
751nginx_conf(ConfFile, Config)->
752    Protocol = ?config(protocol, Config),
753    file:write_file(ConfFile,
754		    [format_nginx_conf(nginx_global(Config)),
755		     format_nginx_conf(nginx_events(Config)),
756		     format_nginx_conf(nginx_http(Protocol, Config))]).
757
758format_nginx_conf(Directives) ->
759    lists:map(fun({Key, Value}) ->
760			  io_lib:format("~s ~s;\n", [Key, Value]);
761		     (Str) ->
762			  Str
763		  end, Directives).
764
765
766nginx_global(Config) ->
767    PrivDir = ?config(priv_dir, Config),
768    [{"pid", filename:join(PrivDir, "nginx.pid")},
769     {"error_log",  filename:join(PrivDir, "nginx.pid")},
770     {"worker_processes", "1"}].
771
772nginx_events(_Config) ->
773    ["events {\n",
774     {"worker_connections",  "1024"},
775     "\n}"
776    ].
777
778nginx_http("http", Config) ->
779    PrivDir = ?config(priv_dir, Config),
780    DataDir = ?config(data_dir, Config),
781    Port = ?config(port, Config),
782    ["http {\n" |
783     nginx_defaults(PrivDir) ++
784	 [" server {",
785	  {root,                DataDir},
786	  {listen,              integer_to_list(Port)},
787	  " location / {\n  try_files $uri $uri/ /index.html;\n}"
788	  "}\n", "}\n"
789	 ]
790    ];
791
792nginx_http("https", Config) ->
793    PrivDir = ?config(priv_dir, Config),
794    DataDir = ?config(data_dir, Config),
795    Port = ?config(port, Config),
796    SSLOpts = ?config(server_verification_opts, Config),
797    Ciphers = proplists:get_value(ciphers, SSLOpts),
798    ReuseSession = ?config(reuse_sessions, Config),
799    ["http {" |
800     nginx_defaults(PrivDir) ++
801	 [" server {",
802	  {"root",                DataDir},
803	  {"listen",              integer_to_list(Port) ++ " ssl"},
804	  {"ssl_certificate",     ?config(certfile, SSLOpts)},
805	  {"ssl_certificate_key", ?config(keyfile, SSLOpts)},
806	  {"ssl_protocols",       "TLSv1 TLSv1.1 TLSv1.2"},
807	  {"ssl_ciphers",         Ciphers},
808	  {"ssl_session_cache",    nginx_reuse_session(ReuseSession)},
809	  " location / {\n  try_files $uri $uri/ /index.html;\n}"
810	  "}\n", "}\n"
811	 ]
812    ].
813
814nginx_defaults(PrivDir) ->
815    [
816     %% Set temp and cache file options that will otherwise default to
817     %% restricted locations accessible only to root.
818     {"client_body_temp_path", filename:join(PrivDir, "client_body")},
819     {"fastcgi_temp_path",   filename:join(PrivDir, "fastcgi_temp")},
820     {"proxy_temp_path", filename:join(PrivDir, "proxy_temp")},
821     {"scgi_temp_path", filename:join(PrivDir, "scgi_temp")},
822     {"uwsgi_temp_path", filename:join(PrivDir, "uwsgi_temp_path")},
823     {"access_log",  filename:join(PrivDir, "access.log")},
824     {"error_log",   filename:join(PrivDir, "error.log")},
825     %% Standard options
826     {"sendfile", "on"},
827     {"tcp_nopush", "on"},
828     {"tcp_nodelay", "on"},
829     {"keepalive_timeout",  "360"},
830     {"types_hash_max_size", "2048"},
831     {"include", "/etc/nginx/mime.types"},
832     {"default_type", "application/octet-stream"}
833    ].
834
835nginx_reuse_session(true) ->
836    "on";
837nginx_reuse_session(false) ->
838    "off".
839
840wait_for_nginx_up(Host, Port) ->
841    case gen_tcp:connect(Host, Port, []) of
842	{ok, Socket} ->
843	    gen_tcp:close(Socket);
844	_  ->
845	    ct:sleep(100),
846	    wait_for_nginx_up(Host, Port)
847    end.
848
849