1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2002-2018. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20%%
21-module(httpc_manager).
22
23-behaviour(gen_server).
24
25-include_lib("inets/src/http_lib/http_internal.hrl").
26-include("httpc_internal.hrl").
27
28%% Internal Application API
29-export([
30	 start_link/3,
31	 request/2,
32	 cancel_request/2,
33	 request_done/2,
34	 retry_request/2,
35	 redirect_request/2,
36	 insert_session/2,
37	 lookup_session/2,
38	 update_session/4,
39	 delete_session/2,
40	 which_sessions/1,
41	 which_session_info/1,
42	 set_options/2,
43	 get_options/2,
44	 store_cookies/3,
45	 which_cookies/1, which_cookies/2, which_cookies/3,
46	 reset_cookies/1,
47	 session_type/1,
48	 info/1
49	]).
50
51%% gen_server callbacks
52-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
53	 code_change/3]).
54
55-record(state,
56	{
57	  cancel = [],	 % [{RequestId, HandlerPid, ClientPid}]
58	  handler_db,    % ets() - Entry: #handler_info{}
59	  cookie_db,     % cookie_db()
60	  session_db,    % ets() - Entry:  #session{}
61	  profile_name,  % atom()
62	  options = #options{}
63	 }).
64
65-define(DELAY, 500).
66
67
68%%====================================================================
69%% Internal Application API
70%%====================================================================
71%%--------------------------------------------------------------------
72%% Function: start_link(ProfileName, CookieDir, ManagedHow) -> {ok, Pid}
73%%
74%% ProfileName - httpc_manager_<Profile>
75%% CookieDir - directory()
76%% ManagedHow - stand_alone | inets
77%%
78%% Description: Starts the http request manager process.
79%% (If ManagedHow = inets then started by the inets supervisor.)
80%%--------------------------------------------------------------------
81
82start_link(Profile, CookieDir, stand_alone) ->
83    ProfileName = httpc:profile_name("stand_alone_", Profile),
84    Args        = [ProfileName, CookieDir],
85    Opts        = [],
86    %% Opts        = [{debug, [log, statistics]}],
87    gen_server:start_link(?MODULE, Args, Opts);
88start_link(Profile, CookieDir, _) ->
89    ProfileName  = httpc:profile_name(Profile),
90    Server       = {local, ProfileName},
91    Args         = [ProfileName, CookieDir],
92    Opts         = [],
93    %% Opts        = [{debug, [log, statistics]}],
94    gen_server:start_link(Server, ?MODULE, Args, Opts).
95
96
97%%--------------------------------------------------------------------
98%% Function: request(Request, ProfileName) ->
99%%                                      {ok, Requestid} | {error, Reason}
100%%	Request = #request{}
101%%      ProfileName = atom()
102%%
103%% Description: Sends a request to the httpc manager process.
104%%--------------------------------------------------------------------
105
106request(Request, ProfileName) ->
107    call(ProfileName, {request, Request}).
108
109
110%%--------------------------------------------------------------------
111%% Function: retry_request(Request, ProfileName) -> _
112%%	Request = #request{}
113%%      ProfileName = atom()
114%%
115%% Description: Resends a request to the httpc manager process, intended
116%% to be called by the httpc handler process if it has to terminate with
117%% a non empty pipeline.
118%%--------------------------------------------------------------------
119
120retry_request(Request, ProfileName) ->
121    cast(ProfileName, {retry_or_redirect_request, Request}).
122
123
124%%--------------------------------------------------------------------
125%% Function: redirect_request(Request, ProfileName) -> _
126%%	Request = #request{}
127%%      ProfileName = atom()
128%%
129%% Description: Sends an atoumatic redirect request to the httpc
130%% manager process, intended to be called by the httpc handler process
131%% when the automatic redirect option is set.
132%%--------------------------------------------------------------------
133
134redirect_request(Request, ProfileName) ->
135    cast(ProfileName, {retry_or_redirect_request, Request}).
136
137
138%%--------------------------------------------------------------------
139%% Function: cancel_request(RequestId, ProfileName) -> ok
140%%	RequestId - reference()
141%%      ProfileName = atom()
142%%
143%% Description: Cancels the request with <RequestId>.
144%%--------------------------------------------------------------------
145
146cancel_request(RequestId, ProfileName) ->
147    cast(ProfileName, {cancel_request, RequestId}).
148
149%%--------------------------------------------------------------------
150%% Function: request_done(RequestId, ProfileName) -> ok
151%%	RequestId - reference()
152%%      ProfileName = atom()
153%%
154%% Description: Inform tha manager that a request has been completed.
155%%--------------------------------------------------------------------
156
157request_done(RequestId, ProfileName) ->
158    cast(ProfileName, {request_done, RequestId}).
159
160
161%%--------------------------------------------------------------------
162%% Function: insert_session(Session, ProfileName) -> _
163%%	Session - #session{}
164%%      ProfileName - atom()
165%%
166%% Description: Inserts session information into the httpc manager
167%% table <ProfileName>_session_db. Intended to be called by
168%% the httpc request handler process.
169%%--------------------------------------------------------------------
170
171insert_session(Session, ProfileName) ->
172    SessionDbName = session_db_name(ProfileName),
173    ?hcrt("insert session", [{session, Session}, {profile, ProfileName}]),
174    ets:insert(SessionDbName, Session).
175
176
177%%--------------------------------------------------------------------
178%% Function: lookup_session(SessionId, ProfileName) -> _
179%%      SessionId - term()
180%%      ProfileName - atom()
181%%
182%% Description: Looks up a session record in the httpc manager
183%% table <ProfileName>__session_db.
184%%--------------------------------------------------------------------
185
186lookup_session(SessionId, ProfileName) ->
187    SessionDbName = session_db_name(ProfileName),
188    ?hcrt("lookup session", [{session_id, SessionId}, {profile, ProfileName}]),
189    ets:lookup(SessionDbName, SessionId).
190
191
192%%--------------------------------------------------------------------
193%% Function: update_session(ProfileName, SessionId, Pos, Value) -> _
194%%	Session - #session{}
195%%      ProfileName - atom()
196%%
197%% Description: Update, only one field (Pos) of the session record
198%%              identified by the SessionId, the session information
199%%              of the httpc manager table <ProfileName>__session_db.
200%%              Intended to be called by the httpc request handler process.
201%%--------------------------------------------------------------------
202
203update_session(ProfileName, SessionId, Pos, Value) ->
204    SessionDbName = session_db_name(ProfileName),
205    ?hcrt("update session",
206	  [{id,      SessionId},
207	   {pos,     Pos},
208	   {value,   Value},
209	   {profile, ProfileName}]),
210    ets:update_element(SessionDbName, SessionId, {Pos, Value}).
211
212
213%%--------------------------------------------------------------------
214%% Function: delete_session(SessionId, ProfileName) -> void()
215%%	SessionId -  {{Host, Port}, HandlerPid}
216%%      ProfileName - atom()
217%%
218%% Description: Deletes session information from the httpc manager
219%% table <ProfileName>__session_db. Intended to be called by
220%% the httpc request handler process.
221%%--------------------------------------------------------------------
222
223delete_session(SessionId, ProfileName) ->
224    SessionDbName = session_db_name(ProfileName),
225    ?hcrt("delete session", [{session_is, SessionId}, {profile, ProfileName}]),
226    ets:delete(SessionDbName, SessionId).
227
228
229%%--------------------------------------------------------------------
230%% Function: which sessions(ProfileName) -> SessionsInfo
231%%      ProfileName - atom()
232%%      SessionsInfo - {GoodSessions, BadSessions, NonSessions}
233%%      GoodSessions - [#session{}]
234%%      BadSessions  - [tuple()]
235%%      NonSessions  - [term()]
236%%
237%% Description: Produces a list of all sessions in the session db.
238%% Used for debugging and since that is the intent, there is some
239%% checking and transforming done, which produces the results.
240%%--------------------------------------------------------------------
241
242which_sessions(ProfileName) ->
243    ?hcrt("which_sessions", [{profile, ProfileName}]),
244    SessionDbName = session_db_name(ProfileName),
245    which_sessions2(SessionDbName).
246
247which_sessions2(SessionDbName) ->
248    Sessions     = which_sessions_order(ets:tab2list(SessionDbName)),
249    GoodSessions = [GoodSession || {good_session, GoodSession} <- Sessions],
250    BadSessions  = [BadSession  || {bad_session,  BadSession}  <- Sessions],
251    NonSessions  = [NonSession  || {non_session,  NonSession}  <- Sessions],
252    {lists:keysort(#session.id, GoodSessions),
253     lists:keysort(#session.id, BadSessions),
254     lists:sort(NonSessions)}.
255
256which_sessions_order([]) ->
257    [];
258which_sessions_order([Session|Sessions]) when is_record(Session, session) ->
259    [{good_session, Session} | which_sessions_order(Sessions)];
260which_sessions_order([BadSession|Sessions])
261  when is_tuple(BadSession) andalso
262       (element(1, BadSession) =:= session) ->
263    [{bad_session, BadSession} | which_sessions_order(Sessions)];
264which_sessions_order([NonSession|Sessions]) ->
265    [{non_session, NonSession} | which_sessions_order(Sessions)].
266
267
268%%--------------------------------------------------------------------
269%% Function: which session_info(ProfileName) -> list()
270%%
271%% Description: Produces a ets table info list of the sessions table
272%%--------------------------------------------------------------------
273
274which_session_info(ProfileName) ->
275    SessionDbName = session_db_name(ProfileName),
276    ?hcrt("which_session_info", [{profile, ProfileName}]),
277    ets:info(SessionDbName).
278
279
280%%--------------------------------------------------------------------
281%% Function: set_options(Options, ProfileName) -> ok
282%%
283%%	Options = [Option]
284%%	Option = {proxy, {Proxy, [NoProxy]}}
285%%              | {max_pipeline_length, integer()} |
286%%                {max_sessions, integer()} | {pipeline_timeout, integer()}
287%%	Proxy = {Host, Port}
288%%	NoProxy - [Domain | HostName | IPAddress]
289%%	Max - integer()
290%%	ProfileName = atom()
291%%
292%% Description: Sets the options to be used by the client.
293%%--------------------------------------------------------------------
294
295set_options(Options, ProfileName) ->
296    cast(ProfileName, {set_options, Options}).
297
298
299%%--------------------------------------------------------------------
300%% Function: get_options(OptionItems, ProfileName) -> Values
301%%
302%%	OptionItems = [OptionItem]
303%%	OptionItem = Any or all fields of the current #options{} record
304%%      Values = [{OptionItem, Value}]
305%%      Value = term()
306%%
307%% Description: Gets the specified options used by the client.
308%%--------------------------------------------------------------------
309
310get_options(Options, ProfileName) ->
311    call(ProfileName, {get_options, Options}).
312
313
314%%--------------------------------------------------------------------
315%% Function: store_cookies(Cookies, Address, ProfileName) -> ok
316%%
317%%	Cookies = [Cookie]
318%%	Cookie = #http_cookie{}
319%%	ProfileName = atom()
320%%
321%% Description: Stores cookies from the server.
322%%--------------------------------------------------------------------
323store_cookies([], _, _) ->
324    ok;
325store_cookies(Cookies, Address, ProfileName) ->
326    cast(ProfileName, {store_cookies, {Cookies, Address}}).
327
328
329%%--------------------------------------------------------------------
330%% Function: reset_cookies(ProfileName) -> void()
331%%
332%%	Url = string()
333%%      ProfileName = atom()
334%%
335%% Description: Resets the cookie database
336%%--------------------------------------------------------------------
337
338reset_cookies(ProfileName) ->
339    call(ProfileName, reset_cookies).
340
341
342%%--------------------------------------------------------------------
343%% Function: which_cookies(ProfileName)               -> [cookie()]
344%%           which_cookies(Url, ProfileName)          -> [cookie()]
345%%           which_cookies(Url, Options, ProfileName) -> [cookie()]
346%%
347%%	Url = string()
348%%	Options = [option()]
349%%      ProfileName = atom()
350%%      option() = {ipv6_host_with_brackets, boolean()}
351%%
352%% Description: Retrieves the cookies that would be sent when
353%% requesting <Url>.
354%%--------------------------------------------------------------------
355
356which_cookies(ProfileName) when is_atom(ProfileName) ->
357    call(ProfileName, which_cookies).
358
359which_cookies(Url, ProfileName)
360  when is_list(Url) andalso is_atom(ProfileName) ->
361    which_cookies(Url, [], ProfileName).
362
363which_cookies(Url, Options, ProfileName)
364  when is_list(Url) andalso is_list(Options) andalso is_atom(ProfileName) ->
365    call(ProfileName, {which_cookies, Url, Options}).
366
367
368%%--------------------------------------------------------------------
369%% Function: info(ProfileName) -> list()
370%%
371%%      ProfileName = atom()
372%%
373%% Description: Retrieves various info about the manager and the
374%%              handlers it manages
375%%--------------------------------------------------------------------
376
377info(ProfileName) ->
378    call(ProfileName, info).
379
380
381%%--------------------------------------------------------------------
382%% Function: session_type(Options) -> ok
383%%
384%%	Options = #options{}
385%%
386%% Description: Determines if to use pipelined sessions or not.
387%%--------------------------------------------------------------------
388
389session_type(#options{pipeline_timeout = 0}) ->
390    keep_alive;
391session_type(_) ->
392    pipeline.
393
394
395%%====================================================================
396%% gen_server callback functions
397%%====================================================================
398
399%%--------------------------------------------------------------------
400%% Function: init([ProfileName, CookiesConf]) -> {ok, State} |
401%%                       {ok, State, Timeout} | ignore |{stop, Reason}
402%% Description: Initiates the httpc_manger process
403%%--------------------------------------------------------------------
404init([ProfileName, CookiesDir]) ->
405    process_flag(trap_exit, true),
406    ?hcrv("starting", [{profile, ProfileName}]),
407    case (catch do_init(ProfileName, CookiesDir)) of
408	{ok, _} = OK ->
409	    ?hcrd("started", [OK]),
410	    OK;
411	{error, Reason} ->
412	    {stop, Reason};
413	Crap ->
414	    {stop, Crap}
415    end.
416
417
418do_init(ProfileName, CookiesDir) ->
419    %% Create session db
420    ?hcrt("create session db", []),
421    SessionDbName = session_db_name(ProfileName),
422    ets:new(SessionDbName,
423	    [public, set, named_table, {keypos, #session.id}]),
424
425    %% Create handler db
426    ?hcrt("create handler/request db", []),
427    HandlerDbName = handler_db_name(ProfileName),
428    ets:new(HandlerDbName, [protected, set, named_table, {keypos, 1}]),
429
430    %% Cookie DB
431    ?hcrt("create cookie db", []),
432    SessionCookieDbName = session_cookie_db_name(ProfileName),
433    CookieDbName        = cookie_db_name(ProfileName),
434    CookieDb            = httpc_cookie:open_db(CookieDbName, CookiesDir,
435					       SessionCookieDbName),
436
437    State = #state{handler_db   = HandlerDbName,
438		   cookie_db    = CookieDb,
439		   session_db   = SessionDbName,
440		   profile_name = ProfileName},
441    {ok, State}.
442
443
444%%--------------------------------------------------------------------
445%% Function: handle_call(Request, From, State) -> {reply, Reply, State} |
446%%          {reply, Reply, State, Timeout} |
447%%          {noreply, State}               |
448%%          {noreply, State, Timeout}      |
449%%          {stop, Reason, Reply, State}   | (terminate/2 is called)
450%%          {stop, Reason, State}            (terminate/2 is called)
451%% Description: Handling call messages
452%%--------------------------------------------------------------------
453handle_call({request, Request}, _, State) ->
454    ?hcri("request", [{request, Request}]),
455    case (catch handle_request(Request, State)) of
456	{reply, Msg, NewState} ->
457	    {reply, Msg, NewState};
458	Error ->
459	    {stop, Error, httpc_response:error(Request, Error), State}
460    end;
461
462handle_call(reset_cookies, _, #state{cookie_db = CookieDb} = State) ->
463    ?hcrv("reset cookies", []),
464    httpc_cookie:reset_db(CookieDb),
465    {reply, ok, State};
466
467handle_call(which_cookies, _, #state{cookie_db = CookieDb} = State) ->
468    ?hcrv("which cookies", []),
469    CookieHeaders = httpc_cookie:which_cookies(CookieDb),
470    {reply, CookieHeaders, State};
471
472handle_call({which_cookies, Url, Options}, _,
473	    #state{cookie_db = CookieDb} = State) ->
474    ?hcrv("which cookies", [{url, Url}, {options, Options}]),
475    case uri_parse(Url) of
476	{ok, {Scheme, Host, Port, Path}} ->
477	    CookieHeaders =
478		httpc_cookie:header(CookieDb, erlang:list_to_existing_atom(Scheme), {Host, Port}, Path),
479	    {reply, CookieHeaders, State};
480	{error, _} = ERROR ->
481	    {reply, ERROR, State}
482    end;
483
484handle_call({get_options, OptionItems}, _, #state{options = Options} = State) ->
485    ?hcrv("get options", [{option_items, OptionItems}]),
486    Reply = [{OptionItem, get_option(OptionItem, Options)} ||
487		OptionItem <- OptionItems],
488    {reply, Reply, State};
489
490handle_call(info, _, State) ->
491    ?hcrv("info", []),
492    Info = get_manager_info(State),
493    {reply, Info, State};
494
495handle_call(Req, From, #state{profile_name = ProfileName} = State) ->
496    error_report(ProfileName,
497		 "received unkown request"
498		 "~n   Req:  ~p"
499		 "~n   From: ~p", [Req, From]),
500    {reply, {error, 'API_violation'}, State}.
501
502
503%%--------------------------------------------------------------------
504%% Function: handle_cast(Msg, State) -> {noreply, State} |
505%%          {noreply, State, Timeout} |
506%%          {stop, Reason, State}            (terminate/2 is called)
507%% Description: Handling cast messages
508%%--------------------------------------------------------------------
509handle_cast({retry_or_redirect_request, {Time, Request}},
510	    #state{profile_name = ProfileName} = State) ->
511    {ok, _} = timer:apply_after(Time, ?MODULE, retry_request, [Request, ProfileName]),
512    {noreply, State};
513
514handle_cast({retry_or_redirect_request, Request}, State) ->
515    case (catch handle_request(Request, State)) of
516	{reply, {ok, _}, NewState} ->
517	    {noreply, NewState};
518	Error  ->
519	    httpc_response:error(Request, Error),
520	    {stop, Error, State}
521    end;
522
523handle_cast({cancel_request, RequestId},
524	    #state{handler_db = HandlerDb} = State) ->
525    case ets:lookup(HandlerDb, RequestId) of
526	[] ->
527	    %% Request already compleated nothing to
528	    %% cancel
529	    {noreply, State};
530	[{_, Pid, _}] ->
531	    httpc_handler:cancel(RequestId, Pid),
532	    ets:delete(State#state.handler_db, RequestId),
533	    {noreply, State}
534    end;
535
536handle_cast({request_done, RequestId}, State) ->
537    ?hcrv("request done", [{request_id, RequestId}]),
538    ets:delete(State#state.handler_db, RequestId),
539    {noreply, State};
540
541handle_cast({set_options, Options}, State = #state{options = OldOptions}) ->
542    ?hcrv("set options", [{options, Options}, {old_options, OldOptions}]),
543    NewOptions =
544	#options{proxy                 = get_proxy(Options, OldOptions),
545		 https_proxy           = get_https_proxy(Options, OldOptions),
546		 pipeline_timeout      = get_pipeline_timeout(Options, OldOptions),
547		 max_pipeline_length   = get_max_pipeline_length(Options, OldOptions),
548		 max_keep_alive_length = get_max_keep_alive_length(Options, OldOptions),
549		 keep_alive_timeout    = get_keep_alive_timeout(Options, OldOptions),
550		 max_sessions          = get_max_sessions(Options, OldOptions),
551		 cookies               = get_cookies(Options, OldOptions),
552		 ipfamily              = get_ipfamily(Options, OldOptions),
553		 ip                    = get_ip(Options, OldOptions),
554		 port                  = get_port(Options, OldOptions),
555		 verbose               = get_verbose(Options, OldOptions),
556		 socket_opts           = get_socket_opts(Options, OldOptions),
557		 unix_socket           = get_unix_socket_opts(Options, OldOptions)
558		},
559    case {OldOptions#options.verbose, NewOptions#options.verbose} of
560	{Same, Same} ->
561	    ok;
562	{_, false} ->
563	    dbg:stop();
564	{false, Level}  ->
565	    dbg:tracer(),
566	    handle_verbose(Level);
567	{_, Level} ->
568	    dbg:stop(),
569	    dbg:tracer(),
570	    handle_verbose(Level)
571    end,
572    {noreply, State#state{options = NewOptions}};
573
574handle_cast({store_cookies, _},
575	    State = #state{options = #options{cookies = disabled}}) ->
576    {noreply, State};
577
578handle_cast({store_cookies, {Cookies, _}}, State) ->
579    ok = do_store_cookies(Cookies, State),
580    {noreply, State};
581
582handle_cast(Msg, #state{profile_name = ProfileName} = State) ->
583    error_report(ProfileName,
584		 "recived unknown message"
585		 "~n   Msg: ~p", [Msg]),
586    {noreply, State}.
587
588%%--------------------------------------------------------------------
589%% Function: handle_info(Info, State) -> {noreply, State} |
590%%          {noreply, State, Timeout} |
591%%          {stop, Reason, State}            (terminate/2 is called)
592%% Description: Handling all non call/cast messages
593%%---------------------------------------------------------
594handle_info({'EXIT', _, _}, State) ->
595    %% Handled in DOWN
596    {noreply, State};
597handle_info({'DOWN', _, _, Pid, _}, State) ->
598    ets:match_delete(State#state.handler_db, {'_', Pid, '_'}),
599    {noreply, State};
600handle_info(Info, State) ->
601    Report = io_lib:format("Unknown message in "
602			   "httpc_manager:handle_info ~p~n", [Info]),
603    error_logger:error_report(Report),
604    {noreply, State}.
605
606%%--------------------------------------------------------------------
607%% Function: terminate(Reason, State) -> _  (ignored by gen_server)
608%% Description: Shutdown the httpc_handler
609%%--------------------------------------------------------------------
610terminate(_, State) ->
611    httpc_cookie:close_db(State#state.cookie_db),
612    ets:delete(State#state.session_db),
613    ets:delete(State#state.handler_db).
614
615
616%%--------------------------------------------------------------------
617%% Func: code_change(_OldVsn, State, Extra) -> {ok, NewState}
618%% Purpose: Convert process state when code is changed
619%%--------------------------------------------------------------------
620code_change(_,
621	    #state{session_db = SessionDB} = State,
622	    upgrade_from_pre_5_8_1) ->
623    Upgrade =
624	fun({session,
625	     Id, ClientClose, Scheme, Socket, SocketType, QueueLen, Type}) ->
626		{ok, #session{id           = Id,
627			      client_close = ClientClose,
628			      scheme       = Scheme,
629			      socket       = Socket,
630			      socket_type  = SocketType,
631			      queue_length = QueueLen,
632			      type         = Type}};
633	   (_) -> % Already upgraded (by handler)
634		ignore
635	end,
636    (catch update_session_table(SessionDB, Upgrade)),
637    {ok, State};
638
639code_change(_,
640	    #state{session_db = SessionDB} = State,
641	    downgrade_to_pre_5_8_1) ->
642    Downgrade =
643	fun(#session{id           = Id,
644		     client_close = ClientClose,
645		     scheme       = Scheme,
646		     socket       = Socket,
647		     socket_type  = SocketType,
648		     queue_length = QueueLen,
649		     type         = Type}) ->
650		{ok, {session,
651		      Id, ClientClose, Scheme, Socket, SocketType,
652		      QueueLen, Type}};
653	   (_) -> % Already downgraded (by handler)
654		ignore
655	end,
656    (catch update_session_table(SessionDB, Downgrade)),
657    {ok, State};
658
659code_change(_, State, _) ->
660    {ok, State}.
661
662%% This function is used to catch everything that falls through the cracks...
663update_session_table(SessionDB, Transform) ->
664    ets:safe_fixtable(SessionDB, true),
665    update_session_table(SessionDB, ets:first(SessionDB), Transform),
666    ets:safe_fixtable(SessionDB, false).
667
668update_session_table(_SessionDB, '$end_of_table', _Transform) ->
669    ok;
670update_session_table(SessionDB, Key, Transform) ->
671    case ets:lookup(SessionDB, Key) of
672	[OldSession] ->
673	    case Transform(OldSession) of
674		{ok, NewSession} ->
675		    ets:insert(SessionDB, NewSession);
676		ignore ->
677		    ok
678	    end;
679	_ ->
680	    ok
681    end,
682    update_session_table(SessionDB, ets:next(SessionDB, Key), Transform).
683
684
685%%--------------------------------------------------------------------
686%% Internal functions
687%%--------------------------------------------------------------------
688
689get_manager_info(#state{handler_db = HDB,
690			session_db = SDB,
691			cookie_db  = CDB,
692			options    = Options} = _State) ->
693    HandlerInfo = get_handler_info(HDB),
694    SessionInfo = which_sessions2(SDB),
695    OptionsInfo =
696	[{Item, get_option(Item, Options)} ||
697		  Item <- record_info(fields, options)],
698    CookieInfo  = httpc_cookie:which_cookies(CDB),
699    [{handlers, HandlerInfo},
700     {sessions, SessionInfo},
701     {options,  OptionsInfo},
702     {cookies,  CookieInfo}].
703
704sort_handlers(Unsorted) ->
705    sort_handlers2(lists:keysort(1, Unsorted)).
706
707sort_handlers2([]) ->
708    [];
709sort_handlers2([{HandlerPid, RequestId}|L]) ->
710    {Handler, Rest} = sort_handlers2(HandlerPid, [RequestId], L),
711    [Handler | sort_handlers2(Rest)].
712
713sort_handlers2(HandlerPid, Reqs, []) ->
714    {{HandlerPid, lists:sort(Reqs)}, []};
715sort_handlers2(HandlerPid, Reqs, [{HandlerPid, ReqId}|Rest]) ->
716    sort_handlers2(HandlerPid, [ReqId|Reqs], Rest);
717sort_handlers2(HandlerPid1, Reqs, [{HandlerPid2, _}|_] = Rest)
718  when HandlerPid1 =/= HandlerPid2 ->
719    {{HandlerPid1, lists:sort(Reqs)}, Rest}.
720
721get_handler_info(Tab) ->
722    Pattern   = {'$2', '$1', '_'},
723    Handlers1 = [{Pid, Id} || [Pid, Id] <- ets:match(Tab, Pattern)],
724    Handlers2 = sort_handlers(Handlers1),
725    [{Pid, Reqs, httpc_handler:info(Pid)} || {Pid, Reqs} <- Handlers2].
726
727handle_request(#request{settings =
728			#http_options{version = "HTTP/1.0"}} = Request,
729	       State) ->
730    %% Act as an HTTP/1.0 client that does not
731    %% use persistent connections
732
733    NewRequest = handle_cookies(generate_request_id(Request), State),
734    NewHeaders =
735	(NewRequest#request.headers)#http_request_h{connection
736						    = "close"},
737    start_handler(NewRequest#request{headers = NewHeaders}, State),
738    {reply, {ok, NewRequest#request.id}, State};
739
740%% Simple socket options handling (ERL-441).
741%%
742%% TODO: Refactor httpc to enable sending socket options in requests
743%%       using persistent connections. This workaround opens a new
744%%       connection for each request with non-empty socket_opts.
745handle_request(Request0 = #request{socket_opts = SocketOpts},
746               State0 = #state{options = Options0})
747  when is_list(SocketOpts) andalso length(SocketOpts) > 0 ->
748    Request = handle_cookies(generate_request_id(Request0), State0),
749    Options = convert_options(SocketOpts, Options0),
750    State = State0#state{options = Options},
751    Headers =
752	(Request#request.headers)#http_request_h{connection
753						    = "close"},
754    %% Reset socket_opts to avoid setopts failure.
755    start_handler(Request#request{headers = Headers, socket_opts = []}, State),
756    %% Do not change the state
757    {reply, {ok, Request#request.id}, State0};
758
759handle_request(Request, State = #state{options = Options}) ->
760    NewRequest = handle_cookies(generate_request_id(Request), State),
761    SessionType = session_type(Options),
762    case select_session(Request#request.method,
763			Request#request.address,
764			Request#request.scheme, SessionType, State) of
765	{ok, HandlerPid} ->
766	    pipeline_or_keep_alive(NewRequest, HandlerPid, State);
767	no_connection ->
768	    start_handler(NewRequest, State);
769	{no_session,  OpenSessions} when OpenSessions
770	< Options#options.max_sessions ->
771	    start_handler(NewRequest, State);
772	{no_session, _} ->
773	    %% Do not start any more persistent connections
774	    %% towards this server.
775	    NewHeaders =
776		(NewRequest#request.headers)#http_request_h{connection
777							    = "close"},
778	    start_handler(NewRequest#request{headers = NewHeaders}, State)
779    end,
780    {reply, {ok, NewRequest#request.id}, State}.
781
782
783%% Convert Request options to State options
784convert_options([], Options) ->
785    Options;
786convert_options([{ipfamily, Value}|T], Options) ->
787    convert_options(T, Options#options{ipfamily = Value});
788convert_options([{ip, Value}|T], Options) ->
789    convert_options(T, Options#options{ip = Value});
790convert_options([{port, Value}|T], Options) ->
791    convert_options(T, Options#options{port = Value});
792convert_options([Option|T], Options = #options{socket_opts = SocketOpts}) ->
793    convert_options(T, Options#options{socket_opts = SocketOpts ++ [Option]}).
794
795start_handler(#request{id   = Id,
796		       from = From} = Request,
797	      #state{profile_name = ProfileName,
798		     handler_db   = HandlerDb,
799		     options      = Options}) ->
800    {ok, Pid} =
801	case is_inets_manager() of
802	    true ->
803		httpc_handler_sup:start_child([whereis(httpc_handler_sup),
804					       Request, Options, ProfileName]);
805	    false ->
806		httpc_handler:start_link(self(), Request, Options, ProfileName)
807	end,
808    HandlerInfo = {Id, Pid, From},
809    ets:insert(HandlerDb, HandlerInfo),
810    erlang:monitor(process, Pid).
811
812
813select_session(Method, HostPort, Scheme, SessionType,
814	       #state{options = #options{max_pipeline_length   = MaxPipe,
815					 max_keep_alive_length = MaxKeepAlive},
816		      session_db = SessionDb}) ->
817    ?hcrd("select session", [{session_type,          SessionType},
818			     {max_pipeline_length,   MaxPipe},
819			     {max_keep_alive_length, MaxKeepAlive}]),
820    case httpc_request:is_idempotent(Method) orelse
821	(SessionType =:= keep_alive) of
822	true ->
823	    %% Look for handlers connecting to this host (HostPort)
824	    %% session with record name field (session) and
825	    %% socket fields ignored. The fields id (part of: HostPort),
826	    %% client_close, scheme and type specified.
827	    %% The fields id (part of: HandlerPid) and queue_length
828	    %% specified.
829	    Pattern = #session{id           = {HostPort, '$1'},
830			       client_close = false,
831			       scheme       = Scheme,
832			       queue_length = '$2',
833			       type         = SessionType,
834			       available    = true,
835			       _            = '_'},
836	    %% {'_', {HostPort, '$1'}, false, Scheme, '_', '$2', SessionTyp},
837	    Candidates = ets:match(SessionDb, Pattern),
838	    ?hcrd("select session", [{host_port,  HostPort},
839				     {scheme,     Scheme},
840				     {type,       SessionType},
841				     {candidates, Candidates}]),
842	    select_session(Candidates, MaxKeepAlive, MaxPipe, SessionType);
843	false ->
844	    no_connection
845    end.
846
847select_session(Candidates, Max, _, keep_alive) ->
848    select_session(Candidates, Max);
849select_session(Candidates, _, Max, pipeline) ->
850    select_session(Candidates, Max).
851
852select_session([] = _Candidates, _Max) ->
853    ?hcrd("select session - no candidate", []),
854    no_connection;
855select_session(Candidates, Max) ->
856    NewCandidates =
857	[{Pid, Length} || [Pid, Length] <- Candidates, Length =< Max],
858    case lists:keysort(2, NewCandidates) of
859	[] ->
860	    {no_session, length(Candidates)};
861	[{HandlerPid, _} | _] ->
862	    ?hcrd("select session - found one", [{handler, HandlerPid}]),
863	    {ok, HandlerPid}
864    end.
865
866pipeline_or_keep_alive(#request{id   = Id,
867				from = From} = Request,
868		       HandlerPid,
869		       #state{handler_db = HandlerDb} = State) ->
870    case httpc_handler:send(Request, HandlerPid) of
871	ok ->
872	    HandlerInfo = {Id, HandlerPid, From},
873	    ets:insert(HandlerDb, HandlerInfo);
874	{error, closed}  -> % timeout pipelining failed
875	    start_handler(Request, State)
876    end.
877
878is_inets_manager() ->
879    case get('$ancestors') of
880	[httpc_profile_sup | _] ->
881	    true;
882	_ ->
883	    false
884    end.
885
886generate_request_id(Request) ->
887    case Request#request.id of
888	undefined ->
889	    RequestId = make_ref(),
890	    Request#request{id = RequestId};
891	_ ->
892	    %% This is an automatic redirect or a retryed pipelined request
893	    %% => keep the old id.
894	    Request
895    end.
896
897handle_cookies(Request, #state{options = #options{cookies = disabled}}) ->
898    Request;
899handle_cookies(
900  #request{scheme  = Scheme,
901	   address = Address,
902	   path    = Path,
903	   headers = #http_request_h{other = Other} = Hdrs} = Request,
904  #state{cookie_db = CookieDb}) ->
905    case httpc_cookie:header(CookieDb, Scheme, Address, Path) of
906	{"cookie", ""} ->
907	    Request;
908	CookieHeader ->
909	    NewHeaders = Hdrs#http_request_h{other = [CookieHeader | Other]},
910	    Request#request{headers = NewHeaders}
911    end.
912
913do_store_cookies([], _) ->
914    ok;
915do_store_cookies([Cookie | Cookies], #state{cookie_db = CookieDb} = State) ->
916    ok = httpc_cookie:insert(CookieDb, Cookie),
917    do_store_cookies(Cookies, State).
918
919session_db_name(ProfileName) ->
920    make_db_name(ProfileName, "__session_db").
921
922cookie_db_name(ProfileName) ->
923    make_db_name(ProfileName, "__cookie_db").
924
925session_cookie_db_name(ProfileName) ->
926    make_db_name(ProfileName, "__session_cookie_db").
927
928handler_db_name(ProfileName) ->
929    make_db_name(ProfileName, "__handler_db").
930
931make_db_name(ProfileName, Post) ->
932    list_to_atom(atom_to_list(ProfileName) ++ Post).
933
934
935%%--------------------------------------------------------------------------
936%% These functions is just simple wrappers to parse specifically HTTP URIs
937%%--------------------------------------------------------------------------
938uri_parse(URI) ->
939    case uri_string:parse(uri_string:normalize(URI)) of
940        #{scheme := Scheme,
941          host := Host,
942          port := Port,
943          path := Path} ->
944            {ok, {Scheme, Host, Port, Path}};
945        #{scheme := Scheme,
946          host := Host,
947          path := Path} ->
948            {ok, {Scheme, Host, scheme_default_port(Scheme), Path}};
949        Other ->
950            {error, maybe_error(Other)}
951    end.
952
953maybe_error({error, Atom, Term}) ->
954    {Atom, Term};
955maybe_error(Other) ->
956    {unexpected, Other}.
957
958scheme_default_port("http") ->
959    80;
960scheme_default_port("https") ->
961    443.
962
963%%--------------------------------------------------------------------------
964
965
966call(ProfileName, Msg) ->
967    Timeout = infinity,
968    call(ProfileName, Msg, Timeout).
969call(ProfileName, Msg, Timeout) ->
970    gen_server:call(ProfileName, Msg, Timeout).
971
972cast(ProfileName, Msg) ->
973   gen_server:cast(ProfileName, Msg).
974
975
976get_option(proxy, #options{proxy = Proxy}) ->
977    Proxy;
978get_option(https_proxy, #options{https_proxy = Proxy}) ->
979    Proxy;
980get_option(pipeline_timeout, #options{pipeline_timeout = Timeout}) ->
981    Timeout;
982get_option(max_pipeline_length, #options{max_pipeline_length = Length}) ->
983    Length;
984get_option(keep_alive_timeout, #options{keep_alive_timeout = Timeout}) ->
985    Timeout;
986get_option(max_keep_alive_length, #options{max_keep_alive_length = Length}) ->
987    Length;
988get_option(max_sessions, #options{max_sessions = MaxSessions}) ->
989    MaxSessions;
990get_option(cookies, #options{cookies = Cookies}) ->
991    Cookies;
992get_option(verbose, #options{verbose = Verbose}) ->
993    Verbose;
994get_option(ipfamily, #options{ipfamily = IpFamily}) ->
995    IpFamily;
996get_option(ip, #options{ip = IP}) ->
997    IP;
998get_option(port, #options{port = Port}) ->
999    Port;
1000get_option(socket_opts, #options{socket_opts = SocketOpts}) ->
1001    SocketOpts;
1002get_option(unix_socket, #options{unix_socket = UnixSocket}) ->
1003    UnixSocket.
1004
1005
1006get_proxy(Opts, #options{proxy = Default}) ->
1007    proplists:get_value(proxy, Opts, Default).
1008
1009get_https_proxy(Opts, #options{https_proxy = Default}) ->
1010    proplists:get_value(https_proxy, Opts, Default).
1011
1012get_pipeline_timeout(Opts, #options{pipeline_timeout = Default}) ->
1013    proplists:get_value(pipeline_timeout, Opts, Default).
1014
1015get_max_pipeline_length(Opts, #options{max_pipeline_length = Default}) ->
1016    proplists:get_value(max_pipeline_length, Opts, Default).
1017
1018get_max_keep_alive_length(Opts, #options{max_keep_alive_length = Default}) ->
1019    proplists:get_value(max_keep_alive_length, Opts, Default).
1020
1021get_keep_alive_timeout(Opts, #options{keep_alive_timeout = Default}) ->
1022    proplists:get_value(keep_alive_timeout, Opts, Default).
1023
1024get_max_sessions(Opts, #options{max_sessions = Default}) ->
1025    proplists:get_value(max_sessions, Opts, Default).
1026
1027get_cookies(Opts, #options{cookies = Default}) ->
1028    proplists:get_value(cookies, Opts, Default).
1029
1030get_ipfamily(Opts, #options{ipfamily = IpFamily}) ->
1031    case lists:keysearch(ipfamily, 1, Opts) of
1032	false ->
1033	    case proplists:get_value(ipv6, Opts) of
1034		enabled ->
1035		    inet6fb4;
1036		disabled ->
1037		    inet;
1038		_ ->
1039		    IpFamily
1040	    end;
1041	{value, {_, Value}} ->
1042	    Value
1043    end.
1044
1045get_ip(Opts, #options{ip = Default}) ->
1046    proplists:get_value(ip, Opts, Default).
1047
1048get_port(Opts, #options{port = Default}) ->
1049    proplists:get_value(port, Opts, Default).
1050
1051get_verbose(Opts, #options{verbose = Default}) ->
1052    proplists:get_value(verbose, Opts, Default).
1053
1054get_socket_opts(Opts, #options{socket_opts = Default}) ->
1055    proplists:get_value(socket_opts, Opts, Default).
1056
1057get_unix_socket_opts(Opts, #options{unix_socket = Default}) ->
1058    proplists:get_value(unix_socket, Opts, Default).
1059
1060handle_verbose(debug) ->
1061    dbg:p(self(), [call]),
1062    dbg:tp(?MODULE, [{'_', [], [{return_trace}]}]);
1063handle_verbose(trace) ->
1064    dbg:p(self(), [call]),
1065    dbg:tpl(?MODULE, [{'_', [], [{return_trace}]}]);
1066handle_verbose(_) ->
1067    ok.
1068
1069error_report(Profile, F, A) ->
1070    Report = io_lib:format("HTTPC-MANAGER<~p> " ++ F ++ "~n", [Profile | A]),
1071    error_logger:error_report(Report).
1072