1%%
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2002-2018. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%
20%%
21-module(httpc_manager).
22
23-behaviour(gen_server).
24
25-include_lib("inets/src/http_lib/http_internal.hrl").
26-include("httpc_internal.hrl").
27
28%% Internal Application API
29-export([
30	 start_link/3,
31	 request/2,
32	 cancel_request/2,
33	 request_done/2,
34	 retry_request/2,
35	 redirect_request/2,
36	 insert_session/2,
37	 lookup_session/2,
38	 update_session/4,
39	 delete_session/2,
40	 which_sessions/1,
41	 which_session_info/1,
42	 set_options/2,
43	 get_options/2,
44	 store_cookies/3,
45	 which_cookies/1, which_cookies/2, which_cookies/3,
46	 reset_cookies/1,
47	 session_type/1,
48	 info/1
49	]).
50
51%% gen_server callbacks
52-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
53	 code_change/3]).
54
55-record(state,
56	{
57	  cancel = [],	 % [{RequestId, HandlerPid, ClientPid}]
58	  handler_db,    % ets() - Entry: #handler_info{}
59	  cookie_db,     % cookie_db()
60	  session_db,    % ets() - Entry:  #session{}
61	  profile_name,  % atom()
62	  options = #options{}
63	 }).
64
65-define(DELAY, 500).
66
67
68%%====================================================================
69%% Internal Application API
70%%====================================================================
71%%--------------------------------------------------------------------
72%% Function: start_link(ProfileName, CookieDir, ManagedHow) -> {ok, Pid}
73%%
74%% ProfileName - httpc_manager_<Profile>
75%% CookieDir - directory()
76%% ManagedHow - stand_alone | inets
77%%
78%% Description: Starts the http request manager process.
79%% (If ManagedHow = inets then started by the inets supervisor.)
80%%--------------------------------------------------------------------
81
82start_link(Profile, CookieDir, stand_alone) ->
83    ProfileName = httpc:profile_name("stand_alone_", Profile),
84    Args        = [ProfileName, CookieDir],
85    Opts        = [],
86    %% Opts        = [{debug, [log, statistics]}],
87    gen_server:start_link(?MODULE, Args, Opts);
88start_link(Profile, CookieDir, _) ->
89    ProfileName  = httpc:profile_name(Profile),
90    Server       = {local, ProfileName},
91    Args         = [ProfileName, CookieDir],
92    Opts         = [],
93    %% Opts        = [{debug, [log, statistics]}],
94    gen_server:start_link(Server, ?MODULE, Args, Opts).
95
96
97%%--------------------------------------------------------------------
98%% Function: request(Request, ProfileName) ->
99%%                                      {ok, Requestid} | {error, Reason}
100%%	Request = #request{}
101%%      ProfileName = atom()
102%%
103%% Description: Sends a request to the httpc manager process.
104%%--------------------------------------------------------------------
105
106request(Request, ProfileName) ->
107    call(ProfileName, {request, Request}).
108
109
110%%--------------------------------------------------------------------
111%% Function: retry_request(Request, ProfileName) -> _
112%%	Request = #request{}
113%%      ProfileName = atom()
114%%
115%% Description: Resends a request to the httpc manager process, intended
116%% to be called by the httpc handler process if it has to terminate with
117%% a non empty pipeline.
118%%--------------------------------------------------------------------
119
120retry_request(Request, ProfileName) ->
121    cast(ProfileName, {retry_or_redirect_request, Request}).
122
123
124%%--------------------------------------------------------------------
125%% Function: redirect_request(Request, ProfileName) -> _
126%%	Request = #request{}
127%%      ProfileName = atom()
128%%
129%% Description: Sends an atoumatic redirect request to the httpc
130%% manager process, intended to be called by the httpc handler process
131%% when the automatic redirect option is set.
132%%--------------------------------------------------------------------
133
134redirect_request(Request, ProfileName) ->
135    cast(ProfileName, {retry_or_redirect_request, Request}).
136
137
138%%--------------------------------------------------------------------
139%% Function: cancel_request(RequestId, ProfileName) -> ok
140%%	RequestId - reference()
141%%      ProfileName = atom()
142%%
143%% Description: Cancels the request with <RequestId>.
144%%--------------------------------------------------------------------
145
146cancel_request(RequestId, ProfileName) ->
147    cast(ProfileName, {cancel_request, RequestId}).
148
149%%--------------------------------------------------------------------
150%% Function: request_done(RequestId, ProfileName) -> ok
151%%	RequestId - reference()
152%%      ProfileName = atom()
153%%
154%% Description: Inform tha manager that a request has been completed.
155%%--------------------------------------------------------------------
156
157request_done(RequestId, ProfileName) ->
158    cast(ProfileName, {request_done, RequestId}).
159
160
161%%--------------------------------------------------------------------
162%% Function: insert_session(Session, ProfileName) -> _
163%%	Session - #session{}
164%%      ProfileName - atom()
165%%
166%% Description: Inserts session information into the httpc manager
167%% table <ProfileName>_session_db. Intended to be called by
168%% the httpc request handler process.
169%%--------------------------------------------------------------------
170
171insert_session(Session, ProfileName) ->
172    SessionDbName = session_db_name(ProfileName),
173    ?hcrt("insert session", [{session, Session}, {profile, ProfileName}]),
174    ets:insert(SessionDbName, Session).
175
176
177%%--------------------------------------------------------------------
178%% Function: lookup_session(SessionId, ProfileName) -> _
179%%      SessionId - term()
180%%      ProfileName - atom()
181%%
182%% Description: Looks up a session record in the httpc manager
183%% table <ProfileName>__session_db.
184%%--------------------------------------------------------------------
185
186lookup_session(SessionId, ProfileName) ->
187    SessionDbName = session_db_name(ProfileName),
188    ?hcrt("lookup session", [{session_id, SessionId}, {profile, ProfileName}]),
189    ets:lookup(SessionDbName, SessionId).
190
191
192%%--------------------------------------------------------------------
193%% Function: update_session(ProfileName, SessionId, Pos, Value) -> _
194%%	Session - #session{}
195%%      ProfileName - atom()
196%%
197%% Description: Update, only one field (Pos) of the session record
198%%              identified by the SessionId, the session information
199%%              of the httpc manager table <ProfileName>__session_db.
200%%              Intended to be called by the httpc request handler process.
201%%--------------------------------------------------------------------
202
203update_session(ProfileName, SessionId, Pos, Value) ->
204    SessionDbName = session_db_name(ProfileName),
205    ?hcrt("update session",
206	  [{id,      SessionId},
207	   {pos,     Pos},
208	   {value,   Value},
209	   {profile, ProfileName}]),
210    ets:update_element(SessionDbName, SessionId, {Pos, Value}).
211
212
213%%--------------------------------------------------------------------
214%% Function: delete_session(SessionId, ProfileName) -> void()
215%%	SessionId -  {{Host, Port}, HandlerPid}
216%%      ProfileName - atom()
217%%
218%% Description: Deletes session information from the httpc manager
219%% table <ProfileName>__session_db. Intended to be called by
220%% the httpc request handler process.
221%%--------------------------------------------------------------------
222
223delete_session(SessionId, ProfileName) ->
224    SessionDbName = session_db_name(ProfileName),
225    ?hcrt("delete session", [{session_is, SessionId}, {profile, ProfileName}]),
226    ets:delete(SessionDbName, SessionId).
227
228
229%%--------------------------------------------------------------------
230%% Function: which sessions(ProfileName) -> SessionsInfo
231%%      ProfileName - atom()
232%%      SessionsInfo - {GoodSessions, BadSessions, NonSessions}
233%%      GoodSessions - [#session{}]
234%%      BadSessions  - [tuple()]
235%%      NonSessions  - [term()]
236%%
237%% Description: Produces a list of all sessions in the session db.
238%% Used for debugging and since that is the intent, there is some
239%% checking and transforming done, which produces the results.
240%%--------------------------------------------------------------------
241
242which_sessions(ProfileName) ->
243    ?hcrt("which_sessions", [{profile, ProfileName}]),
244    SessionDbName = session_db_name(ProfileName),
245    which_sessions2(SessionDbName).
246
247which_sessions2(SessionDbName) ->
248    Sessions     = which_sessions_order(ets:tab2list(SessionDbName)),
249    GoodSessions = [GoodSession || {good_session, GoodSession} <- Sessions],
250    BadSessions  = [BadSession  || {bad_session,  BadSession}  <- Sessions],
251    NonSessions  = [NonSession  || {non_session,  NonSession}  <- Sessions],
252    {lists:keysort(#session.id, GoodSessions),
253     lists:keysort(#session.id, BadSessions),
254     lists:sort(NonSessions)}.
255
256which_sessions_order([]) ->
257    [];
258which_sessions_order([Session|Sessions]) when is_record(Session, session) ->
259    [{good_session, Session} | which_sessions_order(Sessions)];
260which_sessions_order([BadSession|Sessions])
261  when is_tuple(BadSession) andalso
262       (element(1, BadSession) =:= session) ->
263    [{bad_session, BadSession} | which_sessions_order(Sessions)];
264which_sessions_order([NonSession|Sessions]) ->
265    [{non_session, NonSession} | which_sessions_order(Sessions)].
266
267
268%%--------------------------------------------------------------------
269%% Function: which session_info(ProfileName) -> list()
270%%
271%% Description: Produces a ets table info list of the sessions table
272%%--------------------------------------------------------------------
273
274which_session_info(ProfileName) ->
275    SessionDbName = session_db_name(ProfileName),
276    ?hcrt("which_session_info", [{profile, ProfileName}]),
277    ets:info(SessionDbName).
278
279
280%%--------------------------------------------------------------------
281%% Function: set_options(Options, ProfileName) -> ok
282%%
283%%	Options = [Option]
284%%	Option = {proxy, {Proxy, [NoProxy]}}
285%%              | {max_pipeline_length, integer()} |
286%%                {max_sessions, integer()} | {pipeline_timeout, integer()}
287%%	Proxy = {Host, Port}
288%%	NoProxy - [Domain | HostName | IPAddress]
289%%	Max - integer()
290%%	ProfileName = atom()
291%%
292%% Description: Sets the options to be used by the client.
293%%--------------------------------------------------------------------
294
295set_options(Options, ProfileName) ->
296    cast(ProfileName, {set_options, Options}).
297
298
299%%--------------------------------------------------------------------
300%% Function: get_options(OptionItems, ProfileName) -> Values
301%%
302%%	OptionItems = [OptionItem]
303%%	OptionItem = Any or all fields of the current #options{} record
304%%      Values = [{OptionItem, Value}]
305%%      Value = term()
306%%
307%% Description: Gets the specified options used by the client.
308%%--------------------------------------------------------------------
309
310get_options(Options, ProfileName) ->
311    call(ProfileName, {get_options, Options}).
312
313
314%%--------------------------------------------------------------------
315%% Function: store_cookies(Cookies, Address, ProfileName) -> ok
316%%
317%%	Cookies = [Cookie]
318%%	Cookie = #http_cookie{}
319%%	ProfileName = atom()
320%%
321%% Description: Stores cookies from the server.
322%%--------------------------------------------------------------------
323store_cookies([], _, _) ->
324    ok;
325store_cookies(Cookies, Address, ProfileName) ->
326    cast(ProfileName, {store_cookies, {Cookies, Address}}).
327
328
329%%--------------------------------------------------------------------
330%% Function: reset_cookies(ProfileName) -> void()
331%%
332%%	Url = string()
333%%      ProfileName = atom()
334%%
335%% Description: Resets the cookie database
336%%--------------------------------------------------------------------
337
338reset_cookies(ProfileName) ->
339    call(ProfileName, reset_cookies).
340
341
342%%--------------------------------------------------------------------
343%% Function: which_cookies(ProfileName)               -> [cookie()]
344%%           which_cookies(Url, ProfileName)          -> [cookie()]
345%%           which_cookies(Url, Options, ProfileName) -> [cookie()]
346%%
347%%	Url = string()
348%%	Options = [option()]
349%%      ProfileName = atom()
350%%      option() = {ipv6_host_with_brackets, boolean()}
351%%
352%% Description: Retrieves the cookies that would be sent when
353%% requesting <Url>.
354%%--------------------------------------------------------------------
355
356which_cookies(ProfileName) when is_atom(ProfileName) ->
357    call(ProfileName, which_cookies).
358
359which_cookies(Url, ProfileName)
360  when is_list(Url) andalso is_atom(ProfileName) ->
361    which_cookies(Url, [], ProfileName).
362
363which_cookies(Url, Options, ProfileName)
364  when is_list(Url) andalso is_list(Options) andalso is_atom(ProfileName) ->
365    call(ProfileName, {which_cookies, Url, Options}).
366
367
368%%--------------------------------------------------------------------
369%% Function: info(ProfileName) -> list()
370%%
371%%      ProfileName = atom()
372%%
373%% Description: Retrieves various info about the manager and the
374%%              handlers it manages
375%%--------------------------------------------------------------------
376
377info(ProfileName) ->
378    call(ProfileName, info).
379
380
381%%--------------------------------------------------------------------
382%% Function: session_type(Options) -> ok
383%%
384%%	Options = #options{}
385%%
386%% Description: Determines if to use pipelined sessions or not.
387%%--------------------------------------------------------------------
388
389session_type(#options{pipeline_timeout = 0}) ->
390    keep_alive;
391session_type(_) ->
392    pipeline.
393
394
395%%====================================================================
396%% gen_server callback functions
397%%====================================================================
398
399%%--------------------------------------------------------------------
400%% Function: init([ProfileName, CookiesConf]) -> {ok, State} |
401%%                       {ok, State, Timeout} | ignore |{stop, Reason}
402%% Description: Initiates the httpc_manger process
403%%--------------------------------------------------------------------
404init([ProfileName, CookiesDir]) ->
405    process_flag(trap_exit, true),
406    ?hcrv("starting", [{profile, ProfileName}]),
407    case (catch do_init(ProfileName, CookiesDir)) of
408	{ok, _} = OK ->
409	    ?hcrd("started", [OK]),
410	    OK;
411	{error, Reason} ->
412	    {stop, Reason};
413	Crap ->
414	    {stop, Crap}
415    end.
416
417
418do_init(ProfileName, CookiesDir) ->
419    %% Create session db
420    ?hcrt("create session db", []),
421    SessionDbName = session_db_name(ProfileName),
422    ets:new(SessionDbName,
423	    [public, set, named_table, {keypos, #session.id}]),
424
425    %% Create handler db
426    ?hcrt("create handler/request db", []),
427    HandlerDbName = handler_db_name(ProfileName),
428    ets:new(HandlerDbName, [protected, set, named_table, {keypos, 1}]),
429
430    %% Cookie DB
431    ?hcrt("create cookie db", []),
432    SessionCookieDbName = session_cookie_db_name(ProfileName),
433    CookieDbName        = cookie_db_name(ProfileName),
434    CookieDb            = httpc_cookie:open_db(CookieDbName, CookiesDir,
435					       SessionCookieDbName),
436
437    State = #state{handler_db   = HandlerDbName,
438		   cookie_db    = CookieDb,
439		   session_db   = SessionDbName,
440		   profile_name = ProfileName},
441    {ok, State}.
442
443
444%%--------------------------------------------------------------------
445%% Function: handle_call(Request, From, State) -> {reply, Reply, State} |
446%%          {reply, Reply, State, Timeout} |
447%%          {noreply, State}               |
448%%          {noreply, State, Timeout}      |
449%%          {stop, Reason, Reply, State}   | (terminate/2 is called)
450%%          {stop, Reason, State}            (terminate/2 is called)
451%% Description: Handling call messages
452%%--------------------------------------------------------------------
453handle_call({request, Request}, _, State) ->
454    ?hcri("request", [{request, Request}]),
455    case (catch handle_request(Request, State)) of
456	{reply, Msg, NewState} ->
457	    {reply, Msg, NewState};
458	Error ->
459	    {stop, Error, httpc_response:error(Request, Error), State}
460    end;
461
462handle_call(reset_cookies, _, #state{cookie_db = CookieDb} = State) ->
463    ?hcrv("reset cookies", []),
464    httpc_cookie:reset_db(CookieDb),
465    {reply, ok, State};
466
467handle_call(which_cookies, _, #state{cookie_db = CookieDb} = State) ->
468    ?hcrv("which cookies", []),
469    CookieHeaders = httpc_cookie:which_cookies(CookieDb),
470    {reply, CookieHeaders, State};
471
472handle_call({which_cookies, Url, Options}, _,
473	    #state{cookie_db = CookieDb} = State) ->
474    ?hcrv("which cookies", [{url, Url}, {options, Options}]),
475    case uri_parse(Url) of
476	{ok, {Scheme, Host, Port, Path}} ->
477	    CookieHeaders =
478		httpc_cookie:header(CookieDb, erlang:list_to_existing_atom(Scheme), {Host, Port}, Path),
479	    {reply, CookieHeaders, State};
480	{error, _} = ERROR ->
481	    {reply, ERROR, State}
482    end;
483
484handle_call({get_options, OptionItems}, _, #state{options = Options} = State) ->
485    ?hcrv("get options", [{option_items, OptionItems}]),
486    Reply = [{OptionItem, get_option(OptionItem, Options)} ||
487		OptionItem <- OptionItems],
488    {reply, Reply, State};
489
490handle_call(info, _, State) ->
491    ?hcrv("info", []),
492    Info = get_manager_info(State),
493    {reply, Info, State};
494
495handle_call(Req, From, #state{profile_name = ProfileName} = State) ->
496    error_report(ProfileName,
497		 "received unkown request"
498		 "~n   Req:  ~p"
499		 "~n   From: ~p", [Req, From]),
500    {reply, {error, 'API_violation'}, State}.
501
502
503%%--------------------------------------------------------------------
504%% Function: handle_cast(Msg, State) -> {noreply, State} |
505%%          {noreply, State, Timeout} |
506%%          {stop, Reason, State}            (terminate/2 is called)
507%% Description: Handling cast messages
508%%--------------------------------------------------------------------
509handle_cast({retry_or_redirect_request, {Time, Request}},
510	    #state{profile_name = ProfileName} = State) ->
511    {ok, _} = timer:apply_after(Time, ?MODULE, retry_request, [Request, ProfileName]),
512    {noreply, State};
513
514handle_cast({retry_or_redirect_request, Request}, State) ->
515    case (catch handle_request(Request, State)) of
516	{reply, {ok, _}, NewState} ->
517	    {noreply, NewState};
518	Error  ->
519	    httpc_response:error(Request, Error),
520	    {stop, Error, State}
521    end;
522
523handle_cast({cancel_request, RequestId},
524	    #state{handler_db = HandlerDb} = State) ->
525    case ets:lookup(HandlerDb, RequestId) of
526	[] ->
527	    %% Request already compleated nothing to
528	    %% cancel
529	    {noreply, State};
530	[{_, Pid, _}] ->
531	    httpc_handler:cancel(RequestId, Pid),
532	    ets:delete(State#state.handler_db, RequestId),
533	    {noreply, State}
534    end;
535
536handle_cast({request_done, RequestId}, State) ->
537    ?hcrv("request done", [{request_id, RequestId}]),
538    ets:delete(State#state.handler_db, RequestId),
539    {noreply, State};
540
541handle_cast({set_options, Options}, State = #state{options = OldOptions}) ->
542    ?hcrv("set options", [{options, Options}, {old_options, OldOptions}]),
543    NewOptions =
544	#options{proxy                 = get_proxy(Options, OldOptions),
545		 https_proxy           = get_https_proxy(Options, OldOptions),
546		 pipeline_timeout      = get_pipeline_timeout(Options, OldOptions),
547		 max_pipeline_length   = get_max_pipeline_length(Options, OldOptions),
548		 max_keep_alive_length = get_max_keep_alive_length(Options, OldOptions),
549		 keep_alive_timeout    = get_keep_alive_timeout(Options, OldOptions),
550		 max_sessions          = get_max_sessions(Options, OldOptions),
551		 cookies               = get_cookies(Options, OldOptions),
552		 ipfamily              = get_ipfamily(Options, OldOptions),
553		 ip                    = get_ip(Options, OldOptions),
554		 port                  = get_port(Options, OldOptions),
555		 verbose               = get_verbose(Options, OldOptions),
556		 socket_opts           = get_socket_opts(Options, OldOptions),
557		 unix_socket           = get_unix_socket_opts(Options, OldOptions)
558		},
559    case {OldOptions#options.verbose, NewOptions#options.verbose} of
560	{Same, Same} ->
561	    ok;
562	{_, false} ->
563	    dbg:stop();
564	{false, Level}  ->
565	    dbg:tracer(),
566	    handle_verbose(Level);
567	{_, Level} ->
568	    dbg:stop(),
569	    dbg:tracer(),
570	    handle_verbose(Level)
571    end,
572    {noreply, State#state{options = NewOptions}};
573
574handle_cast({store_cookies, _},
575	    State = #state{options = #options{cookies = disabled}}) ->
576    {noreply, State};
577
578handle_cast({store_cookies, {Cookies, _}}, State) ->
579    ok = do_store_cookies(Cookies, State),
580    {noreply, State};
581
582handle_cast(Msg, #state{profile_name = ProfileName} = State) ->
583    error_report(ProfileName,
584		 "recived unknown message"
585		 "~n   Msg: ~p", [Msg]),
586    {noreply, State}.
587
588%%--------------------------------------------------------------------
589%% Function: handle_info(Info, State) -> {noreply, State} |
590%%          {noreply, State, Timeout} |
591%%          {stop, Reason, State}            (terminate/2 is called)
592%% Description: Handling all non call/cast messages
593%%---------------------------------------------------------
594handle_info({'EXIT', _, _}, State) ->
595    %% Handled in DOWN
596    {noreply, State};
597handle_info({'DOWN', _, _, Pid, _}, State) ->
598    ets:match_delete(State#state.handler_db, {'_', Pid, '_'}),
599    {noreply, State};
600handle_info(Info, State) ->
601    Report = io_lib:format("Unknown message in "
602			   "httpc_manager:handle_info ~p~n", [Info]),
603    error_logger:error_report(Report),
604    {noreply, State}.
605
606%%--------------------------------------------------------------------
607%% Function: terminate(Reason, State) -> _  (ignored by gen_server)
608%% Description: Shutdown the httpc_handler
609%%--------------------------------------------------------------------
610terminate(_, State) ->
611    httpc_cookie:close_db(State#state.cookie_db),
612    ets:delete(State#state.session_db),
613    ets:delete(State#state.handler_db).
614
615
616%%--------------------------------------------------------------------
617%% Func: code_change(_OldVsn, State, Extra) -> {ok, NewState}
618%% Purpose: Convert process state when code is changed
619%%--------------------------------------------------------------------
620code_change(_,
621	    #state{session_db = SessionDB} = State,
622	    upgrade_from_pre_5_8_1) ->
623    Upgrade =
624	fun({session,
625	     Id, ClientClose, Scheme, Socket, SocketType, QueueLen, Type}) ->
626		{ok, #session{id           = Id,
627			      client_close = ClientClose,
628			      scheme       = Scheme,
629			      socket       = Socket,
630			      socket_type  = SocketType,
631			      queue_length = QueueLen,
632			      type         = Type}};
633	   (_) -> % Already upgraded (by handler)
634		ignore
635	end,
636    (catch update_session_table(SessionDB, Upgrade)),
637    {ok, State};
638
639code_change(_,
640	    #state{session_db = SessionDB} = State,
641	    downgrade_to_pre_5_8_1) ->
642    Downgrade =
643	fun(#session{id           = Id,
644		     client_close = ClientClose,
645		     scheme       = Scheme,
646		     socket       = Socket,
647		     socket_type  = SocketType,
648		     queue_length = QueueLen,
649		     type         = Type}) ->
650		{ok, {session,
651		      Id, ClientClose, Scheme, Socket, SocketType,
652		      QueueLen, Type}};
653	   (_) -> % Already downgraded (by handler)
654		ignore
655	end,
656    (catch update_session_table(SessionDB, Downgrade)),
657    {ok, State};
658
659code_change(_, State, _) ->
660    {ok, State}.
661
662%% This function is used to catch everything that falls through the cracks...
663update_session_table(SessionDB, Transform) ->
664    ets:safe_fixtable(SessionDB, true),
665    update_session_table(SessionDB, ets:first(SessionDB), Transform),
666    ets:safe_fixtable(SessionDB, false).
667
668update_session_table(_SessionDB, '$end_of_table', _Transform) ->
669    ok;
670update_session_table(SessionDB, Key, Transform) ->
671    case ets:lookup(SessionDB, Key) of
672	[OldSession] ->
673	    case Transform(OldSession) of
674		{ok, NewSession} ->
675		    ets:insert(SessionDB, NewSession);
676		ignore ->
677		    ok
678	    end;
679	_ ->
680	    ok
681    end,
682    update_session_table(SessionDB, ets:next(SessionDB, Key), Transform).
683
684
685%%--------------------------------------------------------------------
686%% Internal functions
687%%--------------------------------------------------------------------
688
689get_manager_info(#state{handler_db = HDB,
690			session_db = SDB,
691			cookie_db  = CDB,
692			options    = Options} = _State) ->
693    HandlerInfo = get_handler_info(HDB),
694    SessionInfo = which_sessions2(SDB),
695    OptionsInfo =
696	[{Item, get_option(Item, Options)} ||
697		  Item <- record_info(fields, options)],
698    CookieInfo  = httpc_cookie:which_cookies(CDB),
699    [{handlers, HandlerInfo},
700     {sessions, SessionInfo},
701     {options,  OptionsInfo},
702     {cookies,  CookieInfo}].
703
704sort_handlers(Unsorted) ->
705    sort_handlers2(lists:keysort(1, Unsorted)).
706
707sort_handlers2([]) ->
708    [];
709sort_handlers2([{HandlerPid, RequestId}|L]) ->
710    {Handler, Rest} = sort_handlers2(HandlerPid, [RequestId], L),
711    [Handler | sort_handlers2(Rest)].
712
713sort_handlers2(HandlerPid, Reqs, []) ->
714    {{HandlerPid, lists:sort(Reqs)}, []};
715sort_handlers2(HandlerPid, Reqs, [{HandlerPid, ReqId}|Rest]) ->
716    sort_handlers2(HandlerPid, [ReqId|Reqs], Rest);
717sort_handlers2(HandlerPid1, Reqs, [{HandlerPid2, _}|_] = Rest)
718  when HandlerPid1 =/= HandlerPid2 ->
719    {{HandlerPid1, lists:sort(Reqs)}, Rest}.
720
721get_handler_info(Tab) ->
722    Pattern   = {'$2', '$1', '_'},
723    Handlers1 = [{Pid, Id} || [Pid, Id] <- ets:match(Tab, Pattern)],
724    Handlers2 = sort_handlers(Handlers1),
725    [{Pid, Reqs, httpc_handler:info(Pid)} || {Pid, Reqs} <- Handlers2].
726
727handle_request(#request{settings =
728			#http_options{version = "HTTP/0.9"}} = Request,
729	       State) ->
730    %% Act as an HTTP/0.9 client that does not know anything
731    %% about persistent connections
732
733    NewRequest = handle_cookies(generate_request_id(Request), State),
734    NewHeaders =
735	(NewRequest#request.headers)#http_request_h{connection
736						    = undefined},
737    start_handler(NewRequest#request{headers = NewHeaders}, State),
738    {reply, {ok, NewRequest#request.id}, State};
739
740handle_request(#request{settings =
741			#http_options{version = "HTTP/1.0"}} = Request,
742	       State) ->
743    %% Act as an HTTP/1.0 client that does not
744    %% use persistent connections
745
746    NewRequest = handle_cookies(generate_request_id(Request), State),
747    NewHeaders =
748	(NewRequest#request.headers)#http_request_h{connection
749						    = "close"},
750    start_handler(NewRequest#request{headers = NewHeaders}, State),
751    {reply, {ok, NewRequest#request.id}, State};
752
753%% Simple socket options handling (ERL-441).
754%%
755%% TODO: Refactor httpc to enable sending socket options in requests
756%%       using persistent connections. This workaround opens a new
757%%       connection for each request with non-empty socket_opts.
758handle_request(Request0 = #request{socket_opts = SocketOpts},
759               State0 = #state{options = Options0})
760  when is_list(SocketOpts) andalso length(SocketOpts) > 0 ->
761    Request = handle_cookies(generate_request_id(Request0), State0),
762    Options = convert_options(SocketOpts, Options0),
763    State = State0#state{options = Options},
764    Headers =
765	(Request#request.headers)#http_request_h{connection
766						    = "close"},
767    %% Reset socket_opts to avoid setopts failure.
768    start_handler(Request#request{headers = Headers, socket_opts = []}, State),
769    %% Do not change the state
770    {reply, {ok, Request#request.id}, State0};
771
772handle_request(Request, State = #state{options = Options}) ->
773    NewRequest = handle_cookies(generate_request_id(Request), State),
774    SessionType = session_type(Options),
775    case select_session(Request#request.method,
776			Request#request.address,
777			Request#request.scheme, SessionType, State) of
778	{ok, HandlerPid} ->
779	    pipeline_or_keep_alive(NewRequest, HandlerPid, State);
780	no_connection ->
781	    start_handler(NewRequest, State);
782	{no_session,  OpenSessions} when OpenSessions
783	< Options#options.max_sessions ->
784	    start_handler(NewRequest, State);
785	{no_session, _} ->
786	    %% Do not start any more persistent connections
787	    %% towards this server.
788	    NewHeaders =
789		(NewRequest#request.headers)#http_request_h{connection
790							    = "close"},
791	    start_handler(NewRequest#request{headers = NewHeaders}, State)
792    end,
793    {reply, {ok, NewRequest#request.id}, State}.
794
795
796%% Convert Request options to State options
797convert_options([], Options) ->
798    Options;
799convert_options([{ipfamily, Value}|T], Options) ->
800    convert_options(T, Options#options{ipfamily = Value});
801convert_options([{ip, Value}|T], Options) ->
802    convert_options(T, Options#options{ip = Value});
803convert_options([{port, Value}|T], Options) ->
804    convert_options(T, Options#options{port = Value});
805convert_options([Option|T], Options = #options{socket_opts = SocketOpts}) ->
806    convert_options(T, Options#options{socket_opts = SocketOpts ++ [Option]}).
807
808start_handler(#request{id   = Id,
809		       from = From} = Request,
810	      #state{profile_name = ProfileName,
811		     handler_db   = HandlerDb,
812		     options      = Options}) ->
813    {ok, Pid} =
814	case is_inets_manager() of
815	    true ->
816		httpc_handler_sup:start_child([whereis(httpc_handler_sup),
817					       Request, Options, ProfileName]);
818	    false ->
819		httpc_handler:start_link(self(), Request, Options, ProfileName)
820	end,
821    HandlerInfo = {Id, Pid, From},
822    ets:insert(HandlerDb, HandlerInfo),
823    erlang:monitor(process, Pid).
824
825
826select_session(Method, HostPort, Scheme, SessionType,
827	       #state{options = #options{max_pipeline_length   = MaxPipe,
828					 max_keep_alive_length = MaxKeepAlive},
829		      session_db = SessionDb}) ->
830    ?hcrd("select session", [{session_type,          SessionType},
831			     {max_pipeline_length,   MaxPipe},
832			     {max_keep_alive_length, MaxKeepAlive}]),
833    case httpc_request:is_idempotent(Method) orelse
834	(SessionType =:= keep_alive) of
835	true ->
836	    %% Look for handlers connecting to this host (HostPort)
837	    %% session with record name field (session) and
838	    %% socket fields ignored. The fields id (part of: HostPort),
839	    %% client_close, scheme and type specified.
840	    %% The fields id (part of: HandlerPid) and queue_length
841	    %% specified.
842	    Pattern = #session{id           = {HostPort, '$1'},
843			       client_close = false,
844			       scheme       = Scheme,
845			       queue_length = '$2',
846			       type         = SessionType,
847			       available    = true,
848			       _            = '_'},
849	    %% {'_', {HostPort, '$1'}, false, Scheme, '_', '$2', SessionTyp},
850	    Candidates = ets:match(SessionDb, Pattern),
851	    ?hcrd("select session", [{host_port,  HostPort},
852				     {scheme,     Scheme},
853				     {type,       SessionType},
854				     {candidates, Candidates}]),
855	    select_session(Candidates, MaxKeepAlive, MaxPipe, SessionType);
856	false ->
857	    no_connection
858    end.
859
860select_session(Candidates, Max, _, keep_alive) ->
861    select_session(Candidates, Max);
862select_session(Candidates, _, Max, pipeline) ->
863    select_session(Candidates, Max).
864
865select_session([] = _Candidates, _Max) ->
866    ?hcrd("select session - no candidate", []),
867    no_connection;
868select_session(Candidates, Max) ->
869    NewCandidates =
870	[{Pid, Length} || [Pid, Length] <- Candidates, Length =< Max],
871    case lists:keysort(2, NewCandidates) of
872	[] ->
873	    {no_session, length(Candidates)};
874	[{HandlerPid, _} | _] ->
875	    ?hcrd("select session - found one", [{handler, HandlerPid}]),
876	    {ok, HandlerPid}
877    end.
878
879pipeline_or_keep_alive(#request{id   = Id,
880				from = From} = Request,
881		       HandlerPid,
882		       #state{handler_db = HandlerDb} = State) ->
883    case httpc_handler:send(Request, HandlerPid) of
884	ok ->
885	    HandlerInfo = {Id, HandlerPid, From},
886	    ets:insert(HandlerDb, HandlerInfo);
887	{error, closed}  -> % timeout pipelining failed
888	    start_handler(Request, State)
889    end.
890
891is_inets_manager() ->
892    case get('$ancestors') of
893	[httpc_profile_sup | _] ->
894	    true;
895	_ ->
896	    false
897    end.
898
899generate_request_id(Request) ->
900    case Request#request.id of
901	undefined ->
902	    RequestId = make_ref(),
903	    Request#request{id = RequestId};
904	_ ->
905	    %% This is an automatic redirect or a retryed pipelined request
906	    %% => keep the old id.
907	    Request
908    end.
909
910handle_cookies(Request, #state{options = #options{cookies = disabled}}) ->
911    Request;
912handle_cookies(
913  #request{scheme  = Scheme,
914	   address = Address,
915	   path    = Path,
916	   headers = #http_request_h{other = Other} = Hdrs} = Request,
917  #state{cookie_db = CookieDb}) ->
918    case httpc_cookie:header(CookieDb, Scheme, Address, Path) of
919	{"cookie", ""} ->
920	    Request;
921	CookieHeader ->
922	    NewHeaders = Hdrs#http_request_h{other = [CookieHeader | Other]},
923	    Request#request{headers = NewHeaders}
924    end.
925
926do_store_cookies([], _) ->
927    ok;
928do_store_cookies([Cookie | Cookies], #state{cookie_db = CookieDb} = State) ->
929    ok = httpc_cookie:insert(CookieDb, Cookie),
930    do_store_cookies(Cookies, State).
931
932session_db_name(ProfileName) ->
933    make_db_name(ProfileName, "__session_db").
934
935cookie_db_name(ProfileName) ->
936    make_db_name(ProfileName, "__cookie_db").
937
938session_cookie_db_name(ProfileName) ->
939    make_db_name(ProfileName, "__session_cookie_db").
940
941handler_db_name(ProfileName) ->
942    make_db_name(ProfileName, "__handler_db").
943
944make_db_name(ProfileName, Post) ->
945    list_to_atom(atom_to_list(ProfileName) ++ Post).
946
947
948%%--------------------------------------------------------------------------
949%% These functions is just simple wrappers to parse specifically HTTP URIs
950%%--------------------------------------------------------------------------
951uri_parse(URI) ->
952    case uri_string:parse(uri_string:normalize(URI)) of
953        #{scheme := Scheme,
954          host := Host,
955          port := Port,
956          path := Path} ->
957            {ok, {Scheme, Host, Port, Path}};
958        #{scheme := Scheme,
959          host := Host,
960          path := Path} ->
961            {ok, {Scheme, Host, scheme_default_port(Scheme), Path}};
962        Other ->
963            {error, maybe_error(Other)}
964    end.
965
966maybe_error({error, Atom, Term}) ->
967    {Atom, Term};
968maybe_error(Other) ->
969    {unexpected, Other}.
970
971scheme_default_port("http") ->
972    80;
973scheme_default_port("https") ->
974    443.
975
976%%--------------------------------------------------------------------------
977
978
979call(ProfileName, Msg) ->
980    Timeout = infinity,
981    call(ProfileName, Msg, Timeout).
982call(ProfileName, Msg, Timeout) ->
983    gen_server:call(ProfileName, Msg, Timeout).
984
985cast(ProfileName, Msg) ->
986   gen_server:cast(ProfileName, Msg).
987
988
989get_option(proxy, #options{proxy = Proxy}) ->
990    Proxy;
991get_option(https_proxy, #options{https_proxy = Proxy}) ->
992    Proxy;
993get_option(pipeline_timeout, #options{pipeline_timeout = Timeout}) ->
994    Timeout;
995get_option(max_pipeline_length, #options{max_pipeline_length = Length}) ->
996    Length;
997get_option(keep_alive_timeout, #options{keep_alive_timeout = Timeout}) ->
998    Timeout;
999get_option(max_keep_alive_length, #options{max_keep_alive_length = Length}) ->
1000    Length;
1001get_option(max_sessions, #options{max_sessions = MaxSessions}) ->
1002    MaxSessions;
1003get_option(cookies, #options{cookies = Cookies}) ->
1004    Cookies;
1005get_option(verbose, #options{verbose = Verbose}) ->
1006    Verbose;
1007get_option(ipfamily, #options{ipfamily = IpFamily}) ->
1008    IpFamily;
1009get_option(ip, #options{ip = IP}) ->
1010    IP;
1011get_option(port, #options{port = Port}) ->
1012    Port;
1013get_option(socket_opts, #options{socket_opts = SocketOpts}) ->
1014    SocketOpts;
1015get_option(unix_socket, #options{unix_socket = UnixSocket}) ->
1016    UnixSocket.
1017
1018
1019get_proxy(Opts, #options{proxy = Default}) ->
1020    proplists:get_value(proxy, Opts, Default).
1021
1022get_https_proxy(Opts, #options{https_proxy = Default}) ->
1023    proplists:get_value(https_proxy, Opts, Default).
1024
1025get_pipeline_timeout(Opts, #options{pipeline_timeout = Default}) ->
1026    proplists:get_value(pipeline_timeout, Opts, Default).
1027
1028get_max_pipeline_length(Opts, #options{max_pipeline_length = Default}) ->
1029    proplists:get_value(max_pipeline_length, Opts, Default).
1030
1031get_max_keep_alive_length(Opts, #options{max_keep_alive_length = Default}) ->
1032    proplists:get_value(max_keep_alive_length, Opts, Default).
1033
1034get_keep_alive_timeout(Opts, #options{keep_alive_timeout = Default}) ->
1035    proplists:get_value(keep_alive_timeout, Opts, Default).
1036
1037get_max_sessions(Opts, #options{max_sessions = Default}) ->
1038    proplists:get_value(max_sessions, Opts, Default).
1039
1040get_cookies(Opts, #options{cookies = Default}) ->
1041    proplists:get_value(cookies, Opts, Default).
1042
1043get_ipfamily(Opts, #options{ipfamily = IpFamily}) ->
1044    case lists:keysearch(ipfamily, 1, Opts) of
1045	false ->
1046	    case proplists:get_value(ipv6, Opts) of
1047		enabled ->
1048		    inet6fb4;
1049		disabled ->
1050		    inet;
1051		_ ->
1052		    IpFamily
1053	    end;
1054	{value, {_, Value}} ->
1055	    Value
1056    end.
1057
1058get_ip(Opts, #options{ip = Default}) ->
1059    proplists:get_value(ip, Opts, Default).
1060
1061get_port(Opts, #options{port = Default}) ->
1062    proplists:get_value(port, Opts, Default).
1063
1064get_verbose(Opts, #options{verbose = Default}) ->
1065    proplists:get_value(verbose, Opts, Default).
1066
1067get_socket_opts(Opts, #options{socket_opts = Default}) ->
1068    proplists:get_value(socket_opts, Opts, Default).
1069
1070get_unix_socket_opts(Opts, #options{unix_socket = Default}) ->
1071    proplists:get_value(unix_socket, Opts, Default).
1072
1073handle_verbose(debug) ->
1074    dbg:p(self(), [call]),
1075    dbg:tp(?MODULE, [{'_', [], [{return_trace}]}]);
1076handle_verbose(trace) ->
1077    dbg:p(self(), [call]),
1078    dbg:tpl(?MODULE, [{'_', [], [{return_trace}]}]);
1079handle_verbose(_) ->
1080    ok.
1081
1082error_report(Profile, F, A) ->
1083    Report = io_lib:format("HTTPC-MANAGER<~p> " ++ F ++ "~n", [Profile | A]),
1084    error_logger:error_report(Report).
1085