1%% -*- erlang -*-
2%% %CopyrightBegin%
3%%
4%% Copyright Ericsson AB 2020. All Rights Reserved.
5%%
6%% Licensed under the Apache License, Version 2.0 (the "License");
7%% you may not use this file except in compliance with the License.
8%% You may obtain a copy of the License at
9%%
10%%     http://www.apache.org/licenses/LICENSE-2.0
11%%
12%% Unless required by applicable law or agreed to in writing, software
13%% distributed under the License is distributed on an "AS IS" BASIS,
14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15%% See the License for the specific language governing permissions and
16%% limitations under the License.
17%%
18%% %CopyrightEnd%
19%%----------------------------------------------------------------------
20%% File    : docgen_xml_to_chunk
21%%
22%% Created : 1 Nov 2018 by Kenneth Lundin <uabkeld@elxa31hr002>
23%%
24%% Does translation of Erlang XML docs to EEP-48 doc chunks.
25%%----------------------------------------------------------------------
26-module(docgen_xml_to_chunk).
27-export([main/1, func_to_tuple/1]).
28
29-include_lib("kernel/include/eep48.hrl").
30
31main([_Application, FromBeam, _Escript, ToChunk]) ->
32    %% The given module is not documented, generate a hidden beam chunk file
33    Name = filename:basename(filename:rootname(FromBeam)) ++ ".erl",
34
35    EmptyDocs = #docs_v1{ anno = erl_anno:set_file(Name, erl_anno:new(0)),
36                          module_doc = hidden, docs = []},
37    ok = file:write_file(ToChunk, term_to_binary(EmptyDocs,[compressed])),
38    ok;
39main([Application, FromXML, FromBeam, _Escript, ToChunk]) ->
40    _ = erlang:process_flag(max_heap_size,20 * 1000 * 1000),
41    case docs(Application, FromXML, FromBeam) of
42        {error, Reason} ->
43            io:format("Failed to create chunks: ~p~n",[Reason]),
44            erlang:halt(1);
45        {docs_v1,_,_,_,_,#{ source := S },[]} when
46              %% This is a list of all modules that do are known not have any functions
47              S =/= "../xml/gen_fsm.xml",
48              S =/= "../xml/shell_default.xml",
49              S =/= "../xml/user.xml",
50              S =/= "../xml/wxClipboardTextEvent.xml",
51              S =/= "../xml/wxDisplayChangedEvent.xml",
52              S =/= "../xml/wxGBSizerItem.xml",
53              S =/= "../xml/wxGraphicsBrush.xml",
54              S =/= "../xml/wxGraphicsFont.xml",
55              S =/= "../xml/wxGraphicsPen.xml",
56              S =/= "../xml/wxInitDialogEvent.xml",
57              S =/= "../xml/wxMaximizeEvent.xml",
58              S =/= "../xml/wxMouseCaptureLostEvent.xml",
59              S =/= "../xml/wxPaintEvent.xml",
60              S =/= "../xml/wxPreviewCanvas.xml",
61              S =/= "../xml/wxSysColourChangedEvent.xml",
62              S =/= "../xml/wxTaskBarIconEvent.xml",
63              S =/= "../xml/wxWindowCreateEvent.xml",
64              S =/= "../xml/wxWindowDestroyEvent.xml",
65              S =/= "../xml/wxDataObject.xml"
66              ->
67            io:format("Failed to create chunks: no functions found ~s~n",[S]),
68            erlang:halt(1),
69            ok;
70        Docs ->
71            ok = file:write_file(ToChunk, term_to_binary(Docs,[compressed]))
72    end.
73
74%% Error handling
75%%----------------------------------------------------------------------
76
77-define(error(Reason),
78	throw({dom_error, Reason})).
79
80%%----------------------------------------------------------------------
81
82%%======================================================================
83%% Records
84%%======================================================================
85
86%%----------------------------------------------------------------------
87%% State record for the validator
88%%----------------------------------------------------------------------
89-record(state, {
90	  tags=[],         %% Tag stack
91	  cno=[],          %% Current node number
92	  namespaces = [], %% NameSpace stack
93	  dom=[]           %% DOM structure
94	 }).
95
96%%======================================================================
97%% External functions
98%%======================================================================
99
100%%----------------------------------------------------------------------
101%% Function: initial_state() -> Result
102%% Parameters:
103%% Result:
104%% Description:
105%%----------------------------------------------------------------------
106initial_state() ->
107    #state{}.
108
109%%----------------------------------------------------------------------
110%% Function: get_dom(State) -> Result
111%% Parameters:
112%% Result:
113%% Description:
114%%----------------------------------------------------------------------
115get_dom(#state{dom=Dom}) ->
116    Dom.
117
118%%----------------------------------------------------------------------
119%% Function: event(Event, LineNo, State) -> Result
120%% Parameters:
121%% Result:
122%% Description:
123%%----------------------------------------------------------------------
124event(Event, _LineNo, State) ->
125    build_dom(Event, State).
126
127
128%%======================================================================
129%% Internal functions
130%%======================================================================
131
132%%----------------------------------------------------------------------
133%% Function  : build_dom(Event, State) -> Result
134%% Parameters: Event = term()
135%%             State = #xmerl_sax_simple_dom_state{}
136%% Result    : #xmerl_sax_simple_dom_state{} |
137%% Description:
138%%----------------------------------------------------------------------
139
140%% Document
141%%----------------------------------------------------------------------
142build_dom(startDocument, State) ->
143    State#state{dom=[startDocument]};
144build_dom(endDocument,
145	  #state{dom=[{Tag, Attributes, Content} |D]} = State) ->
146    case D of
147	[startDocument] ->
148	    State#state{dom=[{Tag, Attributes,
149                              lists:reverse(Content)}]};
150	[Decl, startDocument] ->
151	    State#state{dom=[Decl, {Tag, Attributes,
152                                    lists:reverse(Content)}]};
153	_ ->
154            %% endDocument is also sent by the parser when a fault occur to tell
155            %% the event receiver that no more input will be sent
156	    State
157    end;
158
159%% Element
160%%----------------------------------------------------------------------
161build_dom({startElement, _Uri, LocalName, _QName, Attributes},
162	  #state{tags=T, dom=D} = State) ->
163
164    A = parse_attributes(LocalName, Attributes),
165    CName = list_to_atom(LocalName),
166
167    State#state{tags=[CName |T],
168                dom=[{CName,
169                      lists:reverse(A),
170                      []
171                     } | D]};
172build_dom({endElement, _Uri, LocalName, _QName},
173	  #state{tags=[_ |T],
174                 dom=[{CName, CAttributes, CContent},
175                      {PName, PAttributes, PContent} = _Parent | D]} = State) ->
176    case list_to_atom(LocalName) of
177	CName ->
178            SectionDepth = length([E || E <- T, E =:= section]),
179            MappedCName =
180                case CName of
181                    title ->
182                        lists:nth(SectionDepth+1,[h1,h2,h3,h4,h5,h6]);
183                    section when SectionDepth > 0 ->
184                        'div';
185                    CName -> CName
186                end,
187
188            State#state{tags=T,
189                        dom=[{PName, PAttributes,
190                              [{MappedCName, CAttributes,
191                                lists:reverse(CContent)}
192                               |PContent]
193                             } | D]};
194        _ ->
195            ?error("Got end of element: " ++ LocalName ++ " but expected: " ++
196                       CName)
197    end;
198
199%% Text
200%%----------------------------------------------------------------------
201build_dom({characters, String},
202	  #state{dom=[{Name, Attributes, Content}| D]} = State) ->
203    HtmlEnts = [{"&nbsp;",[160]},
204                {"&times;",[215]},
205                {"&plusmn;",[177]},
206                {"&ouml;","ö"},
207                {"&auml;","ä"},
208                {"&aring;","å"}
209               ],
210
211    NoHtmlEnt =
212        lists:foldl(
213          fun({Pat,Sub},Str) ->
214                  re:replace(Str,Pat,Sub,[global,unicode])
215          end,String,HtmlEnts),
216
217    case re:run(NoHtmlEnt,"&[a-z]*;",[{capture,first,binary},unicode]) of
218        nomatch -> ok;
219        {match,[<<"&lt;">>]} -> ok;
220        {match,[<<"&gt;">>]} -> ok;
221        Else -> throw({found_illigal_thing,Else,String})
222    end,
223    NewContent =
224        [unicode:characters_to_binary(NoHtmlEnt,utf8)| Content],
225    State#state{dom=[{Name, Attributes, NewContent} | D]};
226
227build_dom({ignorableWhitespace, String},
228          #state{dom=[{Name,_,_} = _E|_]} = State) ->
229    case lists:member(Name,
230                      [p,pre,input,code,quote,warning,
231                       note,dont,do,c,b,i,em,strong,
232                       seemfa,seeerl,seetype,seeapp,
233                       seecom,seecref,seefile,seeguide,
234                       tag,item]) of
235        true ->
236%            io:format("Keep ign white: ~p ~p~n",[String, _E]),
237            build_dom({characters, String}, State);
238        false ->
239            State
240    end;
241
242build_dom({startEntity, SysId}, State) ->
243    io:format("startEntity:~p~n",[SysId]),
244    State;
245
246%% Default
247%%----------------------------------------------------------------------
248build_dom(_E, State) ->
249    State.
250
251%%----------------------------------------------------------------------
252%% Function  : parse_attributes(ElName, Attributes) -> Result
253%% Parameters:
254%% Result    :
255%% Description:
256%%----------------------------------------------------------------------
257parse_attributes(ElName, Attributes) ->
258    parse_attributes(ElName, Attributes, 1, []).
259
260parse_attributes(_, [], _, Acc) ->
261    Acc;
262parse_attributes(ElName, [{_Uri, _Prefix, LocalName, AttrValue} |As], N, Acc) ->
263    parse_attributes(ElName, As, N+1, [{list_to_atom(LocalName), AttrValue} |Acc]).
264
265docs(Application, OTPXml, FromBEAM)->
266    case xmerl_sax_parser:file(OTPXml,
267                               [skip_external_dtd,
268                                {event_fun,fun event/3},
269                                {event_state,initial_state()}]) of
270        {ok,Tree,_} ->
271            {ok, {Module, Chunks}} = beam_lib:chunks(FromBEAM,[exports,abstract_code]),
272            Dom = get_dom(Tree),
273            put(application, Application),
274            put(module, filename:basename(filename:rootname(FromBEAM))),
275            NewDom = transform(Dom,[]),
276            Chunk = to_chunk(NewDom, OTPXml, Module, proplists:get_value(abstract_code, Chunks)),
277            verify_chunk(Module,proplists:get_value(exports, Chunks), Chunk),
278            Chunk;
279        Else ->
280            {error,Else}
281    end.
282
283verify_chunk(M, Exports, #docs_v1{ docs = Docs } = Doc) ->
284
285    %% Make sure that each documented function actually is exported
286    Exported = [begin
287                    FA = {F,A},
288                    {M,F,A,lists:member(FA,Exports)}
289                end || {{function,F,A},_,_,_,_} <- Docs],
290    lists:foreach(fun({_M,_F,_A,true}) ->
291                          ok
292                  end,Exported),
293
294    try
295        shell_docs:validate(Doc)
296    catch Err ->
297            throw({maps:get(<<"en">>,Doc#docs_v1.module_doc), Err})
298    end.
299
300%% skip <erlref> but transform and keep its content
301transform([{erlref,_Attr,Content}|T],Acc) ->
302    Module = [Mod || Mod = {module,_,_} <- Content],
303    NewContent = Content -- Module,
304    [{module,SinceAttr,[Mname]}] = Module,
305    Since = case proplists:get_value(since,SinceAttr) of
306                undefined -> [];
307                [] -> [];
308                Vsn -> [{since,Vsn}]
309            end,
310    transform([{module,[{name,Mname}|Since],NewContent}|T],Acc);
311
312%% skip <header> and all of its content
313transform([{header,_Attr,_Content}|T],Acc) ->
314    transform(T,Acc);
315transform([{section,Attr,Content}|T],Acc) ->
316    transform(T,[{section,Attr,transform(Content,[])}|Acc]);
317
318%% transform <list><item> to <ul><li> or <ol><li> depending on type attribute
319transform([{list,Attr,Content}|T],Acc) ->
320    transform([transform_list(Attr,Content)|T],Acc);
321
322%% transform <taglist>(tag,item+)+ to <dl>(dt,item+)+
323transform([{taglist,Attr,Content}|T],Acc) ->
324    transform([transform_taglist(Attr,Content)|T],Acc);
325
326%% remove <anno> as it is only used to validate specs vs xml src
327transform([{anno,[],Content}|T],Acc) ->
328    transform([Content|T],Acc);
329
330%% transform <c> to <code>
331transform([{c,[],Content}|T],Acc) ->
332    transform(T, [{code,[],transform(Content,[])}|Acc]);
333
334%% transform <code> to <pre><code>
335transform([{code,Attr,Content}|T],Acc) ->
336    transform(T, [{pre,[],[{code,a2b(Attr),transform(Content,[])}]}|Acc]);
337%% transform <pre> to <pre><code>
338transform([{pre,Attr,Content}|T],Acc) ->
339    transform(T, [{pre,[],[{code,Attr,transform(Content,[])}]}|Acc]);
340
341%% transform <funcs> with <func> as children
342transform([{funcs,_Attr,Content}|T],Acc) ->
343    Fns = {functions,[],transform_funcs(Content, [])},
344    transform(T,[Fns|Acc]);
345%% transform <datatypes> with <datatype> as children
346transform([{datatypes,_Attr,Content}|T],Acc) ->
347    Dts = transform(Content, []),
348    transform(T,[{datatypes,[],Dts}|Acc]);
349transform([{datatype,_Attr,Content}|T],Acc) ->
350    transform(T,transform_datatype(Content, []) ++ Acc);
351%% Ignore <datatype_title>
352transform([{datatype_title,_Attr,_Content}|T],Acc) ->
353    transform(T,Acc);
354%% transform <desc>Content</desc> to Content
355transform([{desc,_Attr,Content}|T],Acc) ->
356    transform(T,[transform(Content,[])|Acc]);
357%% transform <marker id="name"/>  to <a id="name"/>....
358transform([{marker,Attrs,Content}|T],Acc) ->
359    transform(T,[{a,a2b(Attrs),transform(Content,[])}|Acc]);
360%% transform <url href="external URL"> Content</url> to <a href....
361transform([{url,Attrs,Content}|T],Acc) ->
362    transform(T,[{a,a2b(Attrs),transform(Content,[])}|Acc]);
363%% transform note/warning/do/don't to <p class="thing">
364transform([{What,[],Content}|T],Acc)
365  when What =:= note; What =:= warning; What =:= do; What =:= dont ->
366    WhatP = {'div',[{class,atom_to_binary(What)}], transform(Content,[])},
367    transform(T,[WhatP|Acc]);
368
369transform([{type,_,[]}|_] = Dom,Acc) ->
370    %% Types are laid out sequentially in the source xml so we need to
371    %% parse them like that here too.
372    case transform_types(Dom,[]) of
373        {[],T} ->
374            transform(T,Acc);
375        {Types,T} ->
376            %% We sort the types here because in the source xml
377            %% the description and the declaration do not have
378            %% to be next to each other. But we want to have that
379            %% for the doc chunks.
380            NameSort = fun({li,A,_},{li,B,_}) ->
381                               NameA = proplists:get_value(name,A),
382                               NameB = proplists:get_value(name,B),
383                               if NameA == NameB ->
384                                       length(A) =< length(B);
385                                  true ->
386                                       NameA < NameB
387                               end
388                       end,
389            transform(T,[{ul,[{class,<<"types">>}],lists:sort(NameSort,Types)}|Acc])
390    end;
391transform([{type_desc,Attr,_Content}|T],Acc) ->
392    %% We skip any type_desc with the variable attribute
393    true = proplists:is_defined(variable, Attr),
394    transform(T,Acc);
395transform([{type,[],Content}|T],Acc) ->
396    transform(T,[{ul,[{class,<<"types">>}],transform(Content,[])}|Acc]);
397transform([{v,[],Content}|T],Acc) ->
398    transform(T, [{li,[{class,<<"type">>}],transform(Content,[])}|Acc]);
399transform([{d,[],Content}|T],Acc) ->
400    transform(T, [{li,[{class,<<"description">>}],transform(Content,[])}|Acc]);
401
402transform([Elem = {See,_Attr,_Content}|T],Acc)
403  when See =:= seemfa; See =:= seeerl; See =:= seetype; See =:= seeapp;
404       See =:= seecom; See =:= seecref; See =:= seefile; See =:= seeguide ->
405    transform([transform_see(Elem)|T],Acc);
406
407transform([{term,Attr,[]}|T],Acc) ->
408    transform([list_to_binary(proplists:get_value(id,Attr))|T],Acc);
409
410transform([{fsummary,_,_}|T],Acc) ->
411    %% We skip fsummary as it many times is just a duplicate of the
412    %% first line of the docs.
413    transform(T,Acc);
414
415transform([{input,_,Content}|T],Acc) ->
416    %% Just remove input as it is not used by anything
417    transform(T,[transform(Content,[])|Acc]);
418
419transform([{p,Attr,Content}|T],Acc) ->
420    transform(T,[{p,a2b(Attr),transform(Content,[])}|Acc]);
421transform([{'div',Attr,Content}|T],Acc) ->
422    transform(T,[{'div',a2b(Attr),transform(Content,[])}|Acc]);
423
424%% Tag and Attr is used as is but Content is transformed
425transform([{Tag,Attr,Content}|T],Acc) ->
426    transform(T,[{Tag,Attr,transform(Content,[])}|Acc]);
427transform([Binary|T],Acc) ->
428    transform(T,[Binary|Acc]);
429transform([],Acc) ->
430    lists:flatten(lists:reverse(Acc)).
431
432transform_list([{type,"ordered"}],Content) ->
433    {ol,[],[{li,A2,C2}||{item,A2,C2}<-Content]};
434transform_list(_,Content) ->
435    {ul,[],[{li,A2,C2}||{item,A2,C2}<-Content]}.
436
437transform_types([{type,Attr,[]}|T],Acc) ->
438    case proplists:is_defined(name,Attr) of
439        true ->
440            transform_types(T, [{li,a2b(Attr),[]}|Acc]);
441        false ->
442            true = proplists:is_defined(variable, Attr),
443            transform_types(T, Acc)
444    end;
445transform_types([{type_desc,Attr,Content}|T],Acc) ->
446    case proplists:is_defined(name,Attr) of
447        true ->
448            TypeDesc = transform(Content,[]),
449            transform_types(T, [{li,a2b(Attr) ++ [{class,<<"description">>}],TypeDesc}|Acc]);
450        false ->
451            true = proplists:is_defined(variable, Attr),
452            transform_types(T, Acc)
453    end;
454transform_types([{type,_,_}|_T],_Acc) ->
455    throw(mixed_type_declarations);
456transform_types(Dom,Acc) ->
457    {lists:reverse(Acc),Dom}.
458
459transform_taglist(Attr,Content) ->
460    Items =
461        lists:map(fun({tag,A,C}) ->
462                          {dt,A,C};
463                     ({item,A,C}) ->
464                          {dd,A,C}
465                  end, Content),
466    {dl,Attr,Items}.
467
468%% if we have {func,[],[{name,...},{name,....},...]}
469%% we convert it to one {func,[],[{name,...}] per arity lowest first.
470transform_funcs([Func|T],Acc) ->
471    transform_funcs(T,func2func(Func) ++ Acc);
472transform_funcs([],Acc) ->
473    lists:reverse(Acc).
474
475func2func({fsdescription,_Attr,_Contents}) ->
476    [];
477func2func({func,Attr,Contents}) ->
478
479    ContentsNoName = [NC||NC <- Contents, element(1,NC) /= name],
480
481    EditLink =
482        case proplists:get_value(ghlink,Attr) of
483            undefined ->
484                #{};
485            GhLink ->
486                #{ edit_url =>
487                       iolist_to_binary(["https://github.com/erlang/otp/edit/",GhLink]) }
488        end,
489
490    VerifyNameList =
491        fun(NameList, Test) ->
492                %% Assert that we don't mix ways to write <name>
493                [begin
494                     ok = Test(C),
495                     {proplists:get_value(name,T),proplists:get_value(arity,T)}
496                 end || {name,T,C} <- NameList]
497        end,
498
499    NameList = [Name || {name,_,_} = Name <- Contents],
500
501    %% "Since" is hard to accurately as there can be multiple <name> per <func> and they
502    %% can refer to the same or other arities. This should be improved in the future but
503    %% for now we set since to a comma separated list of all since attributes.
504    SinceMD =
505        case [proplists:get_value(since, SinceAttr) ||
506                 {name,SinceAttr,_} <- NameList, proplists:get_value(since, SinceAttr) =/= []] of
507            [] -> EditLink;
508            Sinces ->
509                EditLink#{ since => unicode:characters_to_binary(
510                                      lists:join(",",lists:usort(Sinces))) }
511        end,
512
513    Functions =
514        case NameList of
515            [{name,_,[]}|_] ->
516                %% Spec style function docs
517                TagsToFA =
518                    fun(Tags) ->
519                            {proplists:get_value(name,Tags),
520                             proplists:get_value(arity,Tags)}
521                    end,
522
523                _ = VerifyNameList(NameList,fun([]) -> ok end),
524
525                FAs = [TagsToFA(FAttr) || {name,FAttr,[]} <- NameList ],
526                SortedFAs = lists:usort(FAs),
527                FAClauses = lists:usort([{TagsToFA(FAttr),proplists:get_value(clause_i,FAttr)}
528                                         || {name,FAttr,[]} <- NameList ]),
529
530                MakeFunc = fun({F,A}, MD, Doc) ->
531                                   Specs = [begin
532                                                {function,Name} = func_to_atom(CF),
533                                                {Name,list_to_integer(CA),C}
534                                            end || {{CF,CA},C} <- FAClauses,
535                                                   F =:= CF, A =:= CA],
536                                   {function,[{name,F},{arity,list_to_integer(A)},
537                                              {signature,[iolist_to_binary([F,"/",A])]},
538                                              {meta,MD#{ signature => Specs }}],
539                                    Doc}
540                           end,
541
542                Base = MakeFunc(hd(SortedFAs), SinceMD, ContentsNoName),
543
544                {BaseF,BaseA} = hd(SortedFAs),
545                MD = SinceMD#{ equiv => {function,list_to_atom(BaseF),list_to_integer(BaseA)}},
546                Equiv = lists:map(
547                          fun(FA) ->
548                                  MakeFunc(FA, MD, [])
549                          end, tl(SortedFAs)),
550                [Base | Equiv];
551            NameList ->
552                %% Manual style function docs
553                FAs = lists:foldl(
554                        fun({name,_,NameString}, Acc) ->
555                                FAs = func_to_tuple(NameString),
556                                lists:foldl(
557                                  fun(FA, FAAcc) ->
558                                          Slogan = maps:get(FA, FAAcc, []),
559                                          FAAcc#{ FA => [strip_tags(NameString)|Slogan] }
560                                  end, Acc, FAs)
561                        end, #{}, NameList),
562
563                _ = VerifyNameList(NameList,fun([_|_]) -> ok end),
564
565                SortedFAs = lists:usort(maps:to_list(FAs)),
566
567                {{BaseF, BaseA}, BaseSig} = hd(SortedFAs),
568
569                Base = {function,[{name,BaseF},{arity,BaseA},
570                                  {signature,BaseSig},
571                                  {meta,SinceMD}],
572                        ContentsNoName},
573
574                Equiv = [{function,
575                          [{name,F},{arity,A},
576                           {signature,Signature},
577                           {meta,SinceMD#{ equiv => {function,list_to_atom(BaseF),BaseA}}}],[]}
578                         || {{F,A},Signature} <- tl(SortedFAs)],
579                [Base | Equiv]
580        end,
581    transform(Functions,[]).
582
583func_to_tuple(Chars) ->
584    try
585        [Name,Args] = string:split(strip_tags(Chars),"("),
586        Arities = parse_args(unicode:characters_to_list(Args)),
587        [{unicode:characters_to_list(Name),Arity} || Arity <- Arities]
588    catch E:R:ST ->
589            io:format("Failed to parse: ~p~n",[Chars]),
590            erlang:raise(E,R,ST)
591    end.
592
593%% This function parses a documentation <name> attribute to figure
594%% out the arities if that function. Example:
595%%    "start([go,Mode] [,Extra])" returns [1, 2].
596%%
597%% This assumes that when a single <name> describes many arities
598%% the arities are listed with [, syntax.
599parse_args(")" ++ _) ->
600    [0];
601parse_args(Args) ->
602    parse_args(unicode:characters_to_list(Args),1,[]).
603parse_args([$[,$,|T],Arity,[]) ->
604    parse_args(T,Arity,[$[]) ++ parse_args(T,Arity+1,[]);
605parse_args([$,|T],Arity,[]) ->
606    parse_args(T,Arity+1,[]);
607parse_args([Open|T],Arity,Stack)
608  when Open =:= $[; Open =:= ${; Open =:= $( ->
609    parse_args(T,Arity,[Open|Stack]);
610parse_args([$]|T],Arity,[$[|Stack]) ->
611    parse_args(T,Arity,Stack);
612parse_args([$}|T],Arity,[${|Stack]) ->
613    parse_args(T,Arity,Stack);
614parse_args([$)|T],Arity,[$(|Stack]) ->
615    parse_args(T,Arity,Stack);
616parse_args([$)|_T],Arity,[]) ->
617    [Arity];
618parse_args([_H|T],Arity,Stack) ->
619    parse_args(T,Arity,Stack).
620
621strip_tags([{_Tag,_Attr,Content}|T]) ->
622    [Content | strip_tags(T)];
623strip_tags([H|T]) when not is_tuple(H) ->
624    [H | strip_tags(T)];
625strip_tags([]) ->
626    [].
627
628transform_datatype(Dom,_Acc) ->
629    ContentsNoName = transform([NC||NC <- Dom, element(1,NC) /= name],[]),
630    [case N of
631          {name,NameAttr,[]} ->
632              {datatype,NameAttr,ContentsNoName};
633          {name,[],Content} ->
634              [{Name,Arity}] = func_to_tuple(Content),
635              Signature = strip_tags(Content),
636              {datatype,[{name,Name},{n_vars,integer_to_list(Arity)},
637                         {signature,Signature}],ContentsNoName}
638      end || N = {name,_,_} <- Dom].
639
640transform_see({See,[{marker,Marker}],Content}) ->
641    AbsMarker =
642        case string:split(Marker, "#") of
643            [AppFile] -> marker_defaults(AppFile);
644            [AppFile, Anchor] -> [marker_defaults(AppFile), "#", Anchor]
645        end,
646
647    {a, [{href,iolist_to_binary(AbsMarker)},
648         {rel,<<"https://erlang.org/doc/link/",(atom_to_binary(See))/binary>>}], Content}.
649
650marker_defaults("") ->
651    [get(application), ":", get(module)];
652marker_defaults(AppFile) ->
653    case string:split(AppFile, ":") of
654        [File] -> [get(application), ":", File];
655        [App, File] -> [App, ":", File]
656    end.
657
658to_chunk(Dom, Source, Module, AST) ->
659    [{module,MAttr,Mcontent}] = Dom,
660
661    ModuleDocs = lists:flatmap(
662                   fun({Tag,_,Content}) when Tag =:= description;
663                                             Tag =:= section ->
664                           Content;
665                      ({_,_,_}) ->
666                           []
667                   end, Mcontent),
668
669    TypeMeta = add_types(AST, maps:from_list([{source,Source}|MAttr])),
670
671    TypeMap = maps:get(types, TypeMeta, []),
672
673    Anno = erl_anno:set_file(atom_to_list(Module)++".erl",erl_anno:new(0)),
674
675    Types = lists:flatten([Types || {datatypes,[],Types} <- Mcontent]),
676
677    TypeEntries =
678        lists:map(
679          fun({datatype,Attr,Descr}) ->
680                  {function, TypeName} = func_to_atom(proplists:get_value(name,Attr)),
681                  TypeArity = case proplists:get_value(n_vars,Attr) of
682                                  undefined ->
683                                      find_type_arity(TypeName, TypeMap);
684                                  Arity ->
685                                      list_to_integer(Arity)
686                              end,
687                  TypeArgs = lists:join(",",[lists:concat(["Arg",I]) || I <- lists:seq(1,TypeArity)]),
688                  PlaceholderSig = io_lib:format("-type ~p(~s) :: term().",[TypeName,TypeArgs]),
689                  TypeSignature = proplists:get_value(
690                                    signature,Attr,[iolist_to_binary(PlaceholderSig)]),
691                  MetaSig =
692                      case maps:get({TypeName, TypeArity}, TypeMap, undefined) of
693                          undefined ->
694                              #{};
695                          Sig ->
696                              #{ signature => [Sig] }
697                      end,
698
699                  MetaDepr
700                      = case apply(otp_internal,obsolete_type,[Module, TypeName, TypeArity]) of
701                            %% apply/3 in order to silence dialyzer
702                            {deprecated, Text} ->
703                                MetaSig#{ deprecated =>
704                                              unicode:characters_to_binary(
705                                                erl_lint:format_error({deprecated_type,{Module,TypeName,TypeArity}, Text})) };
706                            {deprecated, Replacement, Rel} ->
707                                MetaSig#{ deprecated =>
708                                              unicode:characters_to_binary(
709                                                erl_lint:format_error({deprecated_type,{Module,TypeName,TypeArity}, Replacement, Rel})) };
710                            {removed, _Text} ->
711                                %% Just skip
712                                MetaSig;
713                            no ->
714                                MetaSig
715                        end,
716
717                  docs_v1_entry(type, Anno, TypeName, TypeArity, TypeSignature, MetaDepr, Descr)
718          end, Types),
719
720    Functions = lists:flatten([Functions || {functions,[],Functions} <- Mcontent]),
721
722    FuncEntrys =
723        lists:map(
724          fun({function,Attr,Fdoc}) ->
725                  {Type, Name} = func_to_atom(proplists:get_value(name,Attr)),
726                  Arity = proplists:get_value(arity,Attr),
727                  Signature = proplists:get_value(signature,Attr),
728                  FMeta = proplists:get_value(meta,Attr),
729                  MetaWSpec = add_spec(AST,FMeta),
730                  MetaDepr
731                      = case apply(otp_internal,obsolete,[Module, Name, Arity]) of
732                            %% apply/3 in order to silence dialyzer
733                            {deprecated, Text} ->
734                                MetaWSpec#{ deprecated =>
735                                                unicode:characters_to_binary(
736                                                  erl_lint:format_error({deprecated,{Module,Name,Arity}, Text})) };
737                            {deprecated, Replacement, Rel} ->
738                                MetaWSpec#{ deprecated =>
739                                                unicode:characters_to_binary(
740                                                  erl_lint:format_error({deprecated,{Module,Name,Arity}, Replacement, Rel})) };
741                            _ -> MetaWSpec
742                        end,
743                  docs_v1_entry(Type, Anno, Name, Arity, Signature, MetaDepr, Fdoc)
744          end, Functions),
745
746    docs_v1(ModuleDocs, Anno, TypeMeta, FuncEntrys ++ TypeEntries).
747
748docs_v1(DocContents, Anno, Metadata, Docs) ->
749    #docs_v1{ anno = Anno,
750              module_doc = #{<<"en">> => shell_docs:normalize(DocContents)},
751              metadata = maps:merge(Metadata, (#docs_v1{})#docs_v1.metadata),
752              docs = Docs }.
753
754docs_v1_entry(Kind, Anno, Name, Arity, Signature, Metadata, DocContents) ->
755
756    AnnoWLine =
757        case Metadata of
758            #{ signature := [Sig|_] } ->
759                SigAnno = element(2, Sig),
760                erl_anno:set_line(erl_anno:line(SigAnno), Anno);
761            _NoSignature ->
762                Anno
763        end,
764
765    Doc =
766        case DocContents of
767              [] ->
768                  #{};
769              DocContents ->
770                  #{ <<"en">> => shell_docs:normalize(DocContents) }
771          end,
772
773    {{Kind, Name, Arity}, AnnoWLine, lists:flatten(Signature), Doc, Metadata}.
774
775%% A special list_to_atom that handles
776%%  'and'
777%%  Destroy
778%%  'begin'
779func_to_atom(List) ->
780    case erl_scan:string(List) of
781        {ok,[{atom,_,Fn}],_} ->
782            {function, Fn};
783        {ok,[{var,_,Fn}],_} ->
784            {function, Fn};
785        {ok,[{Fn,_}],_} ->
786            {function, Fn};
787        {ok,[{var,_,_},{':',_},{atom,_,Fn}],_} ->
788            {callback, Fn};
789        {ok,[{var,_,_},{':',_},{var,_,Fn}],_} ->
790            {callback, Fn}
791    end.
792
793-define(IS_TYPE(TO),(TO =:= type orelse TO =:= opaque)).
794
795add_spec(no_abstract_code, Meta) ->
796    Meta;
797add_spec({raw_abstract_v1, AST}, Meta = #{ signature := Specs } ) ->
798    Meta#{ signature := add_spec_clauses(AST, merge_clauses(Specs,#{})) };
799add_spec(_, Meta) ->
800    Meta.
801
802add_types(no_abstract_code, Meta) ->
803    Meta;
804add_types({raw_abstract_v1, AST}, Meta) ->
805    Meta#{ types =>
806               maps:from_list(
807                 [{{Name,length(Args)},T} || T = {attribute,_,TO,{Name, _, Args}} <- AST,
808                                             ?IS_TYPE(TO)]) }.
809
810add_spec_clauses(AST, [{{F,A},Clauses}|T]) ->
811    [filter_clauses(find_spec(AST,F,A),Clauses) | add_spec_clauses(AST,T)];
812add_spec_clauses(_AST, []) ->
813    [].
814
815filter_clauses(Spec,[undefined]) ->
816    Spec;
817filter_clauses({attribute,Ln,spec,{FA,Clauses}},ClauseIds) ->
818    {_,FilteredClauses} =
819        lists:foldl(
820          fun({TO,_,_,_} = C,{Cnt,Acc}) when ?IS_TYPE(TO) ->
821                  case lists:member(integer_to_list(Cnt),ClauseIds) of
822                      true ->
823                          {Cnt+1,[C | Acc]};
824                      false ->
825                          {Cnt+1,Acc}
826                  end
827          end, {1, []}, Clauses),
828    {attribute,Ln,spec,{FA,lists:reverse(FilteredClauses)}}.
829
830merge_clauses([{F,A,Clause}|T],Acc) ->
831    merge_clauses(T,Acc#{ {F,A} => [Clause | maps:get({F,A},Acc,[])]});
832merge_clauses([],Acc) ->
833    maps:to_list(Acc).
834
835find_type_arity(Name, [{{Name,_},{attribute,_,TO,{Name,_,Args}}}|_T]) when ?IS_TYPE(TO) ->
836    length(Args);
837find_type_arity(Name, [_|T]) ->
838    find_type_arity(Name,T);
839find_type_arity(Name, Map) when is_map(Map) ->
840    find_type_arity(Name, maps:to_list(Map)).
841
842find_spec(AST, Func, Arity) ->
843    Specs = lists:filter(fun({attribute,_,spec,{{F,A},_}}) ->
844                                 F =:= Func andalso A =:= Arity;
845                            ({attribute,_,spec,{{_,F,A},_}}) ->
846                                 F =:= Func andalso A =:= Arity;
847                            (_) ->
848                                 false
849                         end, AST),
850    case Specs of
851        [S] ->
852            S;
853        [] ->
854            io:format("Could not find spec for ~p/~p~n",[Func,Arity]),
855            exit(1)
856    end.
857
858a2b(Attrs) ->
859    [{Tag,unicode:characters_to_binary(Value)} || {Tag,Value} <- Attrs].
860