1%% -*- erlang -*- 2%% %CopyrightBegin% 3%% 4%% Copyright Ericsson AB 2020. All Rights Reserved. 5%% 6%% Licensed under the Apache License, Version 2.0 (the "License"); 7%% you may not use this file except in compliance with the License. 8%% You may obtain a copy of the License at 9%% 10%% http://www.apache.org/licenses/LICENSE-2.0 11%% 12%% Unless required by applicable law or agreed to in writing, software 13%% distributed under the License is distributed on an "AS IS" BASIS, 14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15%% See the License for the specific language governing permissions and 16%% limitations under the License. 17%% 18%% %CopyrightEnd% 19%%---------------------------------------------------------------------- 20%% File : docgen_xml_to_chunk 21%% 22%% Created : 1 Nov 2018 by Kenneth Lundin <uabkeld@elxa31hr002> 23%% 24%% Does translation of Erlang XML docs to EEP-48 doc chunks. 25%%---------------------------------------------------------------------- 26-module(docgen_xml_to_chunk). 27-export([main/1, func_to_tuple/1]). 28 29-include_lib("kernel/include/eep48.hrl"). 30 31main([_Application, FromBeam, _Escript, ToChunk]) -> 32 %% The given module is not documented, generate a hidden beam chunk file 33 Name = filename:basename(filename:rootname(FromBeam)) ++ ".erl", 34 35 EmptyDocs = #docs_v1{ anno = erl_anno:set_file(Name, erl_anno:new(0)), 36 module_doc = hidden, docs = []}, 37 ok = file:write_file(ToChunk, term_to_binary(EmptyDocs,[compressed])), 38 ok; 39main([Application, FromXML, FromBeam, _Escript, ToChunk]) -> 40 _ = erlang:process_flag(max_heap_size,20 * 1000 * 1000), 41 case docs(Application, FromXML, FromBeam) of 42 {error, Reason} -> 43 io:format("Failed to create chunks: ~p~n",[Reason]), 44 erlang:halt(1); 45 {docs_v1,_,_,_,_,#{ source := S },[]} when 46 %% This is a list of all modules that do are known not have any functions 47 S =/= "../xml/gen_fsm.xml", 48 S =/= "../xml/shell_default.xml", 49 S =/= "../xml/user.xml", 50 S =/= "../xml/wxClipboardTextEvent.xml", 51 S =/= "../xml/wxDisplayChangedEvent.xml", 52 S =/= "../xml/wxGBSizerItem.xml", 53 S =/= "../xml/wxGraphicsBrush.xml", 54 S =/= "../xml/wxGraphicsFont.xml", 55 S =/= "../xml/wxGraphicsPen.xml", 56 S =/= "../xml/wxInitDialogEvent.xml", 57 S =/= "../xml/wxMaximizeEvent.xml", 58 S =/= "../xml/wxMouseCaptureLostEvent.xml", 59 S =/= "../xml/wxPaintEvent.xml", 60 S =/= "../xml/wxPreviewCanvas.xml", 61 S =/= "../xml/wxSysColourChangedEvent.xml", 62 S =/= "../xml/wxTaskBarIconEvent.xml", 63 S =/= "../xml/wxWindowCreateEvent.xml", 64 S =/= "../xml/wxWindowDestroyEvent.xml", 65 S =/= "../xml/wxDataObject.xml" 66 -> 67 io:format("Failed to create chunks: no functions found ~s~n",[S]), 68 erlang:halt(1), 69 ok; 70 Docs -> 71 ok = file:write_file(ToChunk, term_to_binary(Docs,[compressed])) 72 end. 73 74%% Error handling 75%%---------------------------------------------------------------------- 76 77-define(error(Reason), 78 throw({dom_error, Reason})). 79 80%%---------------------------------------------------------------------- 81 82%%====================================================================== 83%% Records 84%%====================================================================== 85 86%%---------------------------------------------------------------------- 87%% State record for the validator 88%%---------------------------------------------------------------------- 89-record(state, { 90 tags=[], %% Tag stack 91 cno=[], %% Current node number 92 namespaces = [], %% NameSpace stack 93 dom=[] %% DOM structure 94 }). 95 96%%====================================================================== 97%% External functions 98%%====================================================================== 99 100%%---------------------------------------------------------------------- 101%% Function: initial_state() -> Result 102%% Parameters: 103%% Result: 104%% Description: 105%%---------------------------------------------------------------------- 106initial_state() -> 107 #state{}. 108 109%%---------------------------------------------------------------------- 110%% Function: get_dom(State) -> Result 111%% Parameters: 112%% Result: 113%% Description: 114%%---------------------------------------------------------------------- 115get_dom(#state{dom=Dom}) -> 116 Dom. 117 118%%---------------------------------------------------------------------- 119%% Function: event(Event, LineNo, State) -> Result 120%% Parameters: 121%% Result: 122%% Description: 123%%---------------------------------------------------------------------- 124event(Event, _LineNo, State) -> 125 build_dom(Event, State). 126 127 128%%====================================================================== 129%% Internal functions 130%%====================================================================== 131 132%%---------------------------------------------------------------------- 133%% Function : build_dom(Event, State) -> Result 134%% Parameters: Event = term() 135%% State = #xmerl_sax_simple_dom_state{} 136%% Result : #xmerl_sax_simple_dom_state{} | 137%% Description: 138%%---------------------------------------------------------------------- 139 140%% Document 141%%---------------------------------------------------------------------- 142build_dom(startDocument, State) -> 143 State#state{dom=[startDocument]}; 144build_dom(endDocument, 145 #state{dom=[{Tag, Attributes, Content} |D]} = State) -> 146 case D of 147 [startDocument] -> 148 State#state{dom=[{Tag, Attributes, 149 lists:reverse(Content)}]}; 150 [Decl, startDocument] -> 151 State#state{dom=[Decl, {Tag, Attributes, 152 lists:reverse(Content)}]}; 153 _ -> 154 %% endDocument is also sent by the parser when a fault occur to tell 155 %% the event receiver that no more input will be sent 156 State 157 end; 158 159%% Element 160%%---------------------------------------------------------------------- 161build_dom({startElement, _Uri, LocalName, _QName, Attributes}, 162 #state{tags=T, dom=D} = State) -> 163 164 A = parse_attributes(LocalName, Attributes), 165 CName = list_to_atom(LocalName), 166 167 State#state{tags=[CName |T], 168 dom=[{CName, 169 lists:reverse(A), 170 [] 171 } | D]}; 172build_dom({endElement, _Uri, LocalName, _QName}, 173 #state{tags=[_ |T], 174 dom=[{CName, CAttributes, CContent}, 175 {PName, PAttributes, PContent} = _Parent | D]} = State) -> 176 case list_to_atom(LocalName) of 177 CName -> 178 SectionDepth = length([E || E <- T, E =:= section]), 179 MappedCName = 180 case CName of 181 title -> 182 lists:nth(SectionDepth+1,[h1,h2,h3,h4,h5,h6]); 183 section when SectionDepth > 0 -> 184 'div'; 185 CName -> CName 186 end, 187 188 State#state{tags=T, 189 dom=[{PName, PAttributes, 190 [{MappedCName, CAttributes, 191 lists:reverse(CContent)} 192 |PContent] 193 } | D]}; 194 _ -> 195 ?error("Got end of element: " ++ LocalName ++ " but expected: " ++ 196 CName) 197 end; 198 199%% Text 200%%---------------------------------------------------------------------- 201build_dom({characters, String}, 202 #state{dom=[{Name, Attributes, Content}| D]} = State) -> 203 HtmlEnts = [{" ",[160]}, 204 {"×",[215]}, 205 {"±",[177]}, 206 {"ö","ö"}, 207 {"ä","ä"}, 208 {"å","å"} 209 ], 210 211 NoHtmlEnt = 212 lists:foldl( 213 fun({Pat,Sub},Str) -> 214 re:replace(Str,Pat,Sub,[global,unicode]) 215 end,String,HtmlEnts), 216 217 case re:run(NoHtmlEnt,"&[a-z]*;",[{capture,first,binary},unicode]) of 218 nomatch -> ok; 219 {match,[<<"<">>]} -> ok; 220 {match,[<<">">>]} -> ok; 221 Else -> throw({found_illigal_thing,Else,String}) 222 end, 223 NewContent = 224 [unicode:characters_to_binary(NoHtmlEnt,utf8)| Content], 225 State#state{dom=[{Name, Attributes, NewContent} | D]}; 226 227build_dom({ignorableWhitespace, String}, 228 #state{dom=[{Name,_,_} = _E|_]} = State) -> 229 case lists:member(Name, 230 [p,pre,input,code,quote,warning, 231 note,dont,do,c,b,i,em,strong, 232 seemfa,seeerl,seetype,seeapp, 233 seecom,seecref,seefile,seeguide, 234 tag,item]) of 235 true -> 236% io:format("Keep ign white: ~p ~p~n",[String, _E]), 237 build_dom({characters, String}, State); 238 false -> 239 State 240 end; 241 242build_dom({startEntity, SysId}, State) -> 243 io:format("startEntity:~p~n",[SysId]), 244 State; 245 246%% Default 247%%---------------------------------------------------------------------- 248build_dom(_E, State) -> 249 State. 250 251%%---------------------------------------------------------------------- 252%% Function : parse_attributes(ElName, Attributes) -> Result 253%% Parameters: 254%% Result : 255%% Description: 256%%---------------------------------------------------------------------- 257parse_attributes(ElName, Attributes) -> 258 parse_attributes(ElName, Attributes, 1, []). 259 260parse_attributes(_, [], _, Acc) -> 261 Acc; 262parse_attributes(ElName, [{_Uri, _Prefix, LocalName, AttrValue} |As], N, Acc) -> 263 parse_attributes(ElName, As, N+1, [{list_to_atom(LocalName), AttrValue} |Acc]). 264 265docs(Application, OTPXml, FromBEAM)-> 266 case xmerl_sax_parser:file(OTPXml, 267 [skip_external_dtd, 268 {event_fun,fun event/3}, 269 {event_state,initial_state()}]) of 270 {ok,Tree,_} -> 271 {ok, {Module, Chunks}} = beam_lib:chunks(FromBEAM,[exports,abstract_code]), 272 Dom = get_dom(Tree), 273 put(application, Application), 274 put(module, filename:basename(filename:rootname(FromBEAM))), 275 NewDom = transform(Dom,[]), 276 Chunk = to_chunk(NewDom, OTPXml, Module, proplists:get_value(abstract_code, Chunks)), 277 verify_chunk(Module,proplists:get_value(exports, Chunks), Chunk), 278 Chunk; 279 Else -> 280 {error,Else} 281 end. 282 283verify_chunk(M, Exports, #docs_v1{ docs = Docs } = Doc) -> 284 285 %% Make sure that each documented function actually is exported 286 Exported = [begin 287 FA = {F,A}, 288 {M,F,A,lists:member(FA,Exports)} 289 end || {{function,F,A},_,_,_,_} <- Docs], 290 lists:foreach(fun({_M,_F,_A,true}) -> 291 ok 292 end,Exported), 293 294 try 295 shell_docs:validate(Doc) 296 catch Err -> 297 throw({maps:get(<<"en">>,Doc#docs_v1.module_doc), Err}) 298 end. 299 300%% skip <erlref> but transform and keep its content 301transform([{erlref,_Attr,Content}|T],Acc) -> 302 Module = [Mod || Mod = {module,_,_} <- Content], 303 NewContent = Content -- Module, 304 [{module,SinceAttr,[Mname]}] = Module, 305 Since = case proplists:get_value(since,SinceAttr) of 306 undefined -> []; 307 [] -> []; 308 Vsn -> [{since,Vsn}] 309 end, 310 transform([{module,[{name,Mname}|Since],NewContent}|T],Acc); 311 312%% skip <header> and all of its content 313transform([{header,_Attr,_Content}|T],Acc) -> 314 transform(T,Acc); 315transform([{section,Attr,Content}|T],Acc) -> 316 transform(T,[{section,Attr,transform(Content,[])}|Acc]); 317 318%% transform <list><item> to <ul><li> or <ol><li> depending on type attribute 319transform([{list,Attr,Content}|T],Acc) -> 320 transform([transform_list(Attr,Content)|T],Acc); 321 322%% transform <taglist>(tag,item+)+ to <dl>(dt,item+)+ 323transform([{taglist,Attr,Content}|T],Acc) -> 324 transform([transform_taglist(Attr,Content)|T],Acc); 325 326%% remove <anno> as it is only used to validate specs vs xml src 327transform([{anno,[],Content}|T],Acc) -> 328 transform([Content|T],Acc); 329 330%% transform <c> to <code> 331transform([{c,[],Content}|T],Acc) -> 332 transform(T, [{code,[],transform(Content,[])}|Acc]); 333 334%% transform <code> to <pre><code> 335transform([{code,Attr,Content}|T],Acc) -> 336 transform(T, [{pre,[],[{code,a2b(Attr),transform(Content,[])}]}|Acc]); 337%% transform <pre> to <pre><code> 338transform([{pre,Attr,Content}|T],Acc) -> 339 transform(T, [{pre,[],[{code,Attr,transform(Content,[])}]}|Acc]); 340 341%% transform <funcs> with <func> as children 342transform([{funcs,_Attr,Content}|T],Acc) -> 343 Fns = {functions,[],transform_funcs(Content, [])}, 344 transform(T,[Fns|Acc]); 345%% transform <datatypes> with <datatype> as children 346transform([{datatypes,_Attr,Content}|T],Acc) -> 347 Dts = transform(Content, []), 348 transform(T,[{datatypes,[],Dts}|Acc]); 349transform([{datatype,_Attr,Content}|T],Acc) -> 350 transform(T,transform_datatype(Content, []) ++ Acc); 351%% Ignore <datatype_title> 352transform([{datatype_title,_Attr,_Content}|T],Acc) -> 353 transform(T,Acc); 354%% transform <desc>Content</desc> to Content 355transform([{desc,_Attr,Content}|T],Acc) -> 356 transform(T,[transform(Content,[])|Acc]); 357%% transform <marker id="name"/> to <a id="name"/>.... 358transform([{marker,Attrs,Content}|T],Acc) -> 359 transform(T,[{a,a2b(Attrs),transform(Content,[])}|Acc]); 360%% transform <url href="external URL"> Content</url> to <a href.... 361transform([{url,Attrs,Content}|T],Acc) -> 362 transform(T,[{a,a2b(Attrs),transform(Content,[])}|Acc]); 363%% transform note/warning/do/don't to <p class="thing"> 364transform([{What,[],Content}|T],Acc) 365 when What =:= note; What =:= warning; What =:= do; What =:= dont -> 366 WhatP = {'div',[{class,atom_to_binary(What)}], transform(Content,[])}, 367 transform(T,[WhatP|Acc]); 368 369transform([{type,_,[]}|_] = Dom,Acc) -> 370 %% Types are laid out sequentially in the source xml so we need to 371 %% parse them like that here too. 372 case transform_types(Dom,[]) of 373 {[],T} -> 374 transform(T,Acc); 375 {Types,T} -> 376 %% We sort the types here because in the source xml 377 %% the description and the declaration do not have 378 %% to be next to each other. But we want to have that 379 %% for the doc chunks. 380 NameSort = fun({li,A,_},{li,B,_}) -> 381 NameA = proplists:get_value(name,A), 382 NameB = proplists:get_value(name,B), 383 if NameA == NameB -> 384 length(A) =< length(B); 385 true -> 386 NameA < NameB 387 end 388 end, 389 transform(T,[{ul,[{class,<<"types">>}],lists:sort(NameSort,Types)}|Acc]) 390 end; 391transform([{type_desc,Attr,_Content}|T],Acc) -> 392 %% We skip any type_desc with the variable attribute 393 true = proplists:is_defined(variable, Attr), 394 transform(T,Acc); 395transform([{type,[],Content}|T],Acc) -> 396 transform(T,[{ul,[{class,<<"types">>}],transform(Content,[])}|Acc]); 397transform([{v,[],Content}|T],Acc) -> 398 transform(T, [{li,[{class,<<"type">>}],transform(Content,[])}|Acc]); 399transform([{d,[],Content}|T],Acc) -> 400 transform(T, [{li,[{class,<<"description">>}],transform(Content,[])}|Acc]); 401 402transform([Elem = {See,_Attr,_Content}|T],Acc) 403 when See =:= seemfa; See =:= seeerl; See =:= seetype; See =:= seeapp; 404 See =:= seecom; See =:= seecref; See =:= seefile; See =:= seeguide -> 405 transform([transform_see(Elem)|T],Acc); 406 407transform([{term,Attr,[]}|T],Acc) -> 408 transform([list_to_binary(proplists:get_value(id,Attr))|T],Acc); 409 410transform([{fsummary,_,_}|T],Acc) -> 411 %% We skip fsummary as it many times is just a duplicate of the 412 %% first line of the docs. 413 transform(T,Acc); 414 415transform([{input,_,Content}|T],Acc) -> 416 %% Just remove input as it is not used by anything 417 transform(T,[transform(Content,[])|Acc]); 418 419transform([{p,Attr,Content}|T],Acc) -> 420 transform(T,[{p,a2b(Attr),transform(Content,[])}|Acc]); 421transform([{'div',Attr,Content}|T],Acc) -> 422 transform(T,[{'div',a2b(Attr),transform(Content,[])}|Acc]); 423 424%% Tag and Attr is used as is but Content is transformed 425transform([{Tag,Attr,Content}|T],Acc) -> 426 transform(T,[{Tag,Attr,transform(Content,[])}|Acc]); 427transform([Binary|T],Acc) -> 428 transform(T,[Binary|Acc]); 429transform([],Acc) -> 430 lists:flatten(lists:reverse(Acc)). 431 432transform_list([{type,"ordered"}],Content) -> 433 {ol,[],[{li,A2,C2}||{item,A2,C2}<-Content]}; 434transform_list(_,Content) -> 435 {ul,[],[{li,A2,C2}||{item,A2,C2}<-Content]}. 436 437transform_types([{type,Attr,[]}|T],Acc) -> 438 case proplists:is_defined(name,Attr) of 439 true -> 440 transform_types(T, [{li,a2b(Attr),[]}|Acc]); 441 false -> 442 true = proplists:is_defined(variable, Attr), 443 transform_types(T, Acc) 444 end; 445transform_types([{type_desc,Attr,Content}|T],Acc) -> 446 case proplists:is_defined(name,Attr) of 447 true -> 448 TypeDesc = transform(Content,[]), 449 transform_types(T, [{li,a2b(Attr) ++ [{class,<<"description">>}],TypeDesc}|Acc]); 450 false -> 451 true = proplists:is_defined(variable, Attr), 452 transform_types(T, Acc) 453 end; 454transform_types([{type,_,_}|_T],_Acc) -> 455 throw(mixed_type_declarations); 456transform_types(Dom,Acc) -> 457 {lists:reverse(Acc),Dom}. 458 459transform_taglist(Attr,Content) -> 460 Items = 461 lists:map(fun({tag,A,C}) -> 462 {dt,A,C}; 463 ({item,A,C}) -> 464 {dd,A,C} 465 end, Content), 466 {dl,Attr,Items}. 467 468%% if we have {func,[],[{name,...},{name,....},...]} 469%% we convert it to one {func,[],[{name,...}] per arity lowest first. 470transform_funcs([Func|T],Acc) -> 471 transform_funcs(T,func2func(Func) ++ Acc); 472transform_funcs([],Acc) -> 473 lists:reverse(Acc). 474 475func2func({fsdescription,_Attr,_Contents}) -> 476 []; 477func2func({func,Attr,Contents}) -> 478 479 ContentsNoName = [NC||NC <- Contents, element(1,NC) /= name], 480 481 EditLink = 482 case proplists:get_value(ghlink,Attr) of 483 undefined -> 484 #{}; 485 GhLink -> 486 #{ edit_url => 487 iolist_to_binary(["https://github.com/erlang/otp/edit/",GhLink]) } 488 end, 489 490 VerifyNameList = 491 fun(NameList, Test) -> 492 %% Assert that we don't mix ways to write <name> 493 [begin 494 ok = Test(C), 495 {proplists:get_value(name,T),proplists:get_value(arity,T)} 496 end || {name,T,C} <- NameList] 497 end, 498 499 NameList = [Name || {name,_,_} = Name <- Contents], 500 501 %% "Since" is hard to accurately as there can be multiple <name> per <func> and they 502 %% can refer to the same or other arities. This should be improved in the future but 503 %% for now we set since to a comma separated list of all since attributes. 504 SinceMD = 505 case [proplists:get_value(since, SinceAttr) || 506 {name,SinceAttr,_} <- NameList, proplists:get_value(since, SinceAttr) =/= []] of 507 [] -> EditLink; 508 Sinces -> 509 EditLink#{ since => unicode:characters_to_binary( 510 lists:join(",",lists:usort(Sinces))) } 511 end, 512 513 Functions = 514 case NameList of 515 [{name,_,[]}|_] -> 516 %% Spec style function docs 517 TagsToFA = 518 fun(Tags) -> 519 {proplists:get_value(name,Tags), 520 proplists:get_value(arity,Tags)} 521 end, 522 523 _ = VerifyNameList(NameList,fun([]) -> ok end), 524 525 FAs = [TagsToFA(FAttr) || {name,FAttr,[]} <- NameList ], 526 SortedFAs = lists:usort(FAs), 527 FAClauses = lists:usort([{TagsToFA(FAttr),proplists:get_value(clause_i,FAttr)} 528 || {name,FAttr,[]} <- NameList ]), 529 530 MakeFunc = fun({F,A}, MD, Doc) -> 531 Specs = [begin 532 {function,Name} = func_to_atom(CF), 533 {Name,list_to_integer(CA),C} 534 end || {{CF,CA},C} <- FAClauses, 535 F =:= CF, A =:= CA], 536 {function,[{name,F},{arity,list_to_integer(A)}, 537 {signature,[iolist_to_binary([F,"/",A])]}, 538 {meta,MD#{ signature => Specs }}], 539 Doc} 540 end, 541 542 Base = MakeFunc(hd(SortedFAs), SinceMD, ContentsNoName), 543 544 {BaseF,BaseA} = hd(SortedFAs), 545 MD = SinceMD#{ equiv => {function,list_to_atom(BaseF),list_to_integer(BaseA)}}, 546 Equiv = lists:map( 547 fun(FA) -> 548 MakeFunc(FA, MD, []) 549 end, tl(SortedFAs)), 550 [Base | Equiv]; 551 NameList -> 552 %% Manual style function docs 553 FAs = lists:foldl( 554 fun({name,_,NameString}, Acc) -> 555 FAs = func_to_tuple(NameString), 556 lists:foldl( 557 fun(FA, FAAcc) -> 558 Slogan = maps:get(FA, FAAcc, []), 559 FAAcc#{ FA => [strip_tags(NameString)|Slogan] } 560 end, Acc, FAs) 561 end, #{}, NameList), 562 563 _ = VerifyNameList(NameList,fun([_|_]) -> ok end), 564 565 SortedFAs = lists:usort(maps:to_list(FAs)), 566 567 {{BaseF, BaseA}, BaseSig} = hd(SortedFAs), 568 569 Base = {function,[{name,BaseF},{arity,BaseA}, 570 {signature,BaseSig}, 571 {meta,SinceMD}], 572 ContentsNoName}, 573 574 Equiv = [{function, 575 [{name,F},{arity,A}, 576 {signature,Signature}, 577 {meta,SinceMD#{ equiv => {function,list_to_atom(BaseF),BaseA}}}],[]} 578 || {{F,A},Signature} <- tl(SortedFAs)], 579 [Base | Equiv] 580 end, 581 transform(Functions,[]). 582 583func_to_tuple(Chars) -> 584 try 585 [Name,Args] = string:split(strip_tags(Chars),"("), 586 Arities = parse_args(unicode:characters_to_list(Args)), 587 [{unicode:characters_to_list(Name),Arity} || Arity <- Arities] 588 catch E:R:ST -> 589 io:format("Failed to parse: ~p~n",[Chars]), 590 erlang:raise(E,R,ST) 591 end. 592 593%% This function parses a documentation <name> attribute to figure 594%% out the arities if that function. Example: 595%% "start([go,Mode] [,Extra])" returns [1, 2]. 596%% 597%% This assumes that when a single <name> describes many arities 598%% the arities are listed with [, syntax. 599parse_args(")" ++ _) -> 600 [0]; 601parse_args(Args) -> 602 parse_args(unicode:characters_to_list(Args),1,[]). 603parse_args([$[,$,|T],Arity,[]) -> 604 parse_args(T,Arity,[$[]) ++ parse_args(T,Arity+1,[]); 605parse_args([$,|T],Arity,[]) -> 606 parse_args(T,Arity+1,[]); 607parse_args([Open|T],Arity,Stack) 608 when Open =:= $[; Open =:= ${; Open =:= $( -> 609 parse_args(T,Arity,[Open|Stack]); 610parse_args([$]|T],Arity,[$[|Stack]) -> 611 parse_args(T,Arity,Stack); 612parse_args([$}|T],Arity,[${|Stack]) -> 613 parse_args(T,Arity,Stack); 614parse_args([$)|T],Arity,[$(|Stack]) -> 615 parse_args(T,Arity,Stack); 616parse_args([$)|_T],Arity,[]) -> 617 [Arity]; 618parse_args([_H|T],Arity,Stack) -> 619 parse_args(T,Arity,Stack). 620 621strip_tags([{_Tag,_Attr,Content}|T]) -> 622 [Content | strip_tags(T)]; 623strip_tags([H|T]) when not is_tuple(H) -> 624 [H | strip_tags(T)]; 625strip_tags([]) -> 626 []. 627 628transform_datatype(Dom,_Acc) -> 629 ContentsNoName = transform([NC||NC <- Dom, element(1,NC) /= name],[]), 630 [case N of 631 {name,NameAttr,[]} -> 632 {datatype,NameAttr,ContentsNoName}; 633 {name,[],Content} -> 634 [{Name,Arity}] = func_to_tuple(Content), 635 Signature = strip_tags(Content), 636 {datatype,[{name,Name},{n_vars,integer_to_list(Arity)}, 637 {signature,Signature}],ContentsNoName} 638 end || N = {name,_,_} <- Dom]. 639 640transform_see({See,[{marker,Marker}],Content}) -> 641 AbsMarker = 642 case string:split(Marker, "#") of 643 [AppFile] -> marker_defaults(AppFile); 644 [AppFile, Anchor] -> [marker_defaults(AppFile), "#", Anchor] 645 end, 646 647 {a, [{href,iolist_to_binary(AbsMarker)}, 648 {rel,<<"https://erlang.org/doc/link/",(atom_to_binary(See))/binary>>}], Content}. 649 650marker_defaults("") -> 651 [get(application), ":", get(module)]; 652marker_defaults(AppFile) -> 653 case string:split(AppFile, ":") of 654 [File] -> [get(application), ":", File]; 655 [App, File] -> [App, ":", File] 656 end. 657 658to_chunk(Dom, Source, Module, AST) -> 659 [{module,MAttr,Mcontent}] = Dom, 660 661 ModuleDocs = lists:flatmap( 662 fun({Tag,_,Content}) when Tag =:= description; 663 Tag =:= section -> 664 Content; 665 ({_,_,_}) -> 666 [] 667 end, Mcontent), 668 669 TypeMeta = add_types(AST, maps:from_list([{source,Source}|MAttr])), 670 671 TypeMap = maps:get(types, TypeMeta, []), 672 673 Anno = erl_anno:set_file(atom_to_list(Module)++".erl",erl_anno:new(0)), 674 675 Types = lists:flatten([Types || {datatypes,[],Types} <- Mcontent]), 676 677 TypeEntries = 678 lists:map( 679 fun({datatype,Attr,Descr}) -> 680 {function, TypeName} = func_to_atom(proplists:get_value(name,Attr)), 681 TypeArity = case proplists:get_value(n_vars,Attr) of 682 undefined -> 683 find_type_arity(TypeName, TypeMap); 684 Arity -> 685 list_to_integer(Arity) 686 end, 687 TypeArgs = lists:join(",",[lists:concat(["Arg",I]) || I <- lists:seq(1,TypeArity)]), 688 PlaceholderSig = io_lib:format("-type ~p(~s) :: term().",[TypeName,TypeArgs]), 689 TypeSignature = proplists:get_value( 690 signature,Attr,[iolist_to_binary(PlaceholderSig)]), 691 MetaSig = 692 case maps:get({TypeName, TypeArity}, TypeMap, undefined) of 693 undefined -> 694 #{}; 695 Sig -> 696 #{ signature => [Sig] } 697 end, 698 699 MetaDepr 700 = case apply(otp_internal,obsolete_type,[Module, TypeName, TypeArity]) of 701 %% apply/3 in order to silence dialyzer 702 {deprecated, Text} -> 703 MetaSig#{ deprecated => 704 unicode:characters_to_binary( 705 erl_lint:format_error({deprecated_type,{Module,TypeName,TypeArity}, Text})) }; 706 {deprecated, Replacement, Rel} -> 707 MetaSig#{ deprecated => 708 unicode:characters_to_binary( 709 erl_lint:format_error({deprecated_type,{Module,TypeName,TypeArity}, Replacement, Rel})) }; 710 {removed, _Text} -> 711 %% Just skip 712 MetaSig; 713 no -> 714 MetaSig 715 end, 716 717 docs_v1_entry(type, Anno, TypeName, TypeArity, TypeSignature, MetaDepr, Descr) 718 end, Types), 719 720 Functions = lists:flatten([Functions || {functions,[],Functions} <- Mcontent]), 721 722 FuncEntrys = 723 lists:map( 724 fun({function,Attr,Fdoc}) -> 725 {Type, Name} = func_to_atom(proplists:get_value(name,Attr)), 726 Arity = proplists:get_value(arity,Attr), 727 Signature = proplists:get_value(signature,Attr), 728 FMeta = proplists:get_value(meta,Attr), 729 MetaWSpec = add_spec(AST,FMeta), 730 MetaDepr 731 = case apply(otp_internal,obsolete,[Module, Name, Arity]) of 732 %% apply/3 in order to silence dialyzer 733 {deprecated, Text} -> 734 MetaWSpec#{ deprecated => 735 unicode:characters_to_binary( 736 erl_lint:format_error({deprecated,{Module,Name,Arity}, Text})) }; 737 {deprecated, Replacement, Rel} -> 738 MetaWSpec#{ deprecated => 739 unicode:characters_to_binary( 740 erl_lint:format_error({deprecated,{Module,Name,Arity}, Replacement, Rel})) }; 741 _ -> MetaWSpec 742 end, 743 docs_v1_entry(Type, Anno, Name, Arity, Signature, MetaDepr, Fdoc) 744 end, Functions), 745 746 docs_v1(ModuleDocs, Anno, TypeMeta, FuncEntrys ++ TypeEntries). 747 748docs_v1(DocContents, Anno, Metadata, Docs) -> 749 #docs_v1{ anno = Anno, 750 module_doc = #{<<"en">> => shell_docs:normalize(DocContents)}, 751 metadata = maps:merge(Metadata, (#docs_v1{})#docs_v1.metadata), 752 docs = Docs }. 753 754docs_v1_entry(Kind, Anno, Name, Arity, Signature, Metadata, DocContents) -> 755 756 AnnoWLine = 757 case Metadata of 758 #{ signature := [Sig|_] } -> 759 SigAnno = element(2, Sig), 760 erl_anno:set_line(erl_anno:line(SigAnno), Anno); 761 _NoSignature -> 762 Anno 763 end, 764 765 Doc = 766 case DocContents of 767 [] -> 768 #{}; 769 DocContents -> 770 #{ <<"en">> => shell_docs:normalize(DocContents) } 771 end, 772 773 {{Kind, Name, Arity}, AnnoWLine, lists:flatten(Signature), Doc, Metadata}. 774 775%% A special list_to_atom that handles 776%% 'and' 777%% Destroy 778%% 'begin' 779func_to_atom(List) -> 780 case erl_scan:string(List) of 781 {ok,[{atom,_,Fn}],_} -> 782 {function, Fn}; 783 {ok,[{var,_,Fn}],_} -> 784 {function, Fn}; 785 {ok,[{Fn,_}],_} -> 786 {function, Fn}; 787 {ok,[{var,_,_},{':',_},{atom,_,Fn}],_} -> 788 {callback, Fn}; 789 {ok,[{var,_,_},{':',_},{var,_,Fn}],_} -> 790 {callback, Fn} 791 end. 792 793-define(IS_TYPE(TO),(TO =:= type orelse TO =:= opaque)). 794 795add_spec(no_abstract_code, Meta) -> 796 Meta; 797add_spec({raw_abstract_v1, AST}, Meta = #{ signature := Specs } ) -> 798 Meta#{ signature := add_spec_clauses(AST, merge_clauses(Specs,#{})) }; 799add_spec(_, Meta) -> 800 Meta. 801 802add_types(no_abstract_code, Meta) -> 803 Meta; 804add_types({raw_abstract_v1, AST}, Meta) -> 805 Meta#{ types => 806 maps:from_list( 807 [{{Name,length(Args)},T} || T = {attribute,_,TO,{Name, _, Args}} <- AST, 808 ?IS_TYPE(TO)]) }. 809 810add_spec_clauses(AST, [{{F,A},Clauses}|T]) -> 811 [filter_clauses(find_spec(AST,F,A),Clauses) | add_spec_clauses(AST,T)]; 812add_spec_clauses(_AST, []) -> 813 []. 814 815filter_clauses(Spec,[undefined]) -> 816 Spec; 817filter_clauses({attribute,Ln,spec,{FA,Clauses}},ClauseIds) -> 818 {_,FilteredClauses} = 819 lists:foldl( 820 fun({TO,_,_,_} = C,{Cnt,Acc}) when ?IS_TYPE(TO) -> 821 case lists:member(integer_to_list(Cnt),ClauseIds) of 822 true -> 823 {Cnt+1,[C | Acc]}; 824 false -> 825 {Cnt+1,Acc} 826 end 827 end, {1, []}, Clauses), 828 {attribute,Ln,spec,{FA,lists:reverse(FilteredClauses)}}. 829 830merge_clauses([{F,A,Clause}|T],Acc) -> 831 merge_clauses(T,Acc#{ {F,A} => [Clause | maps:get({F,A},Acc,[])]}); 832merge_clauses([],Acc) -> 833 maps:to_list(Acc). 834 835find_type_arity(Name, [{{Name,_},{attribute,_,TO,{Name,_,Args}}}|_T]) when ?IS_TYPE(TO) -> 836 length(Args); 837find_type_arity(Name, [_|T]) -> 838 find_type_arity(Name,T); 839find_type_arity(Name, Map) when is_map(Map) -> 840 find_type_arity(Name, maps:to_list(Map)). 841 842find_spec(AST, Func, Arity) -> 843 Specs = lists:filter(fun({attribute,_,spec,{{F,A},_}}) -> 844 F =:= Func andalso A =:= Arity; 845 ({attribute,_,spec,{{_,F,A},_}}) -> 846 F =:= Func andalso A =:= Arity; 847 (_) -> 848 false 849 end, AST), 850 case Specs of 851 [S] -> 852 S; 853 [] -> 854 io:format("Could not find spec for ~p/~p~n",[Func,Arity]), 855 exit(1) 856 end. 857 858a2b(Attrs) -> 859 [{Tag,unicode:characters_to_binary(Value)} || {Tag,Value} <- Attrs]. 860