1-module(rebar_prv_compile). 2 3-behaviour(provider). 4 5-export([init/1, 6 do/1, 7 format_error/1]). 8 9-export([compile/2, compile/3, compile/4]). 10 11-include_lib("providers/include/providers.hrl"). 12-include("rebar.hrl"). 13 14-define(PROVIDER, compile). 15-define(ERLC_HOOK, erlc_compile). 16-define(APP_HOOK, app_compile). 17-define(DEPS, [lock]). 18 19%% =================================================================== 20%% Public API 21%% =================================================================== 22 23-spec init(rebar_state:t()) -> {ok, rebar_state:t()}. 24init(State) -> 25 State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER}, 26 {module, ?MODULE}, 27 {bare, true}, 28 {deps, ?DEPS}, 29 {example, "rebar3 compile"}, 30 {short_desc, "Compile apps .app.src and .erl files."}, 31 {desc, "Compile apps .app.src and .erl files."}, 32 {opts, [{deps_only, $d, "deps_only", undefined, 33 "Only compile dependencies, no project apps will be built."}]}])), 34 {ok, State1}. 35 36-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. 37do(State) -> 38 IsDepsOnly = is_deps_only(State), 39 rebar_paths:set_paths([deps], State), 40 41 Providers = rebar_state:providers(State), 42 MustBuildDeps = rebar_state:deps_to_build(State), 43 Deps = rebar_state:all_deps(State), 44 CompiledDeps = copy_and_build_deps(State, Providers, MustBuildDeps, Deps), 45 State0 = rebar_state:merge_all_deps(State, CompiledDeps), 46 47 State1 = case IsDepsOnly of 48 true -> 49 State0; 50 false -> 51 handle_project_apps(Providers, State0) 52 end, 53 54 rebar_paths:set_paths([plugins], State1), 55 56 {ok, State1}. 57 58is_deps_only(State) -> 59 {Args, _} = rebar_state:command_parsed_args(State), 60 proplists:get_value(deps_only, Args, false). 61 62handle_project_apps(Providers, State) -> 63 Cwd = rebar_state:dir(State), 64 ProjectApps = rebar_state:project_apps(State), 65 {ok, ProjectApps1} = rebar_digraph:compile_order(ProjectApps), 66 67 %% Run top level hooks *before* project apps compiled but *after* deps are 68 rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State), 69 70 ProjectApps2 = copy_and_build_project_apps(State, Providers, ProjectApps1), 71 State2 = rebar_state:project_apps(State, ProjectApps2), 72 73 %% build extra_src_dirs in the root of multi-app projects 74 build_root_extras(State, ProjectApps2), 75 76 State3 = update_code_paths(State2, ProjectApps2), 77 78 rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State3), 79 case rebar_state:has_all_artifacts(State3) of 80 {false, File} -> 81 throw(?PRV_ERROR({missing_artifact, File})); 82 true -> 83 true 84 end, 85 86 State3. 87 88 89-spec format_error(any()) -> iolist(). 90format_error({missing_artifact, File}) -> 91 io_lib:format("Missing artifact ~ts", [File]); 92format_error({bad_project_builder, Name, Type, Module}) -> 93 io_lib:format("Error building application ~s:~n Required project builder ~s function " 94 "~s:build/1 not found", [Name, Type, Module]); 95format_error({unknown_project_type, Name, Type}) -> 96 io_lib:format("Error building application ~s:~n " 97 "No project builder is configured for type ~s", [Name, Type]); 98format_error(Reason) -> 99 io_lib:format("~p", [Reason]). 100 101pick_deps_to_build(State, MustBuild, All, Tag) -> 102 InvalidDags = lists:any( 103 fun({_Mod, Status}) -> 104 %% A DAG that is not found can be valid, it just has no matching 105 %% source file. However, bad_vsn, bad_format, and bad_meta should 106 %% all trigger rebuilds. 107 %% Good: 108 %% lists:member(Status, [valid, not_found]) 109 %% Bad: 110 %% lists:member(Status, [bad_vsn, bad_format, bad_meta]) 111 %% 112 %% Since the fastest check is done on smaller lists handling 113 %% the common case first: 114 not lists:member(Status, [valid, not_found]) 115 end, 116 check_dags(State, Tag) 117 ), 118 case InvalidDags of 119 true -> All; 120 false -> MustBuild 121 end. 122 123copy_and_build_deps(State, Providers, MustBuildApps, AllApps) -> 124 Apps = pick_deps_to_build(State, MustBuildApps, AllApps, apps), 125 Apps0 = [prepare_app(State, Providers, App) || App <- Apps], 126 compile(State, Providers, Apps0, apps). 127 128copy_and_build_project_apps(State, Providers, Apps) -> 129 Apps0 = [prepare_project_app(State, Providers, App) || App <- Apps], 130 compile(State, Providers, Apps0, project_apps). 131 132-spec compile(rebar_state:t(), [rebar_app_info:t()]) -> [rebar_app_info:t()] 133 ; (rebar_state:t(), rebar_app_info:t()) -> rebar_app_info:t(). 134compile(State, AppInfo) -> 135 compile(State, rebar_state:providers(State), AppInfo). 136 137-spec compile(rebar_state:t(), [providers:t()], 138 [rebar_app_info:t()]) -> [rebar_app_info:t()] 139 ; (rebar_state:t(), [providers:t()], 140 rebar_app_info:t()) -> rebar_app_info:t(). 141compile(State, Providers, AppInfo) when not is_list(AppInfo) -> 142 [Res] = compile(State, Providers, [AppInfo], undefined), 143 Res; 144compile(State, Providers, Apps) -> 145 compile(State, Providers, Apps, undefined). 146 147-spec compile(rebar_state:t(), [providers:t()], 148 [rebar_app_info:t()], atom() | undefined) -> [rebar_app_info:t()]. 149compile(State, Providers, Apps, Tag) -> 150 ?DEBUG("Compile (~p)", [if Tag =:= undefined -> untagged; true -> Tag end]), 151 Apps1 = [prepare_compile(State, Providers, App) || App <- Apps], 152 Apps2 = [prepare_compilers(State, Providers, App) || App <- Apps1], 153 Apps3 = run_compilers(State, Providers, Apps2, Tag), 154 Apps4 = [finalize_compilers(State, Providers, App) || App <- Apps3], 155 Apps5 = [prepare_app_file(State, Providers, App) || App <- Apps4], 156 Apps6 = compile_app_files(State, Providers, Apps5), 157 Apps7 = [finalize_app_file(State, Providers, App) || App <- Apps6], 158 [finalize_compile(State, Providers, App) || App <- Apps7]. 159 160prepare_app(_State, _Providers, AppInfo) -> 161 AppDir = rebar_app_info:dir(AppInfo), 162 OutDir = rebar_app_info:out_dir(AppInfo), 163 copy_app_dirs(AppInfo, AppDir, OutDir), 164 AppInfo. 165 166prepare_project_app(_State, _Providers, AppInfo) -> 167 copy_app_dirs(AppInfo, 168 rebar_app_info:dir(AppInfo), 169 rebar_app_info:out_dir(AppInfo)), 170 %% application code path must be added to the source 171 %% otherwise code_server will remember 'lib_dir' for 172 %% this application, and all `-include_lib` directives 173 %% will actually go into _build/profile/lib/... 174 code:add_pathsz([rebar_app_info:dir(AppInfo)]), 175 AppInfo. 176 177prepare_compile(State, Providers, AppInfo) -> 178 AppDir = rebar_app_info:dir(AppInfo), 179 rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State). 180 181prepare_compilers(State, Providers, AppInfo) -> 182 AppDir = rebar_app_info:dir(AppInfo), 183 rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo, State). 184 185run_compilers(State, _Providers, Apps, Tag) -> 186 %% Get the compiler graphs for all modules and artifacts, for each 187 %% compiler we run 188 DAGsInfo = load_dags(State, Tag), 189 DAGs = [{Mod, DAG} || {Mod, {DAG, _Meta}} <- DAGsInfo], 190 %% Compile all the apps 191 build_apps(DAGs, Apps, State), 192 %% Potentially store shared compiler DAGs so next runs can easily 193 %% share the base information for easy re-scans. 194 lists:foreach(fun({Mod, {G, {Dir, DAGLabel, CritMeta}}}) -> 195 rebar_compiler_dag:maybe_store(G, Dir, Mod, DAGLabel, CritMeta), 196 rebar_compiler_dag:terminate(G) 197 end, DAGsInfo), 198 Apps. 199 200load_dags(State, Tag) -> 201 F = fun(Dir, Mod, DAGLabel, CritMeta) -> 202 {rebar_compiler_dag:init(Dir, Mod, DAGLabel, CritMeta), 203 {Dir, DAGLabel, CritMeta}} 204 end, 205 map_dags(F, State, Tag). 206 207check_dags(State, Tag) -> 208 map_dags(fun rebar_compiler_dag:status/4, State, Tag). 209 210map_dags(F, State, Tag) -> 211 DAGLabel = format_label(Tag), 212 %% The Dir for the DAG is set to deps_dir so builds taking place 213 %% in different contexts (i.e. plugins) don't risk clobbering regular deps. 214 Dir = rebar_dir:deps_dir(State), 215 SharedCritMeta = [], 216 [{Mod, F(Dir, Mod, DAGLabel, mod_meta(Mod, SharedCritMeta))} 217 || Mod <- rebar_state:compilers(State)]. 218 219mod_meta(rebar_compiler_erl, CritMeta) -> 220 application:load(compiler), 221 {ok, CompileVsn} = application:get_key(compiler, vsn), 222 [{compiler, CompileVsn} | CritMeta]; 223mod_meta(_, CritMeta) -> 224 CritMeta. 225 226format_label(Tag) -> 227 %% The Tag allows to create a Label when someone cares about a specific 228 %% run for compilation; 229 case Tag of 230 undefined -> undefined; 231 _ -> atom_to_list(Tag) 232 end. 233 234finalize_compilers(State, Providers, AppInfo) -> 235 AppDir = rebar_app_info:dir(AppInfo), 236 rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo, State). 237 238prepare_app_file(State, Providers, AppInfo) -> 239 AppDir = rebar_app_info:dir(AppInfo), 240 rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo, State). 241 242compile_app_files(State, Providers, Apps) -> 243 %% Load plugins back for make_vsn calls in custom resources. 244 %% The rebar_otp_app compilation step is safe regarding the 245 %% overall path management, so we can just load all plugins back 246 %% in memory. 247 rebar_paths:set_paths([plugins], State), 248 NewApps = [compile_app_file(State, Providers, App) || App <- Apps], 249 %% Clean up after ourselves, leave things as they were with deps first 250 rebar_paths:set_paths([deps], State), 251 NewApps. 252 253compile_app_file(State, _Providers, AppInfo) -> 254 case rebar_otp_app:compile(State, AppInfo) of 255 {ok, AppInfo2} -> AppInfo2; 256 Error -> throw(Error) 257 end. 258 259finalize_app_file(State, Providers, AppInfo) -> 260 AppDir = rebar_app_info:dir(AppInfo), 261 rebar_hooks:run_all_hooks(AppDir, post, ?APP_HOOK, Providers, AppInfo, State). 262 263finalize_compile(State, Providers, AppInfo) -> 264 AppDir = rebar_app_info:dir(AppInfo), 265 AppInfo2 = rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo, State), 266 has_all_artifacts(AppInfo), 267 AppInfo2. 268 269build_root_extras(State, Apps) -> 270 %% The root extra src dirs belong to no specific applications; 271 %% because the compiler works on OTP apps, we instead build 272 %% a fake AppInfo record that only contains the root extra_src 273 %% directories, has access to all the top-level apps' public 274 %% include files, and builds to a specific extra outdir. 275 %% TODO: figure out digraph strategy to properly ensure no 276 %% cross-contamination but proper change detection. 277 BaseDir = rebar_state:dir(State), 278 F = fun(App) -> rebar_app_info:dir(App) == BaseDir end, 279 case lists:any(F, Apps) of 280 true -> 281 []; 282 false -> 283 ProjOpts = rebar_state:opts(State), 284 Extras = rebar_dir:extra_src_dirs(ProjOpts, []), 285 {ok, VirtApp} = rebar_app_info:new("extra", "0.0.0", BaseDir, []), 286 287 case extra_virtual_apps(State, VirtApp, Extras) of 288 %% If there are no virtual apps do nothing. 289 [] -> 290 []; 291 VirtApps -> 292 %% re-use the project-apps digraph? 293 run_compilers(State, [], VirtApps, project_apps) 294 end 295 end. 296 297extra_virtual_apps(_, _, []) -> 298 []; 299extra_virtual_apps(State, VApp0, [Dir|Dirs]) -> 300 SrcDir = filename:join([rebar_state:dir(State), Dir]), 301 case ec_file:is_dir(SrcDir) of 302 false -> 303 extra_virtual_apps(State, VApp0, Dirs); 304 true -> 305 BaseDir = filename:join([rebar_dir:base_dir(State), "extras"]), 306 OutDir = filename:join([BaseDir, Dir]), 307 copy(rebar_state:dir(State), BaseDir, Dir), 308 VApp1 = rebar_app_info:out_dir(VApp0, BaseDir), 309 VApp2 = rebar_app_info:ebin_dir(VApp1, OutDir), 310 Opts = rebar_state:opts(State), 311 VApp3 = rebar_app_info:opts(VApp2, Opts), 312 %% ensure we don't end up running a similar extra fake app while 313 %% compiling root-level extras by marking it explicitly null 314 VApp4 = rebar_app_info:set(VApp3, extra_src_dirs, []), 315 %% need a unique name to prevent lookup issues that clobber entries 316 AppName = unicode:characters_to_binary(["extra_", Dir]), 317 VApp5 = rebar_app_info:name(VApp4, AppName), 318 [rebar_app_info:set(VApp5, src_dirs, [OutDir]) 319 | extra_virtual_apps(State, VApp0, Dirs)] 320 end. 321 322%% =================================================================== 323%% Internal functions 324%% =================================================================== 325 326build_apps(DAGs, Apps, State) -> 327 {Rebar3, Custom} = lists:partition( 328 fun(AppInfo) -> 329 Type = rebar_app_info:project_type(AppInfo), 330 Type =:= rebar3 orelse Type =:= undefined 331 end, 332 Apps 333 ), 334 [build_custom_builder_app(AppInfo, State) || AppInfo <- Custom], 335 build_rebar3_apps(DAGs, Rebar3, State). 336 337build_custom_builder_app(AppInfo, State) -> 338 ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]), 339 Type = rebar_app_info:project_type(AppInfo), 340 ProjectBuilders = rebar_state:project_builders(State), 341 case lists:keyfind(Type, 1, ProjectBuilders) of 342 {_, Module} -> 343 %% load plugins since thats where project builders would be, 344 %% prevents parallelism at this level. 345 rebar_paths:set_paths([deps, plugins], State), 346 Res = Module:build(AppInfo), 347 rebar_paths:set_paths([deps], State), 348 case Res of 349 ok -> ok; 350 {error, Reason} -> throw({error, {Module, Reason}}) 351 end; 352 _ -> 353 throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) 354 end. 355 356build_rebar3_apps(DAGs, Apps, _State) when DAGs =:= []; Apps =:= [] -> 357 %% No apps to actually build, probably just other compile phases 358 %% to run for non-rebar3 apps, someone wanting .app files built, 359 %% or just needing the hooks to run maybe. 360 ok; 361build_rebar3_apps(DAGs, Apps, State) -> 362 rebar_paths:set_paths([deps], State), 363 %% To maintain output order, we need to mention each app being compiled 364 %% in order, even if the order isn't really there anymore due to each 365 %% compiler being run in broken sequence. The last compiler tends to be 366 %% the big ERLC one so we use the last compiler for the output. 367 LastDAG = lists:last(DAGs), 368 %% we actually need to compile each DAG one after the other to prevent 369 %% issues where a .yrl file that generates a .erl file gets to be seen. 370 ?INFO("Analyzing applications...", []), 371 [begin 372 {Ctx, ReorderedApps} = rebar_compiler:analyze_all(DAG, Apps), 373 lists:foreach( 374 fun(AppInfo) -> 375 DAG =:= LastDAG andalso 376 ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]), 377 rebar_compiler:compile_analyzed(DAG, AppInfo, Ctx) 378 end, 379 ReorderedApps 380 ), 381 {ExtraCtx, ReorderedExtraApps} = rebar_compiler:analyze_all_extras(DAG, Apps), 382 lists:foreach( 383 fun(AppInfo) -> 384 rebar_compiler:compile_analyzed(DAG, AppInfo, ExtraCtx) 385 end, 386 ReorderedExtraApps 387 ) 388 end || DAG <- DAGs], 389 ok. 390 391update_code_paths(State, ProjectApps) -> 392 ProjAppsPaths = paths_for_apps(ProjectApps), 393 ExtrasPaths = paths_for_extras(State, ProjectApps), 394 DepsPaths = rebar_state:code_paths(State, all_deps), 395 rebar_state:code_paths(State, all_deps, DepsPaths ++ ProjAppsPaths ++ ExtrasPaths). 396 397paths_for_apps(Apps) -> paths_for_apps(Apps, []). 398 399paths_for_apps([], Acc) -> Acc; 400paths_for_apps([App|Rest], Acc) -> 401 {_SrcDirs, ExtraDirs} = resolve_src_dirs(rebar_app_info:opts(App)), 402 Paths = [filename:join([rebar_app_info:out_dir(App), Dir]) || Dir <- ["ebin"|ExtraDirs]], 403 FilteredPaths = lists:filter(fun ec_file:is_dir/1, Paths), 404 paths_for_apps(Rest, Acc ++ FilteredPaths). 405 406paths_for_extras(State, Apps) -> 407 F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end, 408 %% check that this app hasn't already been dealt with 409 case lists:any(F, Apps) of 410 false -> paths_for_extras(State); 411 true -> [] 412 end. 413 414paths_for_extras(State) -> 415 {_SrcDirs, ExtraDirs} = resolve_src_dirs(rebar_state:opts(State)), 416 Paths = [filename:join([rebar_dir:base_dir(State), "extras", Dir]) || Dir <- ExtraDirs], 417 lists:filter(fun ec_file:is_dir/1, Paths). 418 419has_all_artifacts(AppInfo1) -> 420 case rebar_app_info:has_all_artifacts(AppInfo1) of 421 {false, File} -> 422 throw(?PRV_ERROR({missing_artifact, File})); 423 true -> 424 rebar_app_utils:lint_app_info(AppInfo1), 425 true 426 end. 427 428copy_app_dirs(AppInfo, OldAppDir, AppDir) -> 429 case rebar_utils:to_binary(filename:absname(OldAppDir)) =/= 430 rebar_utils:to_binary(filename:absname(AppDir)) of 431 true -> 432 EbinDir = filename:join([OldAppDir, "ebin"]), 433 %% copy all files from ebin if it exists 434 case filelib:is_dir(EbinDir) of 435 true -> 436 OutEbin = filename:join([AppDir, "ebin"]), 437 filelib:ensure_dir(filename:join([OutEbin, "dummy.beam"])), 438 ?DEBUG("Copying existing files from ~ts to ~ts", [EbinDir, OutEbin]), 439 rebar_file_utils:cp_r(filelib:wildcard(filename:join([EbinDir, "*"])), OutEbin); 440 false -> 441 ok 442 end, 443 444 filelib:ensure_dir(filename:join([AppDir, "dummy"])), 445 446 %% link or copy mibs if it exists 447 case filelib:is_dir(filename:join([OldAppDir, "mibs"])) of 448 true -> 449 %% If mibs exist it means we must ensure priv exists. 450 %% mibs files are compiled to priv/mibs/ 451 filelib:ensure_dir(filename:join([OldAppDir, "priv", "dummy"])), 452 symlink_or_copy(OldAppDir, AppDir, "mibs"); 453 false -> 454 ok 455 end, 456 {SrcDirs, ExtraDirs} = resolve_src_dirs(rebar_app_info:opts(AppInfo)), 457 %% link to src_dirs to be adjacent to ebin is needed for R15 use of cover/xref 458 %% priv/ and include/ are symlinked unconditionally to allow hooks 459 %% to write to them _after_ compilation has taken place when the 460 %% initial directory did not, and still work 461 [symlink_or_copy(OldAppDir, AppDir, Dir) || Dir <- ["priv", "include"]], 462 [symlink_or_copy_existing(OldAppDir, AppDir, Dir) || Dir <- SrcDirs], 463 %% copy all extra_src_dirs as they build into themselves and linking means they 464 %% are shared across profiles 465 [copy(OldAppDir, AppDir, Dir) || Dir <- ExtraDirs]; 466 false -> 467 ok 468 end. 469 470symlink_or_copy(OldAppDir, AppDir, Dir) -> 471 Source = filename:join([OldAppDir, Dir]), 472 Target = filename:join([AppDir, Dir]), 473 rebar_file_utils:symlink_or_copy(Source, Target). 474 475symlink_or_copy_existing(OldAppDir, AppDir, Dir) -> 476 Source = filename:join([OldAppDir, Dir]), 477 Target = filename:join([AppDir, Dir]), 478 case ec_file:is_dir(Source) of 479 true -> rebar_file_utils:symlink_or_copy(Source, Target); 480 false -> ok 481 end. 482 483copy(OldAppDir, AppDir, Dir) -> 484 Source = filename:join([OldAppDir, Dir]), 485 Target = filename:join([AppDir, Dir]), 486 case ec_file:is_dir(Source) of 487 true -> copy(Source, Target); 488 false -> ok 489 end. 490 491%% TODO: use ec_file:copy/2 to do this, it preserves timestamps and 492%% may prevent recompilation of files in extra dirs 493copy(Source, Source) -> 494 %% allow users to specify a directory in _build as a directory 495 %% containing additional source/tests 496 ok; 497copy(Source, Target) -> 498 %% important to do this so no files are copied onto themselves 499 %% which truncates them to zero length on some platforms 500 ok = delete_if_symlink(Target), 501 ok = filelib:ensure_dir(filename:join([Target, "dummy.beam"])), 502 {ok, Files} = rebar_utils:list_dir(Source), 503 case [filename:join([Source, F]) || F <- Files] of 504 [] -> ok; 505 Paths -> rebar_file_utils:cp_r(Paths, Target) 506 end. 507 508delete_if_symlink(Path) -> 509 case ec_file:is_symlink(Path) of 510 true -> file:delete(Path); 511 false -> ok 512 end. 513 514resolve_src_dirs(Opts) -> 515 SrcDirs = rebar_dir:src_dirs(Opts, ["src"]), 516 ExtraDirs = rebar_dir:extra_src_dirs(Opts, []), 517 normalize_src_dirs(SrcDirs, ExtraDirs). 518 519%% remove duplicates and make sure no directories that exist 520%% in src_dirs also exist in extra_src_dirs 521normalize_src_dirs(SrcDirs, ExtraDirs) -> 522 S = lists:usort(SrcDirs), 523 E = lists:subtract(lists:usort(ExtraDirs), S), 524 ok = warn_on_problematic_directories(S ++ E), 525 {S, E}. 526 527%% warn when directories called `eunit' and `ct' are added to compile dirs 528warn_on_problematic_directories(AllDirs) -> 529 F = fun(Dir) -> 530 case is_a_problem(Dir) of 531 true -> ?WARN("Possible name clash with directory ~p.", [Dir]); 532 false -> ok 533 end 534 end, 535 lists:foreach(F, AllDirs). 536 537is_a_problem("eunit") -> true; 538is_a_problem("common_test") -> true; 539is_a_problem(_) -> false. 540