1-module(rebar_prv_compile).
2
3-behaviour(provider).
4
5-export([init/1,
6         do/1,
7         format_error/1]).
8
9-export([compile/2, compile/3, compile/4]).
10
11-include_lib("providers/include/providers.hrl").
12-include("rebar.hrl").
13
14-define(PROVIDER, compile).
15-define(ERLC_HOOK, erlc_compile).
16-define(APP_HOOK, app_compile).
17-define(DEPS, [lock]).
18
19%% ===================================================================
20%% Public API
21%% ===================================================================
22
23-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
24init(State) ->
25    State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER},
26                                                               {module, ?MODULE},
27                                                               {bare, true},
28                                                               {deps, ?DEPS},
29                                                               {example, "rebar3 compile"},
30                                                               {short_desc, "Compile apps .app.src and .erl files."},
31                                                               {desc, "Compile apps .app.src and .erl files."},
32                                                               {opts, [{deps_only, $d, "deps_only", undefined,
33                                                                        "Only compile dependencies, no project apps will be built."}]}])),
34    {ok, State1}.
35
36-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
37do(State) ->
38    IsDepsOnly = is_deps_only(State),
39    rebar_paths:set_paths([deps], State),
40
41    Providers = rebar_state:providers(State),
42    MustBuildDeps = rebar_state:deps_to_build(State),
43    Deps = rebar_state:all_deps(State),
44    CompiledDeps = copy_and_build_deps(State, Providers, MustBuildDeps, Deps),
45    State0 = rebar_state:merge_all_deps(State, CompiledDeps),
46
47    State1 = case IsDepsOnly of
48                 true ->
49                     State0;
50                 false ->
51                     handle_project_apps(Providers, State0)
52             end,
53
54    rebar_paths:set_paths([plugins], State1),
55
56    {ok, State1}.
57
58is_deps_only(State) ->
59    {Args, _} = rebar_state:command_parsed_args(State),
60    proplists:get_value(deps_only, Args, false).
61
62handle_project_apps(Providers, State) ->
63    Cwd = rebar_state:dir(State),
64    ProjectApps = rebar_state:project_apps(State),
65    {ok, ProjectApps1} = rebar_digraph:compile_order(ProjectApps),
66
67    %% Run top level hooks *before* project apps compiled but *after* deps are
68    rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State),
69
70    ProjectApps2 = copy_and_build_project_apps(State, Providers, ProjectApps1),
71    State2 = rebar_state:project_apps(State, ProjectApps2),
72
73    %% build extra_src_dirs in the root of multi-app projects
74    build_root_extras(State, ProjectApps2),
75
76    State3 = update_code_paths(State2, ProjectApps2),
77
78    rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State3),
79    case rebar_state:has_all_artifacts(State3) of
80        {false, File} ->
81            throw(?PRV_ERROR({missing_artifact, File}));
82        true ->
83            true
84    end,
85
86    State3.
87
88
89-spec format_error(any()) -> iolist().
90format_error({missing_artifact, File}) ->
91    io_lib:format("Missing artifact ~ts", [File]);
92format_error({bad_project_builder, Name, Type, Module}) ->
93    io_lib:format("Error building application ~s:~n     Required project builder ~s function "
94                  "~s:build/1 not found", [Name, Type, Module]);
95format_error({unknown_project_type, Name, Type}) ->
96    io_lib:format("Error building application ~s:~n     "
97                  "No project builder is configured for type ~s", [Name, Type]);
98format_error(Reason) ->
99    io_lib:format("~p", [Reason]).
100
101pick_deps_to_build(State, MustBuild, All, Tag) ->
102    InvalidDags = lists:any(
103        fun({_Mod, Status}) ->
104            %% A DAG that is not found can be valid, it just has no matching
105            %% source file. However, bad_vsn, bad_format, and bad_meta should
106            %% all trigger rebuilds.
107            %% Good:
108            %%  lists:member(Status, [valid, not_found])
109            %% Bad:
110            %%  lists:member(Status, [bad_vsn, bad_format, bad_meta])
111            %%
112            %% Since the fastest check is done on smaller lists handling
113            %% the common case first:
114            not lists:member(Status, [valid, not_found])
115        end,
116        check_dags(State, Tag)
117    ),
118    case InvalidDags of
119        true -> All;
120        false -> MustBuild
121    end.
122
123copy_and_build_deps(State, Providers, MustBuildApps, AllApps) ->
124    Apps = pick_deps_to_build(State, MustBuildApps, AllApps, apps),
125    Apps0 = [prepare_app(State, Providers, App) || App <- Apps],
126    compile(State, Providers, Apps0, apps).
127
128copy_and_build_project_apps(State, Providers, Apps) ->
129    Apps0 = [prepare_project_app(State, Providers, App) || App <- Apps],
130    compile(State, Providers, Apps0, project_apps).
131
132-spec compile(rebar_state:t(), [rebar_app_info:t()]) -> [rebar_app_info:t()]
133      ;      (rebar_state:t(), rebar_app_info:t()) -> rebar_app_info:t().
134compile(State, AppInfo) ->
135    compile(State, rebar_state:providers(State), AppInfo).
136
137-spec compile(rebar_state:t(), [providers:t()],
138              [rebar_app_info:t()]) -> [rebar_app_info:t()]
139      ;      (rebar_state:t(), [providers:t()],
140              rebar_app_info:t()) -> rebar_app_info:t().
141compile(State, Providers, AppInfo) when not is_list(AppInfo) ->
142    [Res] = compile(State, Providers, [AppInfo], undefined),
143    Res;
144compile(State, Providers, Apps) ->
145    compile(State, Providers, Apps, undefined).
146
147-spec compile(rebar_state:t(), [providers:t()],
148              [rebar_app_info:t()], atom() | undefined) -> [rebar_app_info:t()].
149compile(State, Providers, Apps, Tag) ->
150    ?DEBUG("Compile (~p)", [if Tag =:= undefined -> untagged; true -> Tag end]),
151    Apps1 = [prepare_compile(State, Providers, App) || App <- Apps],
152    Apps2 = [prepare_compilers(State, Providers, App) || App <- Apps1],
153    Apps3 = run_compilers(State, Providers, Apps2, Tag),
154    Apps4 = [finalize_compilers(State, Providers, App) || App <- Apps3],
155    Apps5 = [prepare_app_file(State, Providers, App) || App <- Apps4],
156    Apps6 = compile_app_files(State, Providers, Apps5),
157    Apps7 = [finalize_app_file(State, Providers, App) || App <- Apps6],
158    [finalize_compile(State, Providers, App) || App <- Apps7].
159
160prepare_app(_State, _Providers, AppInfo) ->
161    AppDir = rebar_app_info:dir(AppInfo),
162    OutDir = rebar_app_info:out_dir(AppInfo),
163    copy_app_dirs(AppInfo, AppDir, OutDir),
164    AppInfo.
165
166prepare_project_app(_State, _Providers, AppInfo) ->
167    copy_app_dirs(AppInfo,
168                  rebar_app_info:dir(AppInfo),
169                  rebar_app_info:out_dir(AppInfo)),
170    %% application code path must be added to the source
171    %%  otherwise code_server will remember 'lib_dir' for
172    %%  this application, and all `-include_lib` directives
173    %%  will actually go into _build/profile/lib/...
174    code:add_pathsz([rebar_app_info:dir(AppInfo)]),
175    AppInfo.
176
177prepare_compile(State, Providers, AppInfo) ->
178    AppDir = rebar_app_info:dir(AppInfo),
179    rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER,  Providers, AppInfo, State).
180
181prepare_compilers(State, Providers, AppInfo) ->
182    AppDir = rebar_app_info:dir(AppInfo),
183    rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo, State).
184
185run_compilers(State, _Providers, Apps, Tag) ->
186    %% Get the compiler graphs for all modules and artifacts, for each
187    %% compiler we run
188    DAGsInfo = load_dags(State, Tag),
189    DAGs = [{Mod, DAG} || {Mod, {DAG, _Meta}} <- DAGsInfo],
190    %% Compile all the apps
191    build_apps(DAGs, Apps, State),
192    %% Potentially store shared compiler DAGs so next runs can easily
193    %% share the base information for easy re-scans.
194    lists:foreach(fun({Mod, {G, {Dir, DAGLabel, CritMeta}}}) ->
195        rebar_compiler_dag:maybe_store(G, Dir, Mod, DAGLabel, CritMeta),
196        rebar_compiler_dag:terminate(G)
197    end, DAGsInfo),
198    Apps.
199
200load_dags(State, Tag) ->
201    F = fun(Dir, Mod, DAGLabel, CritMeta) ->
202            {rebar_compiler_dag:init(Dir, Mod, DAGLabel, CritMeta),
203             {Dir, DAGLabel, CritMeta}}
204        end,
205    map_dags(F, State, Tag).
206
207check_dags(State, Tag) ->
208    map_dags(fun rebar_compiler_dag:status/4, State, Tag).
209
210map_dags(F, State, Tag) ->
211    DAGLabel = format_label(Tag),
212    %% The Dir for the DAG is set to deps_dir so builds taking place
213    %% in different contexts (i.e. plugins) don't risk clobbering regular deps.
214    Dir = rebar_dir:deps_dir(State),
215    SharedCritMeta = [],
216    [{Mod, F(Dir, Mod, DAGLabel, mod_meta(Mod, SharedCritMeta))}
217     || Mod <- rebar_state:compilers(State)].
218
219mod_meta(rebar_compiler_erl, CritMeta) ->
220    application:load(compiler),
221    {ok, CompileVsn} = application:get_key(compiler, vsn),
222    [{compiler, CompileVsn} | CritMeta];
223mod_meta(_, CritMeta) ->
224    CritMeta.
225
226format_label(Tag) ->
227    %% The Tag allows to create a Label when someone cares about a specific
228    %% run for compilation;
229    case Tag of
230        undefined -> undefined;
231        _ -> atom_to_list(Tag)
232    end.
233
234finalize_compilers(State, Providers, AppInfo) ->
235    AppDir = rebar_app_info:dir(AppInfo),
236    rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo, State).
237
238prepare_app_file(State, Providers, AppInfo) ->
239    AppDir = rebar_app_info:dir(AppInfo),
240    rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo, State).
241
242compile_app_files(State, Providers, Apps) ->
243    %% Load plugins back for make_vsn calls in custom resources.
244    %% The rebar_otp_app compilation step is safe regarding the
245    %% overall path management, so we can just load all plugins back
246    %% in memory.
247    rebar_paths:set_paths([plugins], State),
248    NewApps = [compile_app_file(State, Providers, App) || App <- Apps],
249    %% Clean up after ourselves, leave things as they were with deps first
250    rebar_paths:set_paths([deps], State),
251    NewApps.
252
253compile_app_file(State, _Providers, AppInfo) ->
254    case rebar_otp_app:compile(State, AppInfo) of
255        {ok, AppInfo2} -> AppInfo2;
256        Error -> throw(Error)
257    end.
258
259finalize_app_file(State, Providers, AppInfo) ->
260    AppDir = rebar_app_info:dir(AppInfo),
261    rebar_hooks:run_all_hooks(AppDir, post, ?APP_HOOK, Providers, AppInfo, State).
262
263finalize_compile(State, Providers, AppInfo) ->
264    AppDir = rebar_app_info:dir(AppInfo),
265    AppInfo2 = rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo, State),
266    has_all_artifacts(AppInfo),
267    AppInfo2.
268
269build_root_extras(State, Apps) ->
270    %% The root extra src dirs belong to no specific applications;
271    %% because the compiler works on OTP apps, we instead build
272    %% a fake AppInfo record that only contains the root extra_src
273    %% directories, has access to all the top-level apps' public
274    %% include files, and builds to a specific extra outdir.
275    %% TODO: figure out digraph strategy to properly ensure no
276    %%       cross-contamination but proper change detection.
277    BaseDir = rebar_state:dir(State),
278    F = fun(App) -> rebar_app_info:dir(App) == BaseDir end,
279    case lists:any(F, Apps) of
280        true ->
281            [];
282        false ->
283            ProjOpts = rebar_state:opts(State),
284            Extras = rebar_dir:extra_src_dirs(ProjOpts, []),
285            {ok, VirtApp} = rebar_app_info:new("extra", "0.0.0", BaseDir, []),
286
287            case extra_virtual_apps(State, VirtApp, Extras) of
288                %% If there are no virtual apps do nothing.
289                [] ->
290                    [];
291                VirtApps ->
292                    %% re-use the project-apps digraph?
293                    run_compilers(State, [], VirtApps, project_apps)
294            end
295    end.
296
297extra_virtual_apps(_, _, []) ->
298    [];
299extra_virtual_apps(State, VApp0, [Dir|Dirs]) ->
300    SrcDir = filename:join([rebar_state:dir(State), Dir]),
301    case ec_file:is_dir(SrcDir) of
302        false ->
303            extra_virtual_apps(State, VApp0, Dirs);
304        true ->
305            BaseDir = filename:join([rebar_dir:base_dir(State), "extras"]),
306            OutDir = filename:join([BaseDir, Dir]),
307            copy(rebar_state:dir(State), BaseDir, Dir),
308            VApp1 = rebar_app_info:out_dir(VApp0, BaseDir),
309            VApp2 = rebar_app_info:ebin_dir(VApp1, OutDir),
310            Opts = rebar_state:opts(State),
311            VApp3 = rebar_app_info:opts(VApp2, Opts),
312            %% ensure we don't end up running a similar extra fake app while
313            %% compiling root-level extras by marking it explicitly null
314            VApp4 = rebar_app_info:set(VApp3, extra_src_dirs, []),
315            %% need a unique name to prevent lookup issues that clobber entries
316            AppName = unicode:characters_to_binary(["extra_", Dir]),
317            VApp5 = rebar_app_info:name(VApp4, AppName),
318            [rebar_app_info:set(VApp5, src_dirs, [OutDir])
319             | extra_virtual_apps(State, VApp0, Dirs)]
320    end.
321
322%% ===================================================================
323%% Internal functions
324%% ===================================================================
325
326build_apps(DAGs, Apps, State) ->
327    {Rebar3, Custom} = lists:partition(
328        fun(AppInfo) ->
329            Type = rebar_app_info:project_type(AppInfo),
330            Type =:= rebar3 orelse Type =:= undefined
331        end,
332        Apps
333    ),
334    [build_custom_builder_app(AppInfo, State) || AppInfo <- Custom],
335    build_rebar3_apps(DAGs, Rebar3, State).
336
337build_custom_builder_app(AppInfo, State) ->
338    ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]),
339    Type = rebar_app_info:project_type(AppInfo),
340    ProjectBuilders = rebar_state:project_builders(State),
341    case lists:keyfind(Type, 1, ProjectBuilders) of
342        {_, Module} ->
343            %% load plugins since thats where project builders would be,
344            %% prevents parallelism at this level.
345            rebar_paths:set_paths([deps, plugins], State),
346            Res = Module:build(AppInfo),
347            rebar_paths:set_paths([deps], State),
348            case Res of
349                ok -> ok;
350                {error, Reason} -> throw({error, {Module, Reason}})
351            end;
352        _ ->
353            throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type}))
354    end.
355
356build_rebar3_apps(DAGs, Apps, _State) when DAGs =:= []; Apps =:= [] ->
357    %% No apps to actually build, probably just other compile phases
358    %% to run for non-rebar3 apps, someone wanting .app files built,
359    %% or just needing the hooks to run maybe.
360    ok;
361build_rebar3_apps(DAGs, Apps, State) ->
362    rebar_paths:set_paths([deps], State),
363    %% To maintain output order, we need to mention each app being compiled
364    %% in order, even if the order isn't really there anymore due to each
365    %% compiler being run in broken sequence. The last compiler tends to be
366    %% the big ERLC one so we use the last compiler for the output.
367    LastDAG = lists:last(DAGs),
368    %% we actually need to compile each DAG one after the other to prevent
369    %% issues where a .yrl file that generates a .erl file gets to be seen.
370    ?INFO("Analyzing applications...", []),
371    [begin
372         {Ctx, ReorderedApps} = rebar_compiler:analyze_all(DAG, Apps),
373         lists:foreach(
374             fun(AppInfo) ->
375                DAG =:= LastDAG andalso
376                  ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]),
377                rebar_compiler:compile_analyzed(DAG, AppInfo, Ctx)
378             end,
379             ReorderedApps
380         ),
381         {ExtraCtx, ReorderedExtraApps} = rebar_compiler:analyze_all_extras(DAG, Apps),
382         lists:foreach(
383             fun(AppInfo) ->
384                rebar_compiler:compile_analyzed(DAG, AppInfo, ExtraCtx)
385             end,
386             ReorderedExtraApps
387         )
388     end || DAG <- DAGs],
389    ok.
390
391update_code_paths(State, ProjectApps) ->
392    ProjAppsPaths = paths_for_apps(ProjectApps),
393    ExtrasPaths = paths_for_extras(State, ProjectApps),
394    DepsPaths = rebar_state:code_paths(State, all_deps),
395    rebar_state:code_paths(State, all_deps, DepsPaths ++ ProjAppsPaths ++ ExtrasPaths).
396
397paths_for_apps(Apps) -> paths_for_apps(Apps, []).
398
399paths_for_apps([], Acc) -> Acc;
400paths_for_apps([App|Rest], Acc) ->
401    {_SrcDirs, ExtraDirs} = resolve_src_dirs(rebar_app_info:opts(App)),
402    Paths = [filename:join([rebar_app_info:out_dir(App), Dir]) || Dir <- ["ebin"|ExtraDirs]],
403    FilteredPaths = lists:filter(fun ec_file:is_dir/1, Paths),
404    paths_for_apps(Rest, Acc ++ FilteredPaths).
405
406paths_for_extras(State, Apps) ->
407    F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end,
408    %% check that this app hasn't already been dealt with
409    case lists:any(F, Apps) of
410        false -> paths_for_extras(State);
411        true  -> []
412    end.
413
414paths_for_extras(State) ->
415    {_SrcDirs, ExtraDirs} = resolve_src_dirs(rebar_state:opts(State)),
416    Paths = [filename:join([rebar_dir:base_dir(State), "extras", Dir]) || Dir <- ExtraDirs],
417    lists:filter(fun ec_file:is_dir/1, Paths).
418
419has_all_artifacts(AppInfo1) ->
420    case rebar_app_info:has_all_artifacts(AppInfo1) of
421        {false, File} ->
422            throw(?PRV_ERROR({missing_artifact, File}));
423        true ->
424            rebar_app_utils:lint_app_info(AppInfo1),
425            true
426    end.
427
428copy_app_dirs(AppInfo, OldAppDir, AppDir) ->
429    case rebar_utils:to_binary(filename:absname(OldAppDir)) =/=
430        rebar_utils:to_binary(filename:absname(AppDir)) of
431        true ->
432            EbinDir = filename:join([OldAppDir, "ebin"]),
433            %% copy all files from ebin if it exists
434            case filelib:is_dir(EbinDir) of
435                true ->
436                    OutEbin = filename:join([AppDir, "ebin"]),
437                    filelib:ensure_dir(filename:join([OutEbin, "dummy.beam"])),
438                    ?DEBUG("Copying existing files from ~ts to ~ts", [EbinDir, OutEbin]),
439                    rebar_file_utils:cp_r(filelib:wildcard(filename:join([EbinDir, "*"])), OutEbin);
440                false ->
441                    ok
442            end,
443
444            filelib:ensure_dir(filename:join([AppDir, "dummy"])),
445
446            %% link or copy mibs if it exists
447            case filelib:is_dir(filename:join([OldAppDir, "mibs"])) of
448                true ->
449                    %% If mibs exist it means we must ensure priv exists.
450                    %% mibs files are compiled to priv/mibs/
451                    filelib:ensure_dir(filename:join([OldAppDir, "priv", "dummy"])),
452                    symlink_or_copy(OldAppDir, AppDir, "mibs");
453                false ->
454                    ok
455            end,
456            {SrcDirs, ExtraDirs} = resolve_src_dirs(rebar_app_info:opts(AppInfo)),
457            %% link to src_dirs to be adjacent to ebin is needed for R15 use of cover/xref
458            %% priv/ and include/ are symlinked unconditionally to allow hooks
459            %% to write to them _after_ compilation has taken place when the
460            %% initial directory did not, and still work
461            [symlink_or_copy(OldAppDir, AppDir, Dir) || Dir <- ["priv", "include"]],
462            [symlink_or_copy_existing(OldAppDir, AppDir, Dir) || Dir <- SrcDirs],
463            %% copy all extra_src_dirs as they build into themselves and linking means they
464            %% are shared across profiles
465            [copy(OldAppDir, AppDir, Dir) || Dir <- ExtraDirs];
466        false ->
467            ok
468    end.
469
470symlink_or_copy(OldAppDir, AppDir, Dir) ->
471    Source = filename:join([OldAppDir, Dir]),
472    Target = filename:join([AppDir, Dir]),
473    rebar_file_utils:symlink_or_copy(Source, Target).
474
475symlink_or_copy_existing(OldAppDir, AppDir, Dir) ->
476    Source = filename:join([OldAppDir, Dir]),
477    Target = filename:join([AppDir, Dir]),
478    case ec_file:is_dir(Source) of
479        true -> rebar_file_utils:symlink_or_copy(Source, Target);
480        false -> ok
481    end.
482
483copy(OldAppDir, AppDir, Dir) ->
484    Source = filename:join([OldAppDir, Dir]),
485    Target = filename:join([AppDir, Dir]),
486    case ec_file:is_dir(Source) of
487        true  -> copy(Source, Target);
488        false -> ok
489    end.
490
491%% TODO: use ec_file:copy/2 to do this, it preserves timestamps and
492%% may prevent recompilation of files in extra dirs
493copy(Source, Source) ->
494    %% allow users to specify a directory in _build as a directory
495    %% containing additional source/tests
496    ok;
497copy(Source, Target) ->
498    %% important to do this so no files are copied onto themselves
499    %% which truncates them to zero length on some platforms
500    ok = delete_if_symlink(Target),
501    ok = filelib:ensure_dir(filename:join([Target, "dummy.beam"])),
502    {ok, Files} = rebar_utils:list_dir(Source),
503    case [filename:join([Source, F]) || F <- Files] of
504        []    -> ok;
505        Paths -> rebar_file_utils:cp_r(Paths, Target)
506    end.
507
508delete_if_symlink(Path) ->
509    case ec_file:is_symlink(Path) of
510        true  -> file:delete(Path);
511        false -> ok
512    end.
513
514resolve_src_dirs(Opts) ->
515    SrcDirs = rebar_dir:src_dirs(Opts, ["src"]),
516    ExtraDirs = rebar_dir:extra_src_dirs(Opts, []),
517    normalize_src_dirs(SrcDirs, ExtraDirs).
518
519%% remove duplicates and make sure no directories that exist
520%% in src_dirs also exist in extra_src_dirs
521normalize_src_dirs(SrcDirs, ExtraDirs) ->
522    S = lists:usort(SrcDirs),
523    E = lists:subtract(lists:usort(ExtraDirs), S),
524    ok = warn_on_problematic_directories(S ++ E),
525    {S, E}.
526
527%% warn when directories called `eunit' and `ct' are added to compile dirs
528warn_on_problematic_directories(AllDirs) ->
529    F = fun(Dir) ->
530        case is_a_problem(Dir) of
531            true  -> ?WARN("Possible name clash with directory ~p.", [Dir]);
532            false -> ok
533        end
534    end,
535    lists:foreach(F, AllDirs).
536
537is_a_problem("eunit") -> true;
538is_a_problem("common_test") -> true;
539is_a_problem(_) -> false.
540