1%% Test suite for the rebar pkg index caching and decompression
2%% mechanisms.
3-module(rebar_pkg_SUITE).
4-compile(export_all).
5-include_lib("common_test/include/ct.hrl").
6-include_lib("eunit/include/eunit.hrl").
7-include("rebar.hrl").
8
9-define(bad_etag, <<"abcdef">>).
10-define(good_etag, <<"22e1d7387c9085a462340088a2a8ba67">>).
11-define(badpkg_checksum, <<"A14E3718B33F8124E98004433193509EC6660F6CA03302657CAB8785751D77A0">>).
12-define(badindex_checksum, <<"7B2CBED315C89F3126B5BF553DD7FF0FB5FE94B064888DD1B095CE8BF4B6A16A">>).
13-define(bad_checksum, <<"D576B442A68C7B92BACDE1EFE9C6E54D8D6C74BDB71D8175B9D3C6EC8C7B62A7">>).
14-define(good_checksum, <<"ABA3B638A653A2414BF9DFAF76D90C937C53D1BE5B5D51A990C6FCC3A775C6F">>).
15-define(BADPKG_ETAG, <<"BADETAG">>).
16
17all() -> [good_uncached, good_cached, badpkg, badhash_nocache,
18          badindexchk, badhash_cache, bad_to_good, good_disconnect,
19          bad_disconnect, pkgs_provider, find_highest_matching].
20
21init_per_suite(Config) ->
22    application:start(meck),
23    Config.
24
25end_per_suite(_Config) ->
26    application:stop(meck).
27
28init_per_testcase(pkgs_provider=Name, Config) ->
29    %% Need to mock out a registry for this test now because it will try to update it automatically
30    Priv = ?config(priv_dir, Config),
31    Tid = ets:new(registry_table, [public]),
32    ets:insert_new(Tid, []),
33    CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]),
34    CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
35    filelib:ensure_dir(filename:join([CacheDir, "registry"])),
36    ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])),
37    Config;
38init_per_testcase(good_uncached=Name, Config0) ->
39    Config = [{good_cache, false},
40              {pkg, {<<"goodpkg">>, <<"1.0.0">>}}
41             | Config0],
42    mock_config(Name, Config);
43init_per_testcase(good_cached=Name, Config0) ->
44    Pkg = {<<"goodpkg">>, <<"1.0.0">>},
45    Config1 = [{good_cache, true},
46               {pkg, Pkg}
47              | Config0],
48    Config = mock_config(Name, Config1),
49    copy_to_cache(Pkg, Config),
50    Config;
51init_per_testcase(badindexchk=Name, Config0) ->
52    Config = [{good_cache, false},
53              {pkg, {<<"badindexchk">>, <<"1.0.0">>}}
54             | Config0],
55    mock_config(Name, Config);
56init_per_testcase(badpkg=Name, Config0) ->
57    Config = [{good_cache, false},
58              {pkg, {<<"badpkg">>, <<"1.0.0">>}}
59             | Config0],
60    mock_config(Name, Config);
61init_per_testcase(badhash_nocache=Name, Config0) ->
62    Config = [{good_cache, false},
63              {pkg, {<<"goodpkg">>, <<"1.0.0">>}}
64             | Config0],
65    mock_config(Name, Config);
66init_per_testcase(badhash_cache=Name, Config0) ->
67    Pkg = {<<"goodpkg">>, <<"1.0.0">>},
68    Config1 = [{good_cache, true},
69               {pkg, Pkg}
70              | Config0],
71    Config = mock_config(Name, Config1),
72    copy_to_cache(Pkg, Config),
73    Config;
74init_per_testcase(bad_to_good=Name, Config0) ->
75    Config1 = [{good_cache, false},
76              {pkg, {<<"goodpkg">>, <<"1.0.0">>}}
77              | Config0],
78    Config = mock_config(Name, Config1),
79    Source = filename:join(?config(data_dir, Config), <<"badpkg-1.0.0.tar">>),
80    Dest = filename:join(?config(cache_dir, Config), <<"goodpkg-1.0.0.tar">>),
81    ec_file:copy(Source, Dest),
82    Config;
83init_per_testcase(good_disconnect=Name, Config0) ->
84    Pkg = {<<"goodpkg">>, <<"1.0.0">>},
85    Config1 = [{good_cache, false},
86               {pkg, Pkg}
87              | Config0],
88    Config = mock_config(Name, Config1),
89    copy_to_cache(Pkg, Config),
90    %% meck:unload(httpc),
91    meck:new(httpc, [passthrough, unsticky]),
92    meck:expect(httpc, request, fun(_, _, _, _) -> {error, econnrefused} end),
93    Config;
94init_per_testcase(bad_disconnect=Name, Config0) ->
95    Pkg = {<<"goodpkg">>, <<"1.0.0">>},
96    Config1 = [{good_cache, false},
97               {pkg, Pkg}
98              | Config0],
99    Config = mock_config(Name, Config1),
100    meck:expect(r3_hex_repo, get_tarball, fun(_, _, _) ->
101                                               {error, econnrefused}
102                                       end),
103    Config;
104init_per_testcase(Name, Config0) ->
105    Config = [{good_cache, false},
106              {pkg, {<<"goodpkg">>, <<"1.0.0">>}}
107              | Config0],
108    mock_config(Name, Config).
109
110end_per_testcase(_, Config) ->
111    unmock_config(Config),
112    Config.
113
114good_uncached(Config) ->
115    Tmp = ?config(tmp_dir, Config),
116    {Pkg,Vsn} = ?config(pkg, Config),
117    State = ?config(state, Config),
118    ?assertEqual(ok,
119                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)),
120    Cache = ?config(cache_dir, Config),
121    ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
122
123good_cached(Config) ->
124    Tmp = ?config(tmp_dir, Config),
125    {Pkg,Vsn} = ?config(pkg, Config),
126    State = ?config(state, Config),
127    Cache = ?config(cache_dir, Config),
128    CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
129    ?assert(filelib:is_regular(CachedFile)),
130    {ok, Content} = file:read_file(CachedFile),
131    ?assertEqual(ok,
132                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)),
133    {ok, Content} = file:read_file(CachedFile).
134
135
136badindexchk(Config) ->
137    Tmp = ?config(tmp_dir, Config),
138    {Pkg,Vsn} = ?config(pkg, Config),
139    State = ?config(state, Config),
140    ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}},
141                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, ?bad_checksum, #{}}, State, #{}, true)),
142    %% The cached file is there for forensic purposes
143    Cache = ?config(cache_dir, Config),
144    ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
145
146badpkg(Config) ->
147    Tmp = ?config(tmp_dir, Config),
148    {Pkg,Vsn} = ?config(pkg, Config),
149    State = ?config(state, Config),
150    Cache = ?config(cache_dir, Config),
151    CachePath = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
152    ETagPath = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".etag">>),
153    rebar_pkg_resource:store_etag_in_cache(ETagPath, ?BADPKG_ETAG),
154    ?assertMatch({error, {hex_tarball, {tarball, {inner_checksum_mismatch, _, _}}}},
155                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?badpkg_checksum, ?badpkg_checksum, #{}}, State, #{}, false)),
156    %% The cached/etag files are there for forensic purposes
157    ?assert(filelib:is_regular(ETagPath)),
158    ?assert(filelib:is_regular(CachePath)).
159
160badhash_nocache(Config) ->
161    Tmp = ?config(tmp_dir, Config),
162    {Pkg,Vsn} = ?config(pkg, Config),
163    State = ?config(state, Config),
164    ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}},
165                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, ?bad_checksum, #{}}, State, #{}, true)),
166    %% The cached file is there for forensic purposes
167    Cache = ?config(cache_dir, Config),
168    ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))).
169
170badhash_cache(Config) ->
171    Tmp = ?config(tmp_dir, Config),
172    {Pkg,Vsn} = ?config(pkg, Config),
173    Cache = ?config(cache_dir, Config),
174    State = ?config(state, Config),
175    CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
176    ?assert(filelib:is_regular(CachedFile)),
177    {ok, Content} = file:read_file(CachedFile),
178    ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}},
179                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, ?bad_checksum, #{}}, State, #{}, true)),
180    %% The cached file is there still, unchanged.
181    ?assert(filelib:is_regular(CachedFile)),
182    ?assertEqual({ok, Content}, file:read_file(CachedFile)).
183
184bad_to_good(Config) ->
185    Tmp = ?config(tmp_dir, Config),
186    {Pkg,Vsn} = ?config(pkg, Config),
187    State = ?config(state, Config),
188    Cache = ?config(cache_dir, Config),
189    CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
190    ?assert(filelib:is_regular(CachedFile)),
191    {ok, Contents} = file:read_file(CachedFile),
192    ?assertEqual(ok,
193                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)),
194    %% Cache has refreshed
195    ?assert({ok, Contents} =/= file:read_file(CachedFile)).
196
197good_disconnect(Config) ->
198    Tmp = ?config(tmp_dir, Config),
199    {Pkg,Vsn} = ?config(pkg, Config),
200    State = ?config(state, Config),
201    Cache = ?config(cache_dir, Config),
202    CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>),
203    ETagFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".etag">>),
204    ?assert(filelib:is_regular(CachedFile)),
205    {ok, Content} = file:read_file(CachedFile),
206    rebar_pkg_resource:store_etag_in_cache(ETagFile, ?BADPKG_ETAG),
207    ?assertEqual(ok,
208                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)),
209    {ok, Content} = file:read_file(CachedFile).
210
211bad_disconnect(Config) ->
212    Tmp = ?config(tmp_dir, Config),
213    {Pkg,Vsn} = ?config(pkg, Config),
214    State = ?config(state, Config),
215    ?assertEqual({fetch_fail, Pkg, Vsn},
216                 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)).
217
218pkgs_provider(Config) ->
219    Config1 = rebar_test_utils:init_rebar_state(Config),
220    rebar_test_utils:run_and_check(
221      Config1, [], ["pkgs", "relx"],
222        {ok, []}
223    ).
224
225find_highest_matching(_Config) ->
226    State = rebar_state:new(),
227    {ok, Vsn} = rebar_packages:find_highest_matching_(
228                  <<"goodpkg">>, ec_semver:parse(<<"1.0.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State),
229    ?assertEqual({{1,0,1},{[],[]}}, Vsn),
230    {ok, Vsn1} = rebar_packages:find_highest_matching(
231                   <<"goodpkg">>, ec_semver:parse(<<"1.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State),
232    ?assertEqual({{1,1,1},{[],[]}}, Vsn1),
233    {ok, Vsn2} = rebar_packages:find_highest_matching(
234                   <<"goodpkg">>, ec_semver:parse(<<"2.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State),
235    ?assertEqual({{2,0,0},{[],[]}}, Vsn2),
236
237    %% regression test. ~> constraints higher than the available packages would result
238    %% in returning the first package version instead of 'none'.
239    ?assertEqual(none, rebar_packages:find_highest_matching_(<<"goodpkg">>, ec_semver:parse(<<"5.0">>),
240                                                             #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State)),
241
242
243    {ok, Vsn3} = rebar_packages:find_highest_matching_(<<"goodpkg">>, ec_semver:parse(<<"3.0.0-rc.0">>),
244                                                       #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State),
245    ?assertEqual({{3,0,0},{[<<"rc">>,0],[]}}, Vsn3).
246
247%%%%%%%%%%%%%%%
248%%% Helpers %%%
249%%%%%%%%%%%%%%%
250mock_config(Name, Config) ->
251    Priv = ?config(priv_dir, Config),
252    CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]),
253    TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]),
254    Tid = ets:new(registry_table, [public]),
255    AllDeps = [
256        {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, ?bad_checksum, [<<"rebar3">>]]},
257        {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]},
258        {{<<"goodpkg">>,<<"1.0.1">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]},
259        {{<<"goodpkg">>,<<"1.1.1">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]},
260        {{<<"goodpkg">>,<<"2.0.0">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]},
261        {{<<"goodpkg">>,<<"3.0.0-rc.0">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]},
262        {{<<"badpkg">>,<<"1.0.0">>}, [[], ?badpkg_checksum, ?badpkg_checksum, [<<"rebar3">>]]}
263    ],
264    ets:insert_new(Tid, AllDeps),
265    CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]),
266    filelib:ensure_dir(filename:join([CacheDir, "registry"])),
267    ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])),
268
269    catch ets:delete(?PACKAGE_TABLE),
270    rebar_packages:new_package_table(),
271    lists:foreach(fun({{N, Vsn}, [Deps, InnerChecksum, OuterChecksum, _]}) ->
272                          case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of
273                              false ->
274                                  ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>},
275                                                                      dependencies=Deps,
276                                                                      retired=false,
277                                                                      inner_checksum=InnerChecksum,
278                                                                      outer_checksum=OuterChecksum});
279                              true ->
280                                  ok
281                          end
282                  end, AllDeps),
283
284
285    meck:new(r3_hex_repo, [passthrough]),
286    meck:expect(r3_hex_repo, get_package,
287                fun(_Config, PkgName) ->
288                        Matches = ets:match_object(Tid, {{PkgName,'_'}, '_'}),
289                        Releases =
290                            [#{outer_checksum => OuterChecksum,
291                               inner_checksum => InnerChecksum,
292                               version => Vsn,
293                               dependencies => Deps} ||
294                                {{_, Vsn}, [Deps, InnerChecksum, OuterChecksum, _]} <- Matches],
295                        {ok, {200, #{}, Releases}}
296                end),
297
298    %% The state returns us a fake registry
299    meck:new(rebar_state, [passthrough]),
300    meck:expect(rebar_state, get,
301                fun(_State, rebar_packages_cdn, _Default) ->
302                        "http://test.com/";
303                   (_, _, Default) ->
304                        Default
305                end),
306    meck:expect(rebar_state, resources,
307                fun(_State) ->
308                        DefaultConfig = r3_hex_core:default_config(),
309                        [rebar_resource_v2:new(pkg, rebar_pkg_resource,
310                                               #{repos => [DefaultConfig#{name => <<"hexpm">>}],
311                                                 base_config => #{}})]
312                end),
313
314    meck:new(rebar_dir, [passthrough]),
315    meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end),
316
317    meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end),
318    meck:expect(rebar_packages, package_dir, fun(_, _) -> {ok, CacheDir} end),
319
320    meck:new(rebar_prv_update, [passthrough]),
321    meck:expect(rebar_prv_update, do, fun(State) -> {ok, State} end),
322
323    %% Cache fetches are mocked -- we assume the server and clients are
324    %% correctly used.
325    GoodCache = ?config(good_cache, Config),
326    {Pkg,Vsn} = ?config(pkg, Config),
327    PkgFile = <<Pkg/binary, "-", Vsn/binary, ".tar">>,
328    {ok, PkgContents} = file:read_file(filename:join(?config(data_dir, Config), PkgFile)),
329
330    meck:expect(r3_hex_repo, get_tarball, fun(_, _, _) when GoodCache ->
331                                               {ok, {304, #{<<"etag">> => ?good_etag}, <<>>}};
332                                          (_, _, _) ->
333                                               {ok, {200, #{<<"etag">> => ?good_etag}, PkgContents}}
334                                       end),
335
336    [{cache_root, CacheRoot},
337     {cache_dir, CacheDir},
338     {tmp_dir, TmpDir},
339     {mock_table, Tid} | Config].
340
341unmock_config(Config) ->
342    meck:unload(),
343    catch ets:delete(?config(mock_table, Config)).
344
345copy_to_cache({Pkg,Vsn}, Config) ->
346    Name = <<Pkg/binary, "-", Vsn/binary, ".tar">>,
347    Source = filename:join(?config(data_dir, Config), Name),
348    Dest = filename:join(?config(cache_dir, Config), Name),
349    ec_file:copy(Source, Dest).
350