1%% Test suite for the rebar pkg index caching and decompression 2%% mechanisms. 3-module(rebar_pkg_SUITE). 4-compile(export_all). 5-include_lib("common_test/include/ct.hrl"). 6-include_lib("eunit/include/eunit.hrl"). 7-include("rebar.hrl"). 8 9-define(bad_etag, <<"abcdef">>). 10-define(good_etag, <<"22e1d7387c9085a462340088a2a8ba67">>). 11-define(badpkg_checksum, <<"A14E3718B33F8124E98004433193509EC6660F6CA03302657CAB8785751D77A0">>). 12-define(badindex_checksum, <<"7B2CBED315C89F3126B5BF553DD7FF0FB5FE94B064888DD1B095CE8BF4B6A16A">>). 13-define(bad_checksum, <<"D576B442A68C7B92BACDE1EFE9C6E54D8D6C74BDB71D8175B9D3C6EC8C7B62A7">>). 14-define(good_checksum, <<"ABA3B638A653A2414BF9DFAF76D90C937C53D1BE5B5D51A990C6FCC3A775C6F">>). 15-define(BADPKG_ETAG, <<"BADETAG">>). 16 17all() -> [good_uncached, good_cached, badpkg, badhash_nocache, 18 badindexchk, badhash_cache, bad_to_good, good_disconnect, 19 bad_disconnect, pkgs_provider, find_highest_matching]. 20 21init_per_suite(Config) -> 22 application:start(meck), 23 Config. 24 25end_per_suite(_Config) -> 26 application:stop(meck). 27 28init_per_testcase(pkgs_provider=Name, Config) -> 29 %% Need to mock out a registry for this test now because it will try to update it automatically 30 Priv = ?config(priv_dir, Config), 31 Tid = ets:new(registry_table, [public]), 32 ets:insert_new(Tid, []), 33 CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]), 34 CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), 35 filelib:ensure_dir(filename:join([CacheDir, "registry"])), 36 ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])), 37 Config; 38init_per_testcase(good_uncached=Name, Config0) -> 39 Config = [{good_cache, false}, 40 {pkg, {<<"goodpkg">>, <<"1.0.0">>}} 41 | Config0], 42 mock_config(Name, Config); 43init_per_testcase(good_cached=Name, Config0) -> 44 Pkg = {<<"goodpkg">>, <<"1.0.0">>}, 45 Config1 = [{good_cache, true}, 46 {pkg, Pkg} 47 | Config0], 48 Config = mock_config(Name, Config1), 49 copy_to_cache(Pkg, Config), 50 Config; 51init_per_testcase(badindexchk=Name, Config0) -> 52 Config = [{good_cache, false}, 53 {pkg, {<<"badindexchk">>, <<"1.0.0">>}} 54 | Config0], 55 mock_config(Name, Config); 56init_per_testcase(badpkg=Name, Config0) -> 57 Config = [{good_cache, false}, 58 {pkg, {<<"badpkg">>, <<"1.0.0">>}} 59 | Config0], 60 mock_config(Name, Config); 61init_per_testcase(badhash_nocache=Name, Config0) -> 62 Config = [{good_cache, false}, 63 {pkg, {<<"goodpkg">>, <<"1.0.0">>}} 64 | Config0], 65 mock_config(Name, Config); 66init_per_testcase(badhash_cache=Name, Config0) -> 67 Pkg = {<<"goodpkg">>, <<"1.0.0">>}, 68 Config1 = [{good_cache, true}, 69 {pkg, Pkg} 70 | Config0], 71 Config = mock_config(Name, Config1), 72 copy_to_cache(Pkg, Config), 73 Config; 74init_per_testcase(bad_to_good=Name, Config0) -> 75 Config1 = [{good_cache, false}, 76 {pkg, {<<"goodpkg">>, <<"1.0.0">>}} 77 | Config0], 78 Config = mock_config(Name, Config1), 79 Source = filename:join(?config(data_dir, Config), <<"badpkg-1.0.0.tar">>), 80 Dest = filename:join(?config(cache_dir, Config), <<"goodpkg-1.0.0.tar">>), 81 ec_file:copy(Source, Dest), 82 Config; 83init_per_testcase(good_disconnect=Name, Config0) -> 84 Pkg = {<<"goodpkg">>, <<"1.0.0">>}, 85 Config1 = [{good_cache, false}, 86 {pkg, Pkg} 87 | Config0], 88 Config = mock_config(Name, Config1), 89 copy_to_cache(Pkg, Config), 90 %% meck:unload(httpc), 91 meck:new(httpc, [passthrough, unsticky]), 92 meck:expect(httpc, request, fun(_, _, _, _) -> {error, econnrefused} end), 93 Config; 94init_per_testcase(bad_disconnect=Name, Config0) -> 95 Pkg = {<<"goodpkg">>, <<"1.0.0">>}, 96 Config1 = [{good_cache, false}, 97 {pkg, Pkg} 98 | Config0], 99 Config = mock_config(Name, Config1), 100 meck:expect(r3_hex_repo, get_tarball, fun(_, _, _) -> 101 {error, econnrefused} 102 end), 103 Config; 104init_per_testcase(Name, Config0) -> 105 Config = [{good_cache, false}, 106 {pkg, {<<"goodpkg">>, <<"1.0.0">>}} 107 | Config0], 108 mock_config(Name, Config). 109 110end_per_testcase(_, Config) -> 111 unmock_config(Config), 112 Config. 113 114good_uncached(Config) -> 115 Tmp = ?config(tmp_dir, Config), 116 {Pkg,Vsn} = ?config(pkg, Config), 117 State = ?config(state, Config), 118 ?assertEqual(ok, 119 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)), 120 Cache = ?config(cache_dir, Config), 121 ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))). 122 123good_cached(Config) -> 124 Tmp = ?config(tmp_dir, Config), 125 {Pkg,Vsn} = ?config(pkg, Config), 126 State = ?config(state, Config), 127 Cache = ?config(cache_dir, Config), 128 CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), 129 ?assert(filelib:is_regular(CachedFile)), 130 {ok, Content} = file:read_file(CachedFile), 131 ?assertEqual(ok, 132 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)), 133 {ok, Content} = file:read_file(CachedFile). 134 135 136badindexchk(Config) -> 137 Tmp = ?config(tmp_dir, Config), 138 {Pkg,Vsn} = ?config(pkg, Config), 139 State = ?config(state, Config), 140 ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, 141 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, ?bad_checksum, #{}}, State, #{}, true)), 142 %% The cached file is there for forensic purposes 143 Cache = ?config(cache_dir, Config), 144 ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))). 145 146badpkg(Config) -> 147 Tmp = ?config(tmp_dir, Config), 148 {Pkg,Vsn} = ?config(pkg, Config), 149 State = ?config(state, Config), 150 Cache = ?config(cache_dir, Config), 151 CachePath = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), 152 ETagPath = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".etag">>), 153 rebar_pkg_resource:store_etag_in_cache(ETagPath, ?BADPKG_ETAG), 154 ?assertMatch({error, {hex_tarball, {tarball, {inner_checksum_mismatch, _, _}}}}, 155 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?badpkg_checksum, ?badpkg_checksum, #{}}, State, #{}, false)), 156 %% The cached/etag files are there for forensic purposes 157 ?assert(filelib:is_regular(ETagPath)), 158 ?assert(filelib:is_regular(CachePath)). 159 160badhash_nocache(Config) -> 161 Tmp = ?config(tmp_dir, Config), 162 {Pkg,Vsn} = ?config(pkg, Config), 163 State = ?config(state, Config), 164 ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, 165 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, ?bad_checksum, #{}}, State, #{}, true)), 166 %% The cached file is there for forensic purposes 167 Cache = ?config(cache_dir, Config), 168 ?assert(filelib:is_regular(filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>))). 169 170badhash_cache(Config) -> 171 Tmp = ?config(tmp_dir, Config), 172 {Pkg,Vsn} = ?config(pkg, Config), 173 Cache = ?config(cache_dir, Config), 174 State = ?config(state, Config), 175 CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), 176 ?assert(filelib:is_regular(CachedFile)), 177 {ok, Content} = file:read_file(CachedFile), 178 ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, 179 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, ?bad_checksum, #{}}, State, #{}, true)), 180 %% The cached file is there still, unchanged. 181 ?assert(filelib:is_regular(CachedFile)), 182 ?assertEqual({ok, Content}, file:read_file(CachedFile)). 183 184bad_to_good(Config) -> 185 Tmp = ?config(tmp_dir, Config), 186 {Pkg,Vsn} = ?config(pkg, Config), 187 State = ?config(state, Config), 188 Cache = ?config(cache_dir, Config), 189 CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), 190 ?assert(filelib:is_regular(CachedFile)), 191 {ok, Contents} = file:read_file(CachedFile), 192 ?assertEqual(ok, 193 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)), 194 %% Cache has refreshed 195 ?assert({ok, Contents} =/= file:read_file(CachedFile)). 196 197good_disconnect(Config) -> 198 Tmp = ?config(tmp_dir, Config), 199 {Pkg,Vsn} = ?config(pkg, Config), 200 State = ?config(state, Config), 201 Cache = ?config(cache_dir, Config), 202 CachedFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".tar">>), 203 ETagFile = filename:join(Cache, <<Pkg/binary, "-", Vsn/binary, ".etag">>), 204 ?assert(filelib:is_regular(CachedFile)), 205 {ok, Content} = file:read_file(CachedFile), 206 rebar_pkg_resource:store_etag_in_cache(ETagFile, ?BADPKG_ETAG), 207 ?assertEqual(ok, 208 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)), 209 {ok, Content} = file:read_file(CachedFile). 210 211bad_disconnect(Config) -> 212 Tmp = ?config(tmp_dir, Config), 213 {Pkg,Vsn} = ?config(pkg, Config), 214 State = ?config(state, Config), 215 ?assertEqual({fetch_fail, Pkg, Vsn}, 216 rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)). 217 218pkgs_provider(Config) -> 219 Config1 = rebar_test_utils:init_rebar_state(Config), 220 rebar_test_utils:run_and_check( 221 Config1, [], ["pkgs", "relx"], 222 {ok, []} 223 ). 224 225find_highest_matching(_Config) -> 226 State = rebar_state:new(), 227 {ok, Vsn} = rebar_packages:find_highest_matching_( 228 <<"goodpkg">>, ec_semver:parse(<<"1.0.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), 229 ?assertEqual({{1,0,1},{[],[]}}, Vsn), 230 {ok, Vsn1} = rebar_packages:find_highest_matching( 231 <<"goodpkg">>, ec_semver:parse(<<"1.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), 232 ?assertEqual({{1,1,1},{[],[]}}, Vsn1), 233 {ok, Vsn2} = rebar_packages:find_highest_matching( 234 <<"goodpkg">>, ec_semver:parse(<<"2.0">>), #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), 235 ?assertEqual({{2,0,0},{[],[]}}, Vsn2), 236 237 %% regression test. ~> constraints higher than the available packages would result 238 %% in returning the first package version instead of 'none'. 239 ?assertEqual(none, rebar_packages:find_highest_matching_(<<"goodpkg">>, ec_semver:parse(<<"5.0">>), 240 #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State)), 241 242 243 {ok, Vsn3} = rebar_packages:find_highest_matching_(<<"goodpkg">>, ec_semver:parse(<<"3.0.0-rc.0">>), 244 #{name => <<"hexpm">>}, ?PACKAGE_TABLE, State), 245 ?assertEqual({{3,0,0},{[<<"rc">>,0],[]}}, Vsn3). 246 247%%%%%%%%%%%%%%% 248%%% Helpers %%% 249%%%%%%%%%%%%%%% 250mock_config(Name, Config) -> 251 Priv = ?config(priv_dir, Config), 252 CacheRoot = filename:join([Priv, "cache", atom_to_list(Name)]), 253 TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]), 254 Tid = ets:new(registry_table, [public]), 255 AllDeps = [ 256 {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, ?bad_checksum, [<<"rebar3">>]]}, 257 {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]}, 258 {{<<"goodpkg">>,<<"1.0.1">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]}, 259 {{<<"goodpkg">>,<<"1.1.1">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]}, 260 {{<<"goodpkg">>,<<"2.0.0">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]}, 261 {{<<"goodpkg">>,<<"3.0.0-rc.0">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]}, 262 {{<<"badpkg">>,<<"1.0.0">>}, [[], ?badpkg_checksum, ?badpkg_checksum, [<<"rebar3">>]]} 263 ], 264 ets:insert_new(Tid, AllDeps), 265 CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), 266 filelib:ensure_dir(filename:join([CacheDir, "registry"])), 267 ok = ets:tab2file(Tid, filename:join([CacheDir, "registry"])), 268 269 catch ets:delete(?PACKAGE_TABLE), 270 rebar_packages:new_package_table(), 271 lists:foreach(fun({{N, Vsn}, [Deps, InnerChecksum, OuterChecksum, _]}) -> 272 case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of 273 false -> 274 ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>}, 275 dependencies=Deps, 276 retired=false, 277 inner_checksum=InnerChecksum, 278 outer_checksum=OuterChecksum}); 279 true -> 280 ok 281 end 282 end, AllDeps), 283 284 285 meck:new(r3_hex_repo, [passthrough]), 286 meck:expect(r3_hex_repo, get_package, 287 fun(_Config, PkgName) -> 288 Matches = ets:match_object(Tid, {{PkgName,'_'}, '_'}), 289 Releases = 290 [#{outer_checksum => OuterChecksum, 291 inner_checksum => InnerChecksum, 292 version => Vsn, 293 dependencies => Deps} || 294 {{_, Vsn}, [Deps, InnerChecksum, OuterChecksum, _]} <- Matches], 295 {ok, {200, #{}, Releases}} 296 end), 297 298 %% The state returns us a fake registry 299 meck:new(rebar_state, [passthrough]), 300 meck:expect(rebar_state, get, 301 fun(_State, rebar_packages_cdn, _Default) -> 302 "http://test.com/"; 303 (_, _, Default) -> 304 Default 305 end), 306 meck:expect(rebar_state, resources, 307 fun(_State) -> 308 DefaultConfig = r3_hex_core:default_config(), 309 [rebar_resource_v2:new(pkg, rebar_pkg_resource, 310 #{repos => [DefaultConfig#{name => <<"hexpm">>}], 311 base_config => #{}})] 312 end), 313 314 meck:new(rebar_dir, [passthrough]), 315 meck:expect(rebar_dir, global_cache_dir, fun(_) -> CacheRoot end), 316 317 meck:expect(rebar_packages, registry_dir, fun(_) -> {ok, CacheDir} end), 318 meck:expect(rebar_packages, package_dir, fun(_, _) -> {ok, CacheDir} end), 319 320 meck:new(rebar_prv_update, [passthrough]), 321 meck:expect(rebar_prv_update, do, fun(State) -> {ok, State} end), 322 323 %% Cache fetches are mocked -- we assume the server and clients are 324 %% correctly used. 325 GoodCache = ?config(good_cache, Config), 326 {Pkg,Vsn} = ?config(pkg, Config), 327 PkgFile = <<Pkg/binary, "-", Vsn/binary, ".tar">>, 328 {ok, PkgContents} = file:read_file(filename:join(?config(data_dir, Config), PkgFile)), 329 330 meck:expect(r3_hex_repo, get_tarball, fun(_, _, _) when GoodCache -> 331 {ok, {304, #{<<"etag">> => ?good_etag}, <<>>}}; 332 (_, _, _) -> 333 {ok, {200, #{<<"etag">> => ?good_etag}, PkgContents}} 334 end), 335 336 [{cache_root, CacheRoot}, 337 {cache_dir, CacheDir}, 338 {tmp_dir, TmpDir}, 339 {mock_table, Tid} | Config]. 340 341unmock_config(Config) -> 342 meck:unload(), 343 catch ets:delete(?config(mock_table, Config)). 344 345copy_to_cache({Pkg,Vsn}, Config) -> 346 Name = <<Pkg/binary, "-", Vsn/binary, ".tar">>, 347 Source = filename:join(?config(data_dir, Config), Name), 348 Dest = filename:join(?config(cache_dir, Config), Name), 349 ec_file:copy(Source, Dest). 350