Vendor all dependencies, make bootstrap use them

And fix Dialyzer oddities

Author: ferd

apps/rebar/src/rebar_prv_dialyzer.erl

@@ -246,6 +246,8 @@ get_files(State, Apps, SkipApps, Mods, SkipMods, ExtraDirs) ->
     Files1 = extras_files(BaseDir, ExtraDirs, Files0),
     ExcludeMods = get_config(State, exclude_mods, []),
     Files2 = mods_files(Mods, ExcludeMods ++ SkipMods, Files1),
+    ?DEBUG("{dialyzer, [{exclude_apps, ~p}, {exclude_mods, ~p}]}.",
+           [ExcludeApps, ExcludeMods]),
     dict:fold(fun(_, File, Acc) -> [File | Acc] end, [], Files2).
 
 apps_files([], _, _ExtraDirs, Files) ->

apps/rebar/src/rebar_prv_vendor.erl

@@ -137,9 +137,13 @@ check_project_layout(State) ->
 
 vendor_plugins(State, PluginVDir) ->
     PluginDir = rebar_dir:plugins_dir(State),
-    {ok, Files} = file:list_dir_all(PluginDir),
-    [rebar_file_utils:mv(Path, filename:join(PluginVDir, PathPart))
-     || PathPart <- Files,
-        Path <- [filename:join(PluginDir, PathPart)],
-        filelib:is_dir(Path)],
-    ok.
+    case file:list_dir_all(PluginDir) of
+        {ok, Files} ->
+            [rebar_file_utils:mv(Path, filename:join(PluginVDir, PathPart))
+            || PathPart <- Files,
+                Path <- [filename:join(PluginDir, PathPart)],
+                filelib:is_dir(Path)],
+            ok;
+        {error, enoent} ->
+            ok
+    end.

bootstrap

@@ -89,12 +89,12 @@ ensure_app(App) ->
     end.
 
 fetch_and_compile({Name, ErlFirstFiles}, Deps) ->
-    case lists:keyfind(Name, 1, Deps) of
-        {Name, Vsn} ->
-            ok = fetch({pkg, atom_to_binary(Name, utf8), list_to_binary(Vsn)}, Name);
-        {Name, _, Source} ->
-            ok = fetch(Source, Name)
-    end,
+    %% Use vendored dependencies, don't hit the network; to update
+    %% dependencies first bootstrap, then use the escript to do
+    %% fancier dep management.
+    filelib:ensure_dir("_build/default/lib/.touch"),
+    [cp_r([DepDir], "_build/default/lib/")
+     || {Dep, DepDir} <- Deps, atom_to_list(Name) =:= Dep],
 
     %% Hack: erlware_commons depends on a .script file to check if it is being built with
     %% rebar2 or rebar3. But since rebar3 isn't built yet it can't get the vsn with get_key.
@@ -103,51 +103,6 @@ fetch_and_compile({Name, ErlFirstFiles}, Deps) ->
 
     compile(Name, ErlFirstFiles).
 
-fetch({pkg, Name, Vsn}, App) ->
-    Dir = filename:join([filename:absname("_build/default/lib/"), App]),
-    case filelib:is_dir(Dir) of
-        false ->
-            CDN = "https://repo.hex.pm/tarballs",
-            Package = binary_to_list(<<Name/binary, "-", Vsn/binary, ".tar">>),
-            Url = join([CDN, Package], "/"),
-            case request(Url) of
-                {ok, Binary} ->
-                    {ok, Contents} = extract(Binary),
-                    ok = erl_tar:extract({binary, Contents}, [{cwd, Dir}, compressed]);
-                {error, {Reason, _}} ->
-                    ReasonText = re:replace(atom_to_list(Reason), "_", " ", [global,{return,list}]),
-                    io:format("Error: Unable to fetch package ~s ~s: ~s~n", [Name, Vsn, ReasonText])
-            end;
-        true ->
-            io:format("Dependency ~s already exists~n", [Name])
-    end.
-
-extract(Binary) ->
-    {ok, Files} = erl_tar:extract({binary, Binary}, [memory]),
-    {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files),
-    {ok, Contents}.
-
-request(Url) ->
-    case os:getenv("REBAR_OFFLINE") of
-        "1" ->
-            {error, {offline, Url}};
-        _ ->
-            request_online(Url)
-    end.
-
-request_online(Url) ->
-    HttpOptions = [{relaxed, true} | get_proxy_auth()],
-
-    case httpc:request(get, {Url, []},
-                       HttpOptions,
-                       [{body_format, binary}],
-                       rebar) of
-        {ok, {{_Version, 200, _Reason}, _Headers, Body}} ->
-            {ok, Body};
-        Error ->
-            Error
-    end.
-
 get_rebar_config() ->
     {ok, [[Home]]} = init:get_argument(home),
     ConfDir = filename:join(Home, ".config/rebar3"),
@@ -637,19 +592,8 @@ reset_env() ->
     application:load(rebar).
 
 get_deps() ->
-    case file:consult("rebar.lock") of
-        {ok, [[]]} ->
-            %% Something went wrong in a previous build, lock file shouldn't be empty
-            io:format("Empty list in lock file, deleting rebar.lock~n"),
-            ok = file:delete("rebar.lock"),
-            {ok, Config} = file:consult("apps/rebar/rebar.config"),
-            proplists:get_value(deps, Config);
-        {ok, [Deps]} ->
-            [{binary_to_atom(Name, utf8), "", Source} || {Name, Source, _Level} <- Deps];
-        _ ->
-            {ok, Config} = file:consult("apps/rebar/rebar.config"),
-            proplists:get_value(deps, Config)
-    end.
+    {ok, Deps} = file:list_dir("vendor"),
+    [{Dep, filename:join("vendor", Dep)} || Dep <- Deps].
 
 format_errors(Source, Errors) ->
     format_errors(Source, "", Errors).
@@ -741,11 +685,11 @@ set_proxy_auth(UserInfo) ->
     %% password may contain url encoded characters, need to decode them first
     put(proxy_auth, [{proxy_auth, {Username, rebar_uri_percent_decode(Password)}}]).
 
-get_proxy_auth() ->
-    case get(proxy_auth) of
-        undefined -> [];
-        ProxyAuth -> ProxyAuth
-    end.
+%get_proxy_auth() ->
+%    case get(proxy_auth) of
+%        undefined -> [];
+%        ProxyAuth -> ProxyAuth
+%    end.
 
 
 %% string:join/2 copy; string:join/2 is getting obsoleted

rebar.config

@@ -1,7 +1,13 @@
 %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
 %% ex: ts=4 sw=4 ft=erlang et
 
-%% The rest of the config is in apps/rebar/
+%% Vendoring deps
+{project_app_dirs, ["apps/*","lib/*",".","vendor/*"]}.
+{project_plugin_dirs, ["plugins/*","vendor_plugins/*"]}.
+
+%% Duplicated from apps/rebar3:
+%% - we want people who rely on rebar3 as a dependency to still be able
+%%   to fetch it with git_subdir and have it work
 {escript_main_app, rebar}.
 {escript_name, rebar3}.
 {escript_wrappers_windows, ["cmd", "powershell"]}.
@@ -12,8 +18,50 @@
 {escript_incl_priv, [{relx, "templates/*"},
                      {rebar, "templates/*"}]}.
 
-{ profiles, [{systest, [
-                {erl_opts, [debug_info, nowarn_export_all]},
-                {ct_opts, [{dir, "systest"}]}
-            ]}
+{profiles, [
+    %% Only works at the top-level
+    {systest, [
+        {erl_opts, [debug_info, nowarn_export_all]},
+        {ct_opts, [{dir, "systest"}]}
+    ]},
+    %% Don't check these vendored deps
+    {dialyzer, [
+        {erl_opts, [debug_info, nowarn_export_all]},
+        %% Ignore deps known to generate warnings
+        {dialyzer, [{exclude_apps, [cth_readable, erlware_commons, relx]}]}
+    ]},
+    %% Duplicated from apps/rebar3:
+    %% - we don't want the test profile applied to our vendored deps.
+    %% - we want people who rely on rebar3 as a dependency to still be able
+    %%   to fetch it with git_subdir and have it work
+    {test, [
+        {deps, [{meck, "0.8.13"}]},
+        {erl_opts, [debug_info, nowarn_export_all]}
+    ]},
+    {prod, [
+        {erl_opts, [no_debug_info]},
+        {overrides, [
+            {override, erlware_commons, [
+                {erl_opts, [{platform_define, "^[0-9]+", namespaced_types},
+                            {platform_define, "^R1[4|5]", deprecated_crypto},
+                            {platform_define, "^((1[8|9])|2)", rand_module},
+                            {platform_define, "^2", unicode_str},
+                            {platform_define, "^(R|1|20)", fun_stacktrace},
+                            no_debug_info,
+                            warnings_as_errors]},
+                {deps, []}, {plugins, []}]},
+            {add, ssl_verify_hostname, [{erl_opts, [no_debug_info]}]},
+            {add, certifi, [{erl_opts, [no_debug_info]}]},
+            {add, cf, [{erl_opts, [no_debug_info]}]},
+            {add, cth_readable, [{erl_opts, [no_debug_info]}]},
+            {add, eunit_formatters, [{erl_opts, [no_debug_info]}]},
+            {override, bbmustache, [
+                {erl_opts, [no_debug_info, {platform_define, "^[0-9]+", namespaced_types}]},
+                {deps, []}, {plugins, []}]},
+            {add, getopt, [{erl_opts, [no_debug_info]}]},
+            {add, providers, [{erl_opts, [no_debug_info]}]},
+            {add, relx, [{erl_opts, [no_debug_info]}]}]}
+    ]}
 ]}.
+
+%% The rest of the config is in apps/rebar/

rebar.lock

@@ -1,35 +1 @@
-{"1.2.0",
-[{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.12.2">>},0},
- {<<"certifi">>,{pkg,<<"certifi">>,<<"2.9.0">>},0},
- {<<"cf">>,{pkg,<<"cf">>,<<"0.3.1">>},0},
- {<<"cth_readable">>,{pkg,<<"cth_readable">>,<<"1.5.1">>},0},
- {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.5.0">>},0},
- {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.5.0">>},0},
- {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},0},
- {<<"providers">>,{pkg,<<"providers">>,<<"1.9.0">>},0},
- {<<"relx">>,{pkg,<<"relx">>,<<"4.7.0">>},0},
- {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.6">>},0}]}.
-[
-{pkg_hash,[
- {<<"bbmustache">>, <<"0CABDCE0DB9FE6D3318131174B9F2B351328A4C0AFBEB3E6E99BB0E02E9B621D">>},
- {<<"certifi">>, <<"6F2A475689DD47F19FB74334859D460A2DC4E3252A3324BD2111B8F0429E7E21">>},
- {<<"cf">>, <<"5CB902239476E141EA70A740340233782D363A31EEA8AD37049561542E6CD641">>},
- {<<"cth_readable">>, <<"F511EFCFDE04A48B014A9197FFF1B4C4860E4E35CDB8E2F3AE3C4178E20299B1">>},
- {<<"erlware_commons">>, <<"918C56D8FB3BE52AF0DF138ED6E0755E764AD4467CD7D025761F7D0A17D3DEC1">>},
- {<<"eunit_formatters">>, <<"6A9133943D36A465D804C1C5B6E6839030434B8879C5600D7DDB5B3BAD4CCB59">>},
- {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>},
- {<<"providers">>, <<"46F6645B0C677B1029E02B013BFD69092A2232854DAF359F2378FA42AC0BEC0D">>},
- {<<"relx">>, <<"CF8F3ECA5FAF0D25746AFAA4042DA4B5061E2CFFC9C190632C6E4F2C20A47573">>},
- {<<"ssl_verify_fun">>, <<"CF344F5692C82D2CD7554F5EC8FD961548D4FD09E7D22F5B62482E5AEAEBD4B0">>}]},
-{pkg_hash_ext,[
- {<<"bbmustache">>, <<"688B33A4D5CC2D51F575ADF0B3683FC40A38314A2F150906EDCFC77F5B577B3B">>},
- {<<"certifi">>, <<"266DA46BDB06D6C6D35FDE799BCB28D36D985D424AD7C08B5BB48F5B5CDD4641">>},
- {<<"cf">>, <<"315E8D447D3A4B02BCDBFA397AD03BBB988A6E0AA6F44D3ADD0F4E3C3BF97672">>},
- {<<"cth_readable">>, <<"686541A22EFE6CA5A41A047B39516C2DD28FB3CADE5F24A2F19145B3967F9D80">>},
- {<<"erlware_commons">>, <<"3E7C6FB2BA4C29B0DD5DFE9D031B66449E2088ECEC1A81465BD9FDE05ED7D0DB">>},
- {<<"eunit_formatters">>, <<"D6C8BA213424944E6E05BBC097C32001CDD0ABE3925D02454F229B20D68763C9">>},
- {<<"getopt">>, <<"53E1AB83B9CEB65C9672D3E7A35B8092E9BDC9B3EE80721471A161C10C59959C">>},
- {<<"providers">>, <<"D287E874406A1505608642B0A3DB5B68D6ADA3F2AB001AEC87E7F4D7C79FC017">>},
- {<<"relx">>, <<"A2ABFAFA70BCE3A0B98A6E5140AC98429B9980F835FFB47357F0645422FC2FA3">>},
- {<<"ssl_verify_fun">>, <<"BDB0D2471F453C88FF3908E7686F86F9BE327D065CC1EC16FA4540197EA04680">>}]}
-].
+[].

vendor/bbmustache/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Hinagiku Soranoba
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+