From d7b0f5ea59b97b588851d1eb836c809a641f7fa2 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 28 Feb 2019 15:08:54 +0000 Subject: [PATCH 1/4] Update rebar.config Switch to develop-2.9 reference --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 2e0eb74c..50fca8d1 100644 --- a/rebar.config +++ b/rebar.config @@ -11,7 +11,7 @@ {deps, [ {kvc, ".*", {git, "git://github.com/etrepum/kvc.git", {tag, "v1.5.0"}}}, - {riak_kv, ".*", {git, "git://github.com/basho/riak_kv.git", {tag, "2.1.7-226"}}}, + {riak_kv, ".*", {git, "git://github.com/basho/riak_kv.git", {branch, "develop-2.9"}}}, {ibrowse, "4.0.2", {git, "git://github.com/cmullaparthi/ibrowse.git", {tag, "v4.0.2"}}}, {fuse, "2.1.0", {git, "https://github.com/jlouis/fuse.git", {tag, "v2.1.0"}}}, {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {tag, "2.5.5"}}} From a0f4b372c1611f41bf3284a005665c823c653902 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 28 Feb 2019 16:56:30 +0000 Subject: [PATCH 2/4] Update rebar.config Point riak-erlang-client at dependency compatible with develop-2.9 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 50fca8d1..590a882c 100644 --- a/rebar.config +++ b/rebar.config @@ -14,7 +14,7 @@ {riak_kv, ".*", {git, "git://github.com/basho/riak_kv.git", {branch, "develop-2.9"}}}, {ibrowse, "4.0.2", {git, "git://github.com/cmullaparthi/ibrowse.git", {tag, "v4.0.2"}}}, {fuse, "2.1.0", {git, "https://github.com/jlouis/fuse.git", {tag, "v2.1.0"}}}, - {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {tag, "2.5.5"}}} + {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {branch, "develop-2.2"}}} ]}. {pre_hooks, [{compile, "./tools/grab-solr.sh"}]}. From ba528251b2340b26620222421707338660435e6e Mon Sep 17 00:00:00 2001 From: Antonio Nikishaev Date: Sun, 3 Mar 2019 09:48:12 +0400 Subject: [PATCH 3/4] [sec] yz xml extractor: prevent XXE attack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit XML External Entity attack 1. if HTTP API is exposed: - read any file on the system — via /search/extract, the error message leaks file content; - send HTTP «GET /» request to any host — by PUT/POSTing text/xml document, or via /search/extract. This is also likely riak DoS if the host is attacker-controlled. 2. if PB API is exposed - send HTTP «GET /» request to any host — by PUT/POST, see above. Example request: ]> &xxe1; --- src/yz_xml_extractor.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/yz_xml_extractor.erl b/src/yz_xml_extractor.erl index 6f3f5a5b..fe17af60 100644 --- a/src/yz_xml_extractor.erl +++ b/src/yz_xml_extractor.erl @@ -56,7 +56,12 @@ extract(Value, Opts) -> extract_fields(Value, #state{field_separator=Sep}). extract_fields(Data, State) -> - Options = [{event_fun, fun sax_cb/3}, {event_state, State}], + Options = [ + {file_type, normal}, + skip_external_dtd, + {event_fun, fun sax_cb/3}, + {event_state, State} + ], case xmerl_sax_parser:stream(Data, Options) of {ok, State2, _Rest} -> State2#state.fields; @@ -88,6 +93,9 @@ sax_cb({characters, Value}, _Location, S) -> Field = {Name, unicode:characters_to_binary(Value)}, S#state{fields = [Field|S#state.fields]}; +sax_cb({externalEntityDecl,_,_,_}, _, _State) -> + throw(external_entity_disarmed); + sax_cb(_Event, _Location, State) -> State. From 5ebab6423ae3cfa39cf6796e220904e06a20d71e Mon Sep 17 00:00:00 2001 From: Antonio Nikishaev Date: Sun, 3 Mar 2019 10:04:01 +0400 Subject: [PATCH 4/4] =?UTF-8?q?[sec]=20http=20search:=20get=20rid=20of=20?= =?UTF-8?q?=E2=80=98yz-fprof=E2=80=99=20header=20handling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It doesn't check user-provided path in any way. This allows overriding any file on the system with riak permissions. --- src/yz_wm_search.erl | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/yz_wm_search.erl b/src/yz_wm_search.erl index 8a9dff61..f49236c7 100644 --- a/src/yz_wm_search.erl +++ b/src/yz_wm_search.erl @@ -23,8 +23,6 @@ -include("yokozuna.hrl"). -include_lib("webmachine/include/webmachine.hrl"). --define(YZ_HEAD_FPROF, "yz-fprof"). - -record(ctx, {security %% security context }). @@ -121,8 +119,6 @@ search(Req, S) -> search(Req, S, Params). search(Req, S, Params) -> - {FProf, FProfFile} = check_for_fprof(Req), - ?IF(FProf, fprof:trace(start, FProfFile)), T1 = os:timestamp(), Index = list_to_binary(wrq:path_info(index, Req)), try @@ -155,8 +151,6 @@ search(Req, S, Params) -> ErrReq2 = wrq:set_resp_header("Content-Type", "text/plain", ErrReq), {{halt, Code}, ErrReq2, S} - after - ?IF(FProf, fprof_analyse(FProfFile)) end. %% @doc Solr returns as chunked but not going to return as chunked from @@ -164,17 +158,6 @@ search(Req, S, Params) -> scrub_headers(RespHeaders) when is_list(RespHeaders) -> lists:keydelete("Transfer-Encoding", 1, RespHeaders). -check_for_fprof(Req) -> - case wrq:get_req_header(?YZ_HEAD_FPROF, Req) of - undefined -> {false, none}; - File -> {true, File} - end. - -fprof_analyse(FileName) -> - fprof:trace(stop), - fprof:profile(file, FileName), - fprof:analyse([{dest, FileName ++ ".analysis"}, {cols, 120}]). - -spec resource_exists(term(), term()) -> {boolean(), term(), term()}. resource_exists(RD, Context) -> IndexName = list_to_binary(wrq:path_info(index, RD)),