additional transport metrics

Summary: Adds latency metrics and counts broken down by completion status.

Differential Revision: D61232187

fbshipit-source-id: f7db5787a65e1ebd6f6f28d57771b4edd35f638e

Author: Madan Jampani

include/wa_raft.hrl

@@ -72,6 +72,7 @@
 -define(RAFT_COUNT(Metric), ?RAFT_METRICS_MODULE:count(Metric)).
 -define(RAFT_COUNTV(Metric, Value), ?RAFT_METRICS_MODULE:countv(Metric, Value)).
 -define(RAFT_GATHER(Metric, Value), ?RAFT_METRICS_MODULE:gather(Metric, Value)).
+-define(RAFT_GATHER_LATENCY(Metric, Value), ?RAFT_METRICS_MODULE:gather_latency(Metric, Value)).
 
 %%-------------------------------------------------------------------
 %% Global Configuration

src/wa_raft_metrics.erl

@@ -18,7 +18,8 @@
 -export([
     count/1,
     countv/2,
-    gather/2
+    gather/2,
+    gather_latency/2
 ]).
 
 %% Public Types
@@ -39,6 +40,8 @@
 -callback countv(metric(), value()) -> ok.
 %% Report the measured value of an occurence of some metric.
 -callback gather(metric(), value()) -> ok.
+%% Report the measured latency of an occurence of some metric.
+-callback gather_latency(metric(), value()) -> ok.
 
 %%-------------------------------------------------------------------
 %% Public Types
@@ -72,3 +75,7 @@ countv(_Metric, _Value) ->
 -spec gather(metric(), value()) -> ok.
 gather(_Metric, _Value) ->
     ok.
+
+-spec gather_latency(metric(), value()) -> ok.
+gather_latency(_Metric, _Value) ->
+    ok.

src/wa_raft_transport.erl

@@ -529,10 +529,16 @@ handle_call(Request, _From, #state{} = State) ->
 -spec handle_cast(Request, State :: #state{}) -> {noreply, NewState :: #state{}}
     when Request :: {complete, ID :: transport_id(), FileID :: file_id(), Status :: term(), Pid :: pid()}.
 handle_cast({complete, ID, FileID, Status, Pid}, #state{counters = Counters} = State) ->
-    ?RAFT_COUNT('raft.transport.file.complete'),
     NowMillis = erlang:system_time(millisecond),
+    ?RAFT_COUNT({'raft.transport.file.send', Status}),
     Result0 = update_file_info(ID, FileID,
         fun (Info) ->
+            case Info of
+                #{start_ts := StartMillis} ->
+                    ?RAFT_GATHER_LATENCY({'raft.transport.file.send', Status, latency_ms}, NowMillis - StartMillis);
+                _ ->
+                    ok
+            end,
             case Status of
                 ok -> Info#{status => completed, end_ts => NowMillis};
                 _  -> Info#{status => failed, end_ts => NowMillis, error => Status}
@@ -824,8 +830,9 @@ maybe_notify_complete(ID, _Info, #state{}) ->
         [ID], #{domain => [whatsapp, wa_raft]}).
 
 -spec maybe_notify(transport_id(), transport_info()) -> transport_info().
-maybe_notify(ID, #{status := Status, notify := Notify} = Info) when Status =/= requested andalso Status =/= running ->
-    ?RAFT_COUNT('raft.transport.complete'),
+maybe_notify(ID, #{status := Status, notify := Notify, start_ts := Start, end_ts := End} = Info) when Status =/= requested andalso Status =/= running ->
+    ?RAFT_COUNT({'raft.transport', Status}),
+    ?RAFT_GATHER_LATENCY({'raft.transport', Status, latency_ms}, End - Start),
     gen_server:reply(Notify, {ok, ID}),
     maps:remove(notify, Info);
 maybe_notify(_ID, Info) ->

src/wa_raft_transport_worker.erl

@@ -88,7 +88,7 @@ handle_call(Request, From, #state{number = Number} = State) ->
 -spec handle_cast(Request, State :: state()) -> {noreply, NewState :: state(), Timeout :: timeout()}
     when Request :: {send, wa_raft_transport:transport_id(), wa_raft_transport:file_id()}.
 handle_cast({send, ID, FileID}, #state{table = Table} = State) ->
-    ?RAFT_COUNT('raft.transport.send'),
+    ?RAFT_COUNT('raft.transport.file.send'),
     wa_raft_transport:update_file_info(ID, FileID,
         fun (Info) -> Info#{status => sending, start_ts => erlang:system_time(millisecond)} end),
     true = ets:insert_new(Table, {make_ref(), ID, FileID}),