Stop custom multi-buy backoff from dropping packets for minutes

include/hpr_metrics.hrl

@@ -16,6 +16,7 @@
 -define(METRICS_BROKEN_ROUTES_GAUGE, "hpr_broken_routes_gauge").
 -define(METRICS_PACKET_REPORT_HISTOGRAM, "hpr_packet_report_histogram").
 -define(METRICS_MULTI_BUY_GET_HISTOGRAM, "hpr_multi_buy_get_histogram").
+-define(METRICS_MULTI_BUY_DECISION_COUNTER, "hpr_multi_buy_decision_counter").
 -define(METRICS_FIND_ROUTES_HISTOGRAM, "hpr_find_routes_histogram").
 -define(METRICS_VM_ETS_MEMORY, "hpr_vm_ets_memory").
 -define(METRICS_VM_PROC_Q, "hpr_vm_process_queue").
@@ -41,6 +42,8 @@
     {?METRICS_PACKET_REPORT_HISTOGRAM, prometheus_histogram, [status], "Packet Reports"},
     {?METRICS_MULTI_BUY_GET_HISTOGRAM, prometheus_histogram, [channel, status],
         "Multi Buy Service Get"},
+    {?METRICS_MULTI_BUY_DECISION_COUNTER, prometheus_counter, [channel, decision],
+        "Multi Buy decision per packet"},
     {?METRICS_FIND_ROUTES_HISTOGRAM, prometheus_histogram, [], "Find Routes"},
     {?METRICS_VM_ETS_MEMORY, prometheus_gauge, [name], "HPR ets memory"},
     {?METRICS_VM_PROC_Q, prometheus_gauge, [name], "Process queue"},

rebar.config

@@ -21,6 +21,11 @@
     {ecron, {git, "https://github.com/zhongwencool/ecron.git", {tag, "v1.1.1"}}},
     {erlang_lorawan, {git, "https://github.com/helium/erlang-lorawan.git", {branch, "master"}}},
     {libp2p_crypto, {git, "https://github.com/helium/libp2p_crypto.git", {tag, "v1.5.2"}}},
+    %% Promoted from a transitive dep of libp2p_crypto, which asks for master
+    %% but was locked to a commit whose NIF unload hook has the wrong
+    %% signature. gcc 14 rejects that outright, so anything building the C
+    %% sources from scratch fails.
+    {enacl, {git, "https://github.com/helium/enacl", {branch, "master"}}},
     {iso8601, ".*", {git, "https://github.com/erlsci/iso8601.git", {tag, "1.3.1"}}},
     {xxhash, {git, "https://github.com/pierreis/erlang-xxhash", {branch, "master"}}},
     {erl_angry_purple_tiger,

rebar.lock

@@ -36,8 +36,8 @@
  {<<"elli">>,{pkg,<<"elli">>,<<"3.3.0">>},0},
  {<<"enacl">>,
   {git,"https://github.com/helium/enacl",
-       {ref,"efb74b1af1df7f46f6ade488f16ec365af3ce47f"}},
-  1},
+       {ref,"8015cd798e81cf40cb3bb6bd36611239f9a713d9"}},
+  0},
  {<<"erl_angry_purple_tiger">>,
   {git,"https://github.com/helium/erl_angry_purple_tiger.git",
        {ref,"c5476b6639314a75a99400c9dfa7603b24a6d18a"}},

src/cli/hpr_cli_config.erl

@@ -941,11 +941,12 @@ mk_top_level_route_info(RouteETS) ->
                 Channel = [RouteID, Protocol, Host, Port],
                 BackoffInfo =
                     case ets:lookup(hpr_multi_buy_backoff_ets, Channel) of
-                        [{_, Until, _}] ->
+                        [{_, Until, _, ProbeUntil}] ->
                             Now = erlang:system_time(millisecond),
-                            case Until > Now of
+                            Deadline = erlang:max(Until, ProbeUntil),
+                            case Deadline > Now of
                                 true ->
-                                    RemainingSecs = (Until - Now) / 1000,
+                                    RemainingSecs = (Deadline - Now) / 1000,
                                     io_lib:format("active (~.1fs remaining)", [RemainingSecs]);
                                 false ->
                                     "none"

src/grpc/iot_config/hpr_route_stream_worker.erl

@@ -480,16 +480,22 @@ terminate(_Reason, #state{checkpoint_timer = CheckpointTimer, stream = Stream} =
         | {skf, hpr_skf:skf()}
 ) -> ok.
 process_route_stream_res(add, {route, Route}) ->
+    PreviousRoute =
+        case hpr_route_storage:lookup(hpr_route:id(Route)) of
+            {ok, RouteETS} -> hpr_route_ets:route(RouteETS);
+            {error, not_found} -> undefined
+        end,
     ok = hpr_route_storage:insert(Route),
-    ok = hpr_multi_buy:init_channel(Route);
+    ok = hpr_multi_buy:update_channel(PreviousRoute, Route);
 process_route_stream_res(add, {eui_pair, EUIPair}) ->
     hpr_eui_pair_storage:insert(EUIPair);
 process_route_stream_res(add, {devaddr_range, DevAddrRange}) ->
     hpr_devaddr_range_storage:insert(DevAddrRange);
 process_route_stream_res(add, {skf, SKF}) ->
     hpr_skf_storage:insert(SKF);
 process_route_stream_res(remove, {route, Route}) ->
-    hpr_route_storage:delete(Route);
+    ok = hpr_route_storage:delete(Route),
+    ok = hpr_multi_buy:cleanup_channel(Route);
 process_route_stream_res(remove, {eui_pair, EUIPair}) ->
     hpr_eui_pair_storage:delete(EUIPair);
 process_route_stream_res(remove, {devaddr_range, DevAddrRange}) ->