Skip to content

Commit 2f144c6

Browse files
authored
Do not panic on failure to acquire connections to store events (#4132)
# Description During a particularly bad period of volatility we failed to acquire a DB connection on time which lead to a panic <details><summary>Partial Stacktrace</summary> <p> ``` 1770314825856 2026-02-05T18:07:05.856Z 2026-02-05T18:07:03.033Z ERROR observe::tracing: thread 'tokio-runtime-worker' panicked at /src/crates/autopilot/src/infra/persistence/mod.rs:301:54: 1770314825856 2026-02-05T18:07:05.856Z failed to acquire tx: PoolTimedOut 1770314825856 2026-02-05T18:07:05.856Z stack backtrace: 1770314825856 2026-02-05T18:07:05.856Z 0: observe::tracing::tracing_panic_hook 1770314825856 2026-02-05T18:07:05.856Z at ./src/crates/observe/src/tracing.rs:187:21 1770314825856 2026-02-05T18:07:05.856Z 1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call 1770314825856 2026-02-05T18:07:05.856Z at ./rustc/254b59607d4417e9dffbc307138ae5c86280fe4c/library/alloc/src/boxed.rs:2220:9 1770314825856 2026-02-05T18:07:05.856Z 2: observe::panic_hook::install::{{closure}} 1770314825856 2026-02-05T18:07:05.856Z at ./src/crates/observe/src/panic_hook.rs:14:9 1770314825856 2026-02-05T18:07:05.856Z 3: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call 1770314825856 2026-02-05T18:07:05.856Z at ./rustc/254b59607d4417e9dffbc307138ae5c86280fe4c/library/alloc/src/boxed.rs:2220:9 1770314825856 2026-02-05T18:07:05.856Z 4: std::panicking::panic_with_hook 1770314825856 2026-02-05T18:07:05.856Z at ./rustc/254b59607d4417e9dffbc307138ae5c86280fe4c/library/std/src/panicking.rs:833:13 1770314825856 2026-02-05T18:07:05.856Z 5: std::panicking::panic_handler::{{closure}} 1770314825856 2026-02-05T18:07:05.856Z at ./rustc/254b59607d4417e9dffbc307138ae5c86280fe4c/library/std/src/panicking.rs:698:13 1770314825856 2026-02-05T18:07:05.856Z 6: std::sys::backtrace::__rust_end_short_backtrace 1770314825856 2026-02-05T18:07:05.856Z at ./rustc/254b59607d4417e9dffbc307138ae5c86280fe4c/library/std/src/sys/backtrace.rs:176:18 1770314825856 2026-02-05T18:07:05.856Z 7: __rustc::rust_begin_unwind 1770314825856 2026-02-05T18:07:05.856Z at ./rustc/254b59607d4417e9dffbc307138ae5c86280fe4c/library/std/src/panicking.rs:689:5 1770314825856 2026-02-05T18:07:05.856Z 8: core::panicking::panic_fmt 1770314825856 2026-02-05T18:07:05.856Z at ./rustc/254b59607d4417e9dffbc307138ae5c86280fe4c/library/core/src/panicking.rs:80:14 1770314825856 2026-02-05T18:07:05.856Z 9: core::result::unwrap_failed 1770314825856 2026-02-05T18:07:05.856Z at ./rustc/254b59607d4417e9dffbc307138ae5c86280fe4c/library/core/src/result.rs:1867:5 1770314825856 2026-02-05T18:07:05.856Z 10: core::result::Result<T,E>::expect 1770314825856 2026-02-05T18:07:05.856Z at ./rustc/254b59607d4417e9dffbc307138ae5c86280fe4c/library/core/src/result.rs:1185:23 1770314825856 2026-02-05T18:07:05.856Z 11: autopilot::infra::persistence::Persistence::store_order_events::{{closure}} 1770314825856 2026-02-05T18:07:05.856Z at ./src/crates/autopilot/src/infra/persistence/mod.rs:301:54 ``` </p> </details> # Changes - [ ] Remove the panic - [ ] Log on error and continue ## How to test NA
1 parent 950180e commit 2f144c6

File tree

2 files changed

+12
-2
lines changed

2 files changed

+12
-2
lines changed

crates/autopilot/src/infra/persistence/mod.rs

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -298,8 +298,17 @@ impl Persistence {
298298
let order_uids = order_uids.into_iter().collect();
299299
tokio::spawn(
300300
async move {
301-
let mut tx = db.pool.acquire().await.expect("failed to acquire tx");
302-
store_order_events(&mut tx, order_uids, label, Utc::now()).await;
301+
match db.pool.acquire().await {
302+
Ok(mut tx) => {
303+
store_order_events(&mut tx, order_uids, label, Utc::now()).await;
304+
}
305+
Err(err) => {
306+
tracing::error!(
307+
?err,
308+
"failed to acquire a connection to store order events!"
309+
);
310+
}
311+
};
303312
}
304313
.instrument(tracing::Span::current()),
305314
);

crates/database/src/orders.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -964,6 +964,7 @@ pub async fn user_orders_with_quote(
964964
FROM orders o
965965
WHERE o.cancellation_timestamp IS NULL
966966
AND o.true_valid_to >= $1
967+
AND NOT EXISTS (SELECT 1 FROM ethflow_refunds r WHERE r.order_uid = o.uid)
967968
AND NOT EXISTS (SELECT 1 FROM invalidations i WHERE i.order_uid = o.uid)
968969
AND NOT EXISTS (SELECT 1 FROM onchain_order_invalidations oi WHERE oi.uid = o.uid)
969970
AND NOT EXISTS (SELECT 1 FROM onchain_placed_orders op WHERE op.uid = o.uid AND op.placement_error IS NOT NULL)

0 commit comments

Comments
 (0)