diff --git a/Cargo.lock b/Cargo.lock index 775f634162ba2..09120d5acdbc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.17.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "ahash" version = "0.8.11" @@ -940,17 +946,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.66" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.8.0", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -2461,7 +2467,7 @@ checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" dependencies = [ "crc32fast", "libz-sys", - "miniz_oxide", + "miniz_oxide 0.5.1", ] [[package]] @@ -2660,9 +2666,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.26.1" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -3697,9 +3703,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.153" +version = "0.2.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" [[package]] name = "libloading" @@ -3963,6 +3969,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "0.8.11" @@ -5611,6 +5626,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "backtrace", "bytes", "chrono", "clap", @@ -7243,9 +7259,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.29.0" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] @@ -8655,9 +8671,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.16" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" diff --git a/clippy.toml b/clippy.toml index f9a001298bca7..c768a36390d26 100644 --- a/clippy.toml +++ b/clippy.toml @@ -6,10 +6,8 @@ await-holding-invalid-types = [ disallowed-methods = [ { path = "std::panic::catch_unwind", reason = "use `mz_ore::panic::catch_unwind` instead" }, { path = "futures::FutureExt::catch_unwind", reason = "use `mz_ore::future::FutureExt::catch_unwind` instead" }, - { path = "futures_executor::block_on", reason = "use `tokio::runtime::Handle::block_on` instead" }, { path = "futures::executor::block_on", reason = "use `tokio::runtime::Handle::block_on` instead" }, - { path = "tokio::spawn", reason = "use the spawn wrappers in `mz_ore::task` instead" }, { path = "tokio::task::spawn", reason = "use the spawn wrappers in `mz_ore::task` instead" }, { path = "tokio::task::spawn_blocking", reason = "use the spawn wrappers in `mz_ore::task` instead" }, @@ -17,11 +15,9 @@ disallowed-methods = [ { path = "tokio::runtime::Handle::spawn_blocking", reason = "use the spawn wrappers in `mz_ore::task` instead" }, { path = "tokio::runtime::Runtime::spawn", reason = "use the spawn wrappers in `mz_ore::task` instead" }, { path = "tokio::runtime::Runtime::spawn_blocking", reason = "use the spawn wrappers in `mz_ore::task` instead" }, - # Note these wrappers aren't implemented yet, as we haven't needed them. { path = "tokio::task::spawn_local", reason = "use the spawn wrappers in `mz_ore::task` instead" }, { path = "tokio::task::LocalSet::spawn_local", reason = "use the spawn wrappers in `mz_ore::task` instead" }, - # Note that `spawn_blocking` and the local varieties are not yet implemented, as they haven't been needed yet. { path = "tokio::task::JoinSet::spawn", reason = "use the spawn wrappers in `mz_ore::task` instead" }, { path = "tokio::task::JoinSet::spawn_on", reason = "use the spawn wrappers in `mz_ore::task` instead" }, @@ -29,7 +25,6 @@ disallowed-methods = [ { path = "tokio::task::JoinSet::spawn_blocking_on", reason = "use the spawn wrappers in `mz_ore::task` instead" }, { path = "tokio::task::JoinSet::spawn_local", reason = "use the spawn wrappers in `mz_ore::task` instead" }, { path = "tokio::task::JoinSet::spawn_local", reason = "use the spawn wrappers in `mz_ore::task` instead" }, - # These are banned because we want to ensure people don't forget to use .name(...), so we require the use of `mz_ore` { path = "tokio::task::Builder::spawn", reason = "use the spawn wrappers in `mz_ore::task` instead" }, { path = "tokio::task::Builder::spawn_on", reason = "use the spawn wrappers in `mz_ore::task` instead" }, @@ -41,15 +36,12 @@ disallowed-methods = [ { path = "tokio::task::join_set::Builder::spawn_on", reason = "use the spawn wrappers in `mz_ore::task` instead" }, { path = "tokio::task::join_set::Builder::spawn_local", reason = "use the spawn wrappers in `mz_ore::task` instead" }, { path = "tokio::task::join_set::Builder::spawn_local", reason = "use the spawn wrappers in `mz_ore::task` instead" }, - { path = "rdkafka::config::ClientConfig::new", reason = "use the `client::create_new_client_config` wrapper in `kafka_util` instead" }, - { path = "aws_config::defaults", reason = "use the `mz_aws_config::defaults` function instead" }, { path = "aws_config::load_defaults", reason = "use the `mz_aws_config::defaults` function instead" }, { path = "aws_config::from_env", reason = "use the `mz_aws_config::defaults` function instead" }, { path = "aws_config::load_from_env", reason = "use the `mz_aws_config::defaults` function instead" }, { path = "aws_sdk_s3::Client::new", reason = "use the `mz_aws_util::s3::new_client` function instead" }, - # Prevent access to Differential APIs that want to use the default trace or use a default name, or where we offer # our own wrapper { path = "differential_dataflow::Collection::consolidate", reason = "use the `mz_timely_util::operator::CollectionExt::consolidate_named` function instead" }, @@ -77,7 +69,6 @@ disallowed-methods = [ { path = "differential_dataflow::operators::reduce::join::Join::antijoin", reason = "use the `differential_dataflow::operators::join::Join::join_core` function instead" }, { path = "differential_dataflow::operators::reduce::join::Join::join_map", reason = "use the `differential_dataflow::operators::join::Join::join_core` function instead" }, { path = "differential_dataflow::operators::reduce::join::Join::semijoin", reason = "use the `differential_dataflow::operators::join::Join::join_core` function instead" }, - # Prevent access to Timely APIs with untested semantics. { path = "timely::worker::Worker::drop_dataflow", reason = "Might break logging dataflows, check with #team-compute." }, ] diff --git a/src/ore/Cargo.toml b/src/ore/Cargo.toml index ec74b4ff7e527..974a28856c70f 100644 --- a/src/ore/Cargo.toml +++ b/src/ore/Cargo.toml @@ -21,8 +21,9 @@ anyhow = { version = "1.0.66", optional = true } async-trait = { version = "0.1.68", optional = true } bytes = { version = "1.3.0", optional = true } chrono = { version = "0.4.35", default-features = false, features = [ - "std", + "std", ], optional = true } +backtrace = "0.3.74" clap = { version = "3.2.24", features = ["env"], optional = true } compact_bytes = { version = "0.1.2", optional = true } ctor = { version = "0.1.26", optional = true } @@ -43,7 +44,7 @@ paste = "1.0.11" pin-project = "1.0.12" prometheus = { version = "0.13.3", default-features = false, optional = true } proptest = { version = "1.0.0", default-features = false, features = [ - "std", + "std", ], optional = true } rand = { version = "0.8.5", optional = true } smallvec = { version = "1.10.0", optional = true } @@ -51,11 +52,11 @@ stacker = { version = "0.1.15", optional = true } sentry = { version = "0.29.1", optional = true, features = ["debug-images"] } serde = { version = "1.0.152", features = ["derive"] } tokio = { version = "1.38.0", features = [ - "io-util", - "net", - "rt-multi-thread", - "sync", - "time", + "io-util", + "net", + "rt-multi-thread", + "sync", + "time", ], optional = true } tokio-openssl = { version = "0.6.3", optional = true } thiserror = "1.0.37" @@ -67,10 +68,10 @@ tracing-capture = { version = "0.1.0", optional = true } # Note that this feature is distinct from `tracing`'s `log` feature, which has `tracing` macros emit `log` records if # there is no global `tracing` subscriber. tracing-subscriber = { version = "0.3.16", default-features = false, features = [ - "env-filter", - "fmt", - "json", - "tracing-log", + "env-filter", + "fmt", + "json", + "tracing-log", ], optional = true } uuid = { version = "1.7.0", optional = true } url = { version = "2.3.1", features = ["serde"] } @@ -88,7 +89,7 @@ hyper-util = { version = "0.1.6", optional = true } opentelemetry = { version = "0.24.0", features = ["trace"], optional = true } opentelemetry-otlp = { version = "0.17.0", optional = true } opentelemetry_sdk = { version = "0.24.1", features = [ - "rt-tokio", + "rt-tokio", ], optional = true } console-subscriber = { version = "0.4.0", optional = true } sentry-tracing = { version = "0.29.1", optional = true } @@ -108,37 +109,37 @@ tracing-subscriber = "0.3.16" [features] default = ["tokio-console", "workspace-hack", "mz-ore-proc/workspace-hack"] async = [ - "async-trait", - "futures", - "metrics", - "openssl", - "tokio-openssl", - "tokio", - "tracing", + "async-trait", + "futures", + "metrics", + "openssl", + "tokio-openssl", + "tokio", + "tracing", ] bytes_ = ["bytes", "compact_bytes", "smallvec", "smallvec/const_generics", "region", "tracing_"] network = ["async", "bytes", "smallvec", "tonic", "tracing"] process = ["libc"] region = ["lgalloc"] tracing_ = [ - "anyhow", - "tracing", - "tracing-subscriber", - "tracing-subscriber/ansi", - "tracing-opentelemetry", - "tokio-native-tls", - "native-tls", - "http", - "hyper-tls", - "hyper-util", - "metrics", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry_sdk", - "tonic", - "sentry", - "sentry-tracing", - "yansi", + "anyhow", + "tracing", + "tracing-subscriber", + "tracing-subscriber/ansi", + "tracing-opentelemetry", + "tokio-native-tls", + "native-tls", + "http", + "hyper-tls", + "hyper-util", + "metrics", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", + "tonic", + "sentry", + "sentry-tracing", + "yansi", ] tokio-console = ["console-subscriber", "tokio", "tokio/tracing", "network"] cli = ["clap"] diff --git a/src/ore/src/stack.rs b/src/ore/src/stack.rs index b40450608c2c9..c9287a0235626 100644 --- a/src/ore/src/stack.rs +++ b/src/ore/src/stack.rs @@ -97,6 +97,44 @@ pub fn maybe_grow(f: F) -> R where F: FnOnce() -> R, { + if stacker::remaining_stack() + .map(|r| r < STACK_RED_ZONE) + .unwrap_or(false) + { + #[inline(never)] + fn trace() { + let mut limit = None; + let mut last_sp = None; + let mut depth = 0; + backtrace::trace(|frame| { + if limit.is_none() { + limit = Some(frame.sp().wrapping_add(STACK_SIZE)); + eprintln!("Stack limit: {:?}", limit.unwrap()); + } + + let frame_size = + unsafe { frame.sp().byte_offset_from(last_sp.unwrap_or(frame.sp())) }; + + // Resolve this instruction pointer to a symbol name + backtrace::resolve_frame(frame, |symbol| { + eprintln!( + "{depth}\t{:?}\t{frame_size}\t{:?}, at {}:{}", + frame.sp(), + symbol.name(), + symbol.filename().and_then(|s| s.to_str()).unwrap_or("???"), + symbol.lineno().unwrap_or(0), + ); + }); + + last_sp = Some(frame.sp()); + + depth += 1; + + frame.sp() <= limit.unwrap() + }); + trace(); + } + } stacker::maybe_grow(STACK_RED_ZONE, STACK_SIZE, f) } diff --git a/src/sql/src/func.rs b/src/sql/src/func.rs index b4437836fb88f..10827929477fa 100644 --- a/src/sql/src/func.rs +++ b/src/sql/src/func.rs @@ -1649,14 +1649,19 @@ macro_rules! builtins { let mut builtins = BTreeMap::new(); $( - let impls = vec![$(impl_def!($params, $op, $return_type, $oid)),+]; - let func = Func::$ty(impls); - let expect_set_return = matches!(&func, Func::Table(_)); - for imp in func.func_impls() { - assert_eq!(imp.return_is_set, expect_set_return, "wrong set return value for func with oid {}", imp.oid); + { + fn insert_fn(builtins: &mut BTreeMap<&'static str, Func>) { + let impls = vec![$(impl_def!($params, $op, $return_type, $oid)),+]; + let func = Func::$ty(impls); + let expect_set_return = matches!(&func, Func::Table(_)); + for imp in func.func_impls() { + assert_eq!(imp.return_is_set, expect_set_return, "wrong set return value for func with oid {}", imp.oid); + } + let old = builtins.insert($name, func); + mz_ore::assert_none!(old, "duplicate entry in builtins list"); + } + insert_fn(&mut builtins); } - let old = builtins.insert($name, func); - mz_ore::assert_none!(old, "duplicate entry in builtins list"); )+ builtins }};