Skip to content

Commit aa63bb2

Browse files
antiguruantiguru
committed
Use MzOffsetOptimized in FlatStack storage
Signed-off-by: Moritz Hoffmann <[email protected]>
1 parent fc42987 commit aa63bb2

File tree

5 files changed

+219
-116
lines changed

5 files changed

+219
-116
lines changed

‎Cargo.lock

Lines changed: 8 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -267,20 +267,22 @@ debug = 2
267267
# version of Materialize.
268268
[patch."https://github.com/TimelyDataflow/timely-dataflow"]
269269
# Projects that do not reliably release to crates.io.
270-
timely = { git = "https://github.com/MaterializeInc/timely-dataflow.git" }
271-
timely_bytes = { git = "https://github.com/MaterializeInc/timely-dataflow.git" }
272-
timely_communication = { git = "https://github.com/MaterializeInc/timely-dataflow.git" }
273-
timely_container = { git = "https://github.com/MaterializeInc/timely-dataflow.git" }
274-
timely_logging = { git = "https://github.com/MaterializeInc/timely-dataflow.git" }
270+
timely = { git = "https://github.com/antiguru/timely-dataflow.git", branch = "flatcontainer_storage" }
271+
timely_bytes = { git = "https://github.com/antiguru/timely-dataflow.git", branch = "flatcontainer_storage" }
272+
timely_communication = { git = "https://github.com/antiguru/timely-dataflow.git", branch = "flatcontainer_storage" }
273+
timely_container = { git = "https://github.com/antiguru/timely-dataflow.git", branch = "flatcontainer_storage" }
274+
timely_logging = { git = "https://github.com/antiguru/timely-dataflow.git", branch = "flatcontainer_storage" }
275275
[patch.crates-io]
276276
# Projects that do not reliably release to crates.io.
277-
timely = { git = "https://github.com/MaterializeInc/timely-dataflow.git" }
278-
timely_bytes = { git = "https://github.com/MaterializeInc/timely-dataflow.git" }
279-
timely_communication = { git = "https://github.com/MaterializeInc/timely-dataflow.git" }
280-
timely_container = { git = "https://github.com/MaterializeInc/timely-dataflow.git" }
281-
timely_logging = { git = "https://github.com/MaterializeInc/timely-dataflow.git" }
282-
differential-dataflow = { git = "https://github.com/antiguru/differential-dataflow.git", branch = "consolidate_layout_merger_chunk" }
283-
dogsdogsdogs = { git = "https://github.com/antiguru/differential-dataflow.git", branch = "consolidate_layout_merger_chunk" }
277+
timely = { git = "https://github.com/antiguru/timely-dataflow.git", branch = "flatcontainer_storage" }
278+
timely_bytes = { git = "https://github.com/antiguru/timely-dataflow.git", branch = "flatcontainer_storage" }
279+
timely_communication = { git = "https://github.com/antiguru/timely-dataflow.git", branch = "flatcontainer_storage" }
280+
timely_container = { git = "https://github.com/antiguru/timely-dataflow.git", branch = "flatcontainer_storage" }
281+
timely_logging = { git = "https://github.com/antiguru/timely-dataflow.git", branch = "flatcontainer_storage" }
282+
differential-dataflow = { git = "https://github.com/antiguru/differential-dataflow.git", branch = "region_update" }
283+
dogsdogsdogs = { git = "https://github.com/antiguru/differential-dataflow.git", branch = "region_update" }
284+
285+
flatcontainer = { git = "https://github.com/antiguru/flatcontainer.git" }
284286

285287
# Waiting on https://github.com/sfackler/rust-postgres/pull/752.
286288
postgres = { git = "https://github.com/MaterializeInc/rust-postgres" }

‎src/compute/src/typedefs.rs

Lines changed: 14 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ use differential_dataflow::trace::implementations::merge_batcher_col::Columnatio
1919
use differential_dataflow::trace::implementations::ord_neu::{FlatValSpine, OrdValBatch};
2020
use differential_dataflow::trace::wrappers::enter::TraceEnter;
2121
use differential_dataflow::trace::wrappers::frontier::TraceFrontier;
22-
use mz_ore::flatcontainer::{MzRegionPreference, MzTupleABCRegion, MzTupleABRegion};
22+
use mz_ore::flatcontainer::{
23+
MzOffsetOptimized, MzRegionPreference, MzTupleABCRegion, MzTupleABRegion,
24+
};
2325
use mz_repr::Diff;
2426
use mz_storage_types::errors::DataflowError;
2527
use timely::dataflow::ScopeParent;
@@ -39,7 +41,7 @@ pub(crate) mod spines {
3941
use differential_dataflow::trace::implementations::spine_fueled::Spine;
4042
use differential_dataflow::trace::implementations::{Layout, Update};
4143
use differential_dataflow::trace::rc_blanket_impls::RcBuilder;
42-
use mz_ore::flatcontainer::MzRegion;
44+
use mz_ore::flatcontainer::{MzOffsetOptimized, MzRegion};
4345
use mz_timely_util::containers::stack::StackWrapper;
4446
use timely::container::columnation::{Columnation, TimelyStack};
4547
use timely::container::flatcontainer::FlatStack;
@@ -106,60 +108,6 @@ pub(crate) mod spines {
106108
type Val = VR::Owned;
107109
type Time = TR::Owned;
108110
type Diff = RR::Owned;
109-
type ItemRef<'a> = ((Self::KeyGat<'a>, Self::ValGat<'a>), Self::TimeGat<'a>, Self::DiffGat<'a>)
110-
where
111-
Self: 'a;
112-
type KeyGat<'a> = KR::ReadItem<'a>
113-
where
114-
Self: 'a;
115-
type ValGat<'a> = VR::ReadItem<'a>
116-
where
117-
Self: 'a;
118-
type TimeGat<'a> = TR::ReadItem<'a>
119-
where
120-
Self: 'a;
121-
type DiffGat<'a> = RR::ReadItem<'a>
122-
where
123-
Self: 'a;
124-
125-
fn into_parts<'a>(
126-
((key, val), time, diff): Self::ItemRef<'a>,
127-
) -> (
128-
Self::KeyGat<'a>,
129-
Self::ValGat<'a>,
130-
Self::TimeGat<'a>,
131-
Self::DiffGat<'a>,
132-
) {
133-
(key, val, time, diff)
134-
}
135-
136-
fn reborrow_key<'b, 'a: 'b>(item: Self::KeyGat<'a>) -> Self::KeyGat<'b>
137-
where
138-
Self: 'a,
139-
{
140-
KR::reborrow(item)
141-
}
142-
143-
fn reborrow_val<'b, 'a: 'b>(item: Self::ValGat<'a>) -> Self::ValGat<'b>
144-
where
145-
Self: 'a,
146-
{
147-
VR::reborrow(item)
148-
}
149-
150-
fn reborrow_time<'b, 'a: 'b>(item: Self::TimeGat<'a>) -> Self::TimeGat<'b>
151-
where
152-
Self: 'a,
153-
{
154-
TR::reborrow(item)
155-
}
156-
157-
fn reborrow_diff<'b, 'a: 'b>(item: Self::DiffGat<'a>) -> Self::DiffGat<'b>
158-
where
159-
Self: 'a,
160-
{
161-
RR::reborrow(item)
162-
}
163111
}
164112

165113
/// Layout implementation for [`MzFlatLayout`]. Mostly equivalent to differential's
@@ -182,10 +130,10 @@ pub(crate) mod spines {
182130
for<'a> RR::ReadItem<'a>: Copy + Ord,
183131
{
184132
type Target = Self;
185-
type KeyContainer = FlatStack<KR>;
186-
type ValContainer = FlatStack<VR>;
187-
type TimeContainer = FlatStack<TR>;
188-
type DiffContainer = FlatStack<RR>;
133+
type KeyContainer = FlatStack<KR, MzOffsetOptimized>;
134+
type ValContainer = FlatStack<VR, MzOffsetOptimized>;
135+
type TimeContainer = FlatStack<TR, MzOffsetOptimized>;
136+
type DiffContainer = FlatStack<RR, MzOffsetOptimized>;
189137
type OffsetContainer = OffsetOptimized;
190138
}
191139
}
@@ -234,8 +182,12 @@ pub type KeyValBatcher<K, V, T, D> = MergeBatcher<
234182
>;
235183

236184
pub type FlatKeyValBatch<K, V, T, R> = OrdValBatch<MzFlatLayout<K, V, T, R>>;
237-
pub type FlatKeyValSpine<K, V, T, R, C> =
238-
FlatValSpine<MzFlatLayout<K, V, T, R>, MzTupleABCRegion<MzTupleABRegion<K, V>, T, R>, C>;
185+
pub type FlatKeyValSpine<K, V, T, R, C> = FlatValSpine<
186+
MzFlatLayout<K, V, T, R>,
187+
MzTupleABCRegion<MzTupleABRegion<K, V>, T, R>,
188+
C,
189+
MzOffsetOptimized,
190+
>;
239191
pub type FlatKeyValSpineDefault<K, V, T, R, C> = FlatKeyValSpine<
240192
<K as MzRegionPreference>::Region,
241193
<V as MzRegionPreference>::Region,

0 commit comments

Comments
 (0)