From 96079f8d3194e514bebbc8f685bcc60f831eb57f Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Fri, 3 Mar 2023 13:46:55 -0500 Subject: [PATCH 1/5] Owned streams take 2 Signed-off-by: Moritz Hoffmann --- container/src/lib.rs | 8 +- mdbook/src/chapter_2/chapter_2_3.md | 16 +-- mdbook/src/chapter_2/chapter_2_4.md | 4 +- mdbook/src/chapter_4/chapter_4_2.md | 14 +-- mdbook/src/chapter_4/chapter_4_3.md | 2 +- timely/examples/bfs.rs | 4 +- timely/examples/hashjoin.rs | 2 +- timely/examples/loopdemo.rs | 8 +- timely/examples/pagerank.rs | 2 +- timely/examples/pingpong.rs | 2 +- timely/examples/unionfind.rs | 8 +- .../src/dataflow/channels/pushers/buffer.rs | 6 +- timely/src/dataflow/channels/pushers/mod.rs | 6 +- timely/src/dataflow/channels/pushers/owned.rs | 48 ++++++++ timely/src/dataflow/mod.rs | 2 +- .../operators/aggregation/aggregate.rs | 18 ++- .../operators/aggregation/state_machine.rs | 14 ++- timely/src/dataflow/operators/branch.rs | 22 ++-- timely/src/dataflow/operators/broadcast.rs | 10 +- .../operators/core/capture/capture.rs | 16 +-- .../dataflow/operators/core/capture/replay.rs | 12 +- timely/src/dataflow/operators/core/concat.rs | 39 +++--- .../src/dataflow/operators/core/enterleave.rs | 44 +++---- .../src/dataflow/operators/core/exchange.rs | 15 +-- .../src/dataflow/operators/core/feedback.rs | 28 ++--- timely/src/dataflow/operators/core/filter.rs | 10 +- timely/src/dataflow/operators/core/input.rs | 66 +++++++--- timely/src/dataflow/operators/core/inspect.rs | 20 +-- timely/src/dataflow/operators/core/map.rs | 12 +- timely/src/dataflow/operators/core/ok_err.rs | 14 +-- timely/src/dataflow/operators/core/probe.rs | 12 +- timely/src/dataflow/operators/core/rc.rs | 12 +- timely/src/dataflow/operators/core/reclock.rs | 12 +- .../src/dataflow/operators/core/to_stream.rs | 12 +- .../operators/core/unordered_input.rs | 22 ++-- timely/src/dataflow/operators/count.rs | 12 +- timely/src/dataflow/operators/delay.rs | 16 +-- timely/src/dataflow/operators/filter.rs | 10 +- .../src/dataflow/operators/flow_controlled.rs | 4 +- .../dataflow/operators/generic/builder_raw.rs | 18 +-- .../dataflow/operators/generic/builder_rc.rs | 20 +-- .../src/dataflow/operators/generic/handles.rs | 4 +- .../dataflow/operators/generic/notificator.rs | 2 +- .../dataflow/operators/generic/operator.rs | 80 ++++++------ timely/src/dataflow/operators/input.rs | 10 +- timely/src/dataflow/operators/map.rs | 16 +-- timely/src/dataflow/operators/partition.rs | 23 ++-- timely/src/dataflow/operators/result.rs | 32 ++--- timely/src/dataflow/operators/to_stream.rs | 8 +- .../src/dataflow/operators/unordered_input.rs | 6 +- timely/src/dataflow/stream.rs | 116 ++++++++++++++++-- 51 files changed, 553 insertions(+), 366 deletions(-) create mode 100644 timely/src/dataflow/channels/pushers/owned.rs diff --git a/container/src/lib.rs b/container/src/lib.rs index d48e450150..3e8293b22d 100644 --- a/container/src/lib.rs +++ b/container/src/lib.rs @@ -14,10 +14,6 @@ pub mod flatcontainer; /// /// A container must implement default. The default implementation is not required to allocate /// memory for variable-length components. -/// -/// We require the container to be cloneable to enable efficient copies when providing references -/// of containers to operators. Care must be taken that the type's `clone_from` implementation -/// is efficient (which is not necessarily the case when deriving `Clone`.) pub trait Container: Default { /// The type of elements when reading non-destructively from the container. type ItemRef<'a> where Self: 'a; @@ -105,7 +101,7 @@ pub trait PushInto { /// decide to represent a push order for `extract` and `finish`, or not. pub trait ContainerBuilder: Default + 'static { /// The container type we're building. - type Container: Container + Clone + 'static; + type Container: Container + 'static; /// Extract assembled containers, potentially leaving unfinished data behind. Can /// be called repeatedly, for example while the caller can send data. /// @@ -170,7 +166,7 @@ impl> PushInto for CapacityContainerBuil } } -impl ContainerBuilder for CapacityContainerBuilder { +impl ContainerBuilder for CapacityContainerBuilder { type Container = C; #[inline] diff --git a/mdbook/src/chapter_2/chapter_2_3.md b/mdbook/src/chapter_2/chapter_2_3.md index 71e5774bec..abc3d3afe0 100644 --- a/mdbook/src/chapter_2/chapter_2_3.md +++ b/mdbook/src/chapter_2/chapter_2_3.md @@ -126,12 +126,12 @@ use timely::dataflow::operators::{ToStream, Partition, Inspect}; fn main() { timely::example(|scope| { - let streams = (0..10).to_stream(scope) + let mut streams = (0..10).to_stream(scope) .partition(3, |x| (x % 3, x)); - streams[0].inspect(|x| println!("seen 0: {:?}", x)); - streams[1].inspect(|x| println!("seen 1: {:?}", x)); - streams[2].inspect(|x| println!("seen 2: {:?}", x)); + streams.pop().unwrap().inspect(|x| println!("seen 2: {:?}", x)); + streams.pop().unwrap().inspect(|x| println!("seen 1: {:?}", x)); + streams.pop().unwrap().inspect(|x| println!("seen 0: {:?}", x)); }); } ``` @@ -147,11 +147,11 @@ use timely::dataflow::operators::{ToStream, Partition, Concat, Inspect}; fn main() { timely::example(|scope| { - let streams = (0..10).to_stream(scope) + let mut streams = (0..10).to_stream(scope) .partition(3, |x| (x % 3, x)); - streams[0] - .concat(&streams[1]) - .concat(&streams[2]) + streams.pop().unwrap() + .concat(streams.pop().unwrap()) + .concat(streams.pop().unwrap()) .inspect(|x| println!("seen: {:?}", x)); }); } diff --git a/mdbook/src/chapter_2/chapter_2_4.md b/mdbook/src/chapter_2/chapter_2_4.md index dbbf7fbc1f..19c76326b3 100644 --- a/mdbook/src/chapter_2/chapter_2_4.md +++ b/mdbook/src/chapter_2/chapter_2_4.md @@ -182,7 +182,7 @@ fn main() { let in1 = (0 .. 10).to_stream(scope); let in2 = (0 .. 10).to_stream(scope); - in1.binary_frontier(&in2, Pipeline, Pipeline, "concat_buffer", |capability, info| { + in1.binary_frontier(in2, Pipeline, Pipeline, "concat_buffer", |capability, info| { let mut notificator = FrontierNotificator::default(); let mut stash = HashMap::new(); @@ -233,7 +233,7 @@ fn main() { let in1 = (0 .. 10).to_stream(scope); let in2 = (0 .. 10).to_stream(scope); - in1.binary_frontier(&in2, Pipeline, Pipeline, "concat_buffer", |capability, info| { + in1.binary_frontier(in2, Pipeline, Pipeline, "concat_buffer", |capability, info| { let mut stash = HashMap::new(); diff --git a/mdbook/src/chapter_4/chapter_4_2.md b/mdbook/src/chapter_4/chapter_4_2.md index 7c057da389..45230367a7 100644 --- a/mdbook/src/chapter_4/chapter_4_2.md +++ b/mdbook/src/chapter_4/chapter_4_2.md @@ -24,7 +24,7 @@ fn main() { // circulate numbers, Collatz stepping each time. (1 .. 10) .to_stream(scope) - .concat(&stream) + .concat(stream) .map(|x| if x % 2 == 0 { x / 2 } else { 3 * x + 1 } ) .inspect(|x| println!("{:?}", x)) .filter(|x| *x != 1) @@ -63,17 +63,17 @@ fn main() { let results1 = stream1.map(|x| 3 * x + 1); // partition the input and feedback streams by even-ness. - let parts = + let mut parts = (1 .. 10) .to_stream(scope) - .concat(&results0) - .concat(&results1) + .concat(results0) + .concat(results1) .inspect(|x| println!("{:?}", x)) .partition(2, |x| (x % 2, x)); // connect each part appropriately. - parts[0].connect_loop(handle0); - parts[1].connect_loop(handle1); + parts.pop().unwrap().connect_loop(handle1); + parts.pop().unwrap().connect_loop(handle0); }); } ``` @@ -103,7 +103,7 @@ fn main() { input .enter(subscope) - .concat(&stream) + .concat(stream) .map(|x| if x % 2 == 0 { x / 2 } else { 3 * x + 1 } ) .inspect(|x| println!("{:?}", x)) .filter(|x| *x != 1) diff --git a/mdbook/src/chapter_4/chapter_4_3.md b/mdbook/src/chapter_4/chapter_4_3.md index beeabd8345..32a0e4a564 100644 --- a/mdbook/src/chapter_4/chapter_4_3.md +++ b/mdbook/src/chapter_4/chapter_4_3.md @@ -76,7 +76,7 @@ fn main() { // Assign timestamps to records so that not much work is in each time. .delay(|number, time| number / 100 ) // Buffer records until all prior timestamps have completed. - .binary_frontier(&cycle, Pipeline, Pipeline, "Buffer", move |capability, info| { + .binary_frontier(cycle, Pipeline, Pipeline, "Buffer", move |capability, info| { move |input1, input2, output| { diff --git a/timely/examples/bfs.rs b/timely/examples/bfs.rs index 1f29f3962e..c8a8cc3511 100644 --- a/timely/examples/bfs.rs +++ b/timely/examples/bfs.rs @@ -45,7 +45,7 @@ fn main() { // use the stream of edges graph.binary_notify( - &stream, + stream, Exchange::new(|x: &(u32, u32)| u64::from(x.0)), Exchange::new(|x: &(u32, u32)| u64::from(x.0)), "BFS", @@ -130,7 +130,7 @@ fn main() { }); } ) - .concat(&(0..1).map(|x| (x,x)).to_stream(scope)) + .concat((0..1).map(|x| (x,x)).to_stream(scope)) .connect_loop(handle); }); }).unwrap(); // asserts error-free execution; diff --git a/timely/examples/hashjoin.rs b/timely/examples/hashjoin.rs index c5383f993e..706bdc422d 100644 --- a/timely/examples/hashjoin.rs +++ b/timely/examples/hashjoin.rs @@ -32,7 +32,7 @@ fn main() { let exchange2 = Exchange::new(|x: &(u64, u64)| x.0); stream1 - .binary(&stream2, exchange1, exchange2, "HashJoin", |_capability, _info| { + .binary(stream2, exchange1, exchange2, "HashJoin", |_capability, _info| { let mut map1 = HashMap::>::new(); let mut map2 = HashMap::>::new(); diff --git a/timely/examples/loopdemo.rs b/timely/examples/loopdemo.rs index df7dc3fe80..5681a21c94 100644 --- a/timely/examples/loopdemo.rs +++ b/timely/examples/loopdemo.rs @@ -27,12 +27,12 @@ fn main() { let step = stream - .concat(&loop_stream) + .concat(loop_stream) .map(|x| if x % 2 == 0 { x / 2 } else { 3 * x + 1 }) .filter(|x| x > &1); - - step.connect_loop(loop_handle); - step.probe_with(&mut probe); + step + .probe_with(&mut probe) + .connect_loop(loop_handle); }); let ns_per_request = 1_000_000_000 / rate; diff --git a/timely/examples/pagerank.rs b/timely/examples/pagerank.rs index 8c9cdb08f0..301719f79f 100644 --- a/timely/examples/pagerank.rs +++ b/timely/examples/pagerank.rs @@ -23,7 +23,7 @@ fn main() { // bring edges and ranks together! let changes = edge_stream.binary_frontier( - &rank_stream, + rank_stream, Exchange::new(|x: &((usize, usize), i64)| (x.0).0 as u64), Exchange::new(|x: &(usize, i64)| x.0 as u64), "PageRank", diff --git a/timely/examples/pingpong.rs b/timely/examples/pingpong.rs index 2104de289b..edd644d385 100644 --- a/timely/examples/pingpong.rs +++ b/timely/examples/pingpong.rs @@ -14,7 +14,7 @@ fn main() { (0 .. elements) .filter(move |&x| (x as usize) % peers == index) .to_stream(scope) - .concat(&cycle) + .concat(cycle) .exchange(|&x| x) .map_in_place(|x| *x += 1) .branch_when(move |t| t < &iterations).1 diff --git a/timely/examples/unionfind.rs b/timely/examples/unionfind.rs index 04641c1748..a0855d4b2c 100644 --- a/timely/examples/unionfind.rs +++ b/timely/examples/unionfind.rs @@ -47,12 +47,12 @@ fn main() { }).unwrap(); // asserts error-free execution; } -trait UnionFind { - fn union_find(&self) -> Self; +trait UnionFind { + fn union_find(self) -> OwnedStream>; } -impl UnionFind for Stream { - fn union_find(&self) -> Stream { +impl>> UnionFind for S { + fn union_find(self) -> OwnedStream> { self.unary(Pipeline, "UnionFind", |_,_| { diff --git a/timely/src/dataflow/channels/pushers/buffer.rs b/timely/src/dataflow/channels/pushers/buffer.rs index 4ea872fb7f..e34c21978c 100644 --- a/timely/src/dataflow/channels/pushers/buffer.rs +++ b/timely/src/dataflow/channels/pushers/buffer.rs @@ -6,7 +6,7 @@ use crate::container::{ContainerBuilder, CapacityContainerBuilder, PushInto}; use crate::dataflow::channels::Message; use crate::dataflow::operators::Capability; use crate::progress::Timestamp; -use crate::{Container, Data}; +use crate::Container; /// Buffers data sent at the same time, for efficient communication. /// @@ -44,7 +44,7 @@ impl Buffer { } } -impl>> Buffer, P> where T: Eq+Clone { +impl>> Buffer, P> where T: Eq+Clone { /// Returns a `Session`, which accepts data to send at the associated time #[inline] pub fn session(&mut self, time: &T) -> Session, P> { @@ -133,7 +133,7 @@ pub struct Session<'a, T, CB, P> { buffer: &'a mut Buffer, } -impl<'a, T, C: Container + Data, P> Session<'a, T, CapacityContainerBuilder, P> +impl<'a, T, C: Container + 'static, P> Session<'a, T, CapacityContainerBuilder, P> where T: Eq + Clone + 'a, P: Push> + 'a, diff --git a/timely/src/dataflow/channels/pushers/mod.rs b/timely/src/dataflow/channels/pushers/mod.rs index 295d033cab..fa6ee3b250 100644 --- a/timely/src/dataflow/channels/pushers/mod.rs +++ b/timely/src/dataflow/channels/pushers/mod.rs @@ -1,7 +1,9 @@ -pub use self::tee::{Tee, TeeHelper}; -pub use self::exchange::Exchange; pub use self::counter::Counter; +pub use self::exchange::Exchange; +pub use self::owned::PushOwned; +pub use self::tee::{Tee, TeeHelper}; +pub mod owned; pub mod tee; pub mod exchange; pub mod counter; diff --git a/timely/src/dataflow/channels/pushers/owned.rs b/timely/src/dataflow/channels/pushers/owned.rs new file mode 100644 index 0000000000..4286a40784 --- /dev/null +++ b/timely/src/dataflow/channels/pushers/owned.rs @@ -0,0 +1,48 @@ +//! A `Push` implementor with a single target. + +use std::cell::RefCell; +use std::fmt; +use std::rc::Rc; + +use timely_communication::Push; +use crate::{Data, Container}; +use crate::dataflow::channels::Message; + +/// A pusher that can bind to a single downstream pusher. +pub struct PushOwned(Rc>>>>>); + +impl PushOwned { + /// Create a new `PushOwned`. Similarly to `Tee`, it returns a pair where either element + /// can be used as pusher or registrar. + pub fn new() -> (Self, Self) { + let zelf = Self(Rc::new(RefCell::new(None))); + (zelf.clone(), zelf) + } + + /// Set the downstream pusher. + pub fn set> + 'static>(self, pusher: P) { + *self.0.borrow_mut() = Some(Box::new(pusher)); + } +} + +impl fmt::Debug for PushOwned { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PushOwned").finish_non_exhaustive() + } +} + +impl Clone for PushOwned { + fn clone(&self) -> Self { + Self(Rc::clone(&self.0)) + } +} + +impl Push> for PushOwned { + #[inline] + fn push(&mut self, message: &mut Option>) { + let mut pusher = self.0.borrow_mut(); + if let Some(pusher) = pusher.as_mut() { + pusher.push(message); + } + } +} diff --git a/timely/src/dataflow/mod.rs b/timely/src/dataflow/mod.rs index 043317fd8d..f263451014 100644 --- a/timely/src/dataflow/mod.rs +++ b/timely/src/dataflow/mod.rs @@ -13,7 +13,7 @@ //! }); //! ``` -pub use self::stream::{StreamCore, Stream}; +pub use self::stream::{StreamCore, Stream, StreamLike, OwnedStream}; pub use self::scopes::{Scope, ScopeParent}; pub use self::operators::core::input::Handle as InputHandleCore; diff --git a/timely/src/dataflow/operators/aggregation/aggregate.rs b/timely/src/dataflow/operators/aggregation/aggregate.rs index 196f1c5d01..68be9b8956 100644 --- a/timely/src/dataflow/operators/aggregation/aggregate.rs +++ b/timely/src/dataflow/operators/aggregation/aggregate.rs @@ -3,7 +3,7 @@ use std::hash::Hash; use std::collections::HashMap; use crate::{Data, ExchangeData}; -use crate::dataflow::{Stream, Scope}; +use crate::dataflow::{Scope, StreamLike, OwnedStream}; use crate::dataflow::operators::generic::operator::Operator; use crate::dataflow::channels::pact::Exchange; @@ -61,19 +61,25 @@ pub trait Aggregate { /// }); /// ``` fn aggregateR+'static, H: Fn(&K)->u64+'static>( - &self, + self, fold: F, emit: E, - hash: H) -> Stream where S::Timestamp: Eq; + hash: H) -> OwnedStream> where S::Timestamp: Eq; } -impl Aggregate for Stream { +impl Aggregate for S +where + G: Scope, + K: ExchangeData + Hash + Eq + Clone, + V: ExchangeData, + S: StreamLike>, +{ fn aggregateR+'static, H: Fn(&K)->u64+'static>( - &self, + self, fold: F, emit: E, - hash: H) -> Stream where S::Timestamp: Eq { + hash: H) -> OwnedStream> where G::Timestamp: Eq { let mut aggregates = HashMap::new(); self.unary_notify(Exchange::new(move |(k, _)| hash(k)), "Aggregate", vec![], move |input, output, notificator| { diff --git a/timely/src/dataflow/operators/aggregation/state_machine.rs b/timely/src/dataflow/operators/aggregation/state_machine.rs index 7347f8c12b..33a6a59c04 100644 --- a/timely/src/dataflow/operators/aggregation/state_machine.rs +++ b/timely/src/dataflow/operators/aggregation/state_machine.rs @@ -3,7 +3,7 @@ use std::hash::Hash; use std::collections::HashMap; use crate::{Data, ExchangeData}; -use crate::dataflow::{Stream, Scope}; +use crate::dataflow::{OwnedStream, Scope, StreamLike}; use crate::dataflow::operators::generic::operator::Operator; use crate::dataflow::channels::pact::Exchange; @@ -51,17 +51,23 @@ pub trait StateMachine { I: IntoIterator, // type of output iterator F: Fn(&K, V, &mut D)->(bool, I)+'static, // state update logic H: Fn(&K)->u64+'static, // "hash" function for keys - >(&self, fold: F, hash: H) -> Stream where S::Timestamp : Hash+Eq ; + >(self, fold: F, hash: H) -> OwnedStream> where S::Timestamp : Hash+Eq ; } -impl StateMachine for Stream { +impl StateMachine for S +where + G: Scope, + K: ExchangeData + Hash + Eq + Clone, + V: ExchangeData, + S: StreamLike>, +{ fn state_machine< R: Data, // output type D: Default+'static, // per-key state (data) I: IntoIterator, // type of output iterator F: Fn(&K, V, &mut D)->(bool, I)+'static, // state update logic H: Fn(&K)->u64+'static, // "hash" function for keys - >(&self, fold: F, hash: H) -> Stream where S::Timestamp : Hash+Eq { + >(self, fold: F, hash: H) -> OwnedStream> where G::Timestamp : Hash+Eq { let mut pending: HashMap<_, Vec<(K, V)>> = HashMap::new(); // times -> (keys -> state) let mut states = HashMap::new(); // keys -> state diff --git a/timely/src/dataflow/operators/branch.rs b/timely/src/dataflow/operators/branch.rs index c3865cb2b4..50a71477a9 100644 --- a/timely/src/dataflow/operators/branch.rs +++ b/timely/src/dataflow/operators/branch.rs @@ -2,7 +2,7 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; -use crate::dataflow::{Scope, Stream, StreamCore}; +use crate::dataflow::{Scope, OwnedStream, StreamLike}; use crate::{Container, Data}; /// Extension trait for `Stream`. @@ -29,16 +29,16 @@ pub trait Branch { /// }); /// ``` fn branch( - &self, + self, condition: impl Fn(&S::Timestamp, &D) -> bool + 'static, - ) -> (Stream, Stream); + ) -> (OwnedStream>, OwnedStream>); } -impl Branch for Stream { +impl>> Branch for S { fn branch( - &self, - condition: impl Fn(&S::Timestamp, &D) -> bool + 'static, - ) -> (Stream, Stream) { + self, + condition: impl Fn(&G::Timestamp, &D) -> bool + 'static, + ) -> (OwnedStream>, OwnedStream>) { let mut builder = OperatorBuilder::new("Branch".to_owned(), self.scope()); let mut input = builder.new_input(self, Pipeline); @@ -69,7 +69,7 @@ impl Branch for Stream { } /// Extension trait for `Stream`. -pub trait BranchWhen: Sized { +pub trait BranchWhen: Sized { /// Takes one input stream and splits it into two output streams. /// For each time, the supplied closure is called. If it returns `true`, /// the records for that will be sent to the second returned stream, otherwise @@ -89,11 +89,11 @@ pub trait BranchWhen: Sized { /// after_five.inspect(|x| println!("Times 5 and later: {:?}", x)); /// }); /// ``` - fn branch_when(&self, condition: impl Fn(&T) -> bool + 'static) -> (Self, Self); + fn branch_when(self, condition: impl Fn(&G::Timestamp) -> bool + 'static) -> (OwnedStream, OwnedStream); } -impl BranchWhen for StreamCore { - fn branch_when(&self, condition: impl Fn(&S::Timestamp) -> bool + 'static) -> (Self, Self) { +impl> BranchWhen for S { + fn branch_when(self, condition: impl Fn(&G::Timestamp) -> bool + 'static) -> (OwnedStream, OwnedStream) { let mut builder = OperatorBuilder::new("Branch".to_owned(), self.scope()); let mut input = builder.new_input(self, Pipeline); diff --git a/timely/src/dataflow/operators/broadcast.rs b/timely/src/dataflow/operators/broadcast.rs index 21d80d6da4..5daa0b11de 100644 --- a/timely/src/dataflow/operators/broadcast.rs +++ b/timely/src/dataflow/operators/broadcast.rs @@ -1,11 +1,11 @@ //! Broadcast records to all workers. use crate::ExchangeData; -use crate::dataflow::{Stream, Scope}; +use crate::dataflow::{OwnedStream, StreamLike, Scope}; use crate::dataflow::operators::{Map, Exchange}; /// Broadcast records to all workers. -pub trait Broadcast { +pub trait Broadcast { /// Broadcast records to all workers. /// /// # Examples @@ -18,11 +18,11 @@ pub trait Broadcast { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn broadcast(&self) -> Self; + fn broadcast(self) -> OwnedStream>; } -impl Broadcast for Stream { - fn broadcast(&self) -> Stream { +impl>> Broadcast for S { + fn broadcast(self) -> OwnedStream> { // NOTE: Simplified implementation due to underlying motion // in timely dataflow internals. Optimize once they have diff --git a/timely/src/dataflow/operators/core/capture/capture.rs b/timely/src/dataflow/operators/core/capture/capture.rs index 82bef229a1..077b80a962 100644 --- a/timely/src/dataflow/operators/core/capture/capture.rs +++ b/timely/src/dataflow/operators/core/capture/capture.rs @@ -5,19 +5,19 @@ //! and there are several default implementations, including a linked-list, Rust's MPSC //! queue, and a binary serializer wrapping any `W: Write`. -use crate::dataflow::{Scope, StreamCore}; +use crate::dataflow::{Scope, StreamLike}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::channels::pullers::Counter as PullCounter; use crate::dataflow::operators::generic::builder_raw::OperatorBuilder; -use crate::{Container, Data}; +use crate::Container; use crate::progress::ChangeBatch; use crate::progress::Timestamp; use super::{Event, EventPusher}; /// Capture a stream of timestamped data for later replay. -pub trait Capture { +pub trait Capture : Sized { /// Captures a stream of timestamped data for later replay. /// /// # Examples @@ -103,18 +103,18 @@ pub trait Capture { /// /// assert_eq!(recv0.extract()[0].1, (0..10).collect::>()); /// ``` - fn capture_into+'static>(&self, pusher: P); + fn capture_into+'static>(self, pusher: P); /// Captures a stream using Rust's MPSC channels. - fn capture(&self) -> ::std::sync::mpsc::Receiver> { + fn capture(self) -> ::std::sync::mpsc::Receiver> { let (send, recv) = ::std::sync::mpsc::channel(); self.capture_into(send); recv } } -impl Capture for StreamCore { - fn capture_into+'static>(&self, mut event_pusher: P) { +impl> Capture for S { + fn capture_into+'static>(self, mut event_pusher: P) { let mut builder = OperatorBuilder::new("Capture".to_owned(), self.scope()); let mut input = PullCounter::new(builder.new_input(self, Pipeline)); @@ -125,7 +125,7 @@ impl Capture for StreamCore : Sized { - /// Replays `self` into the provided scope, as a `StreamCore`. - fn replay_into>(self, scope: &mut S) -> StreamCore { + /// Replays `self` into the provided scope, as a `OwnedStream`. + fn replay_into>(self, scope: &mut S) -> OwnedStream { self.replay_core(scope, Some(std::time::Duration::new(0, 0))) } - /// Replays `self` into the provided scope, as a `StreamCore`. + /// Replays `self` into the provided scope, as a `OwnedStream`. /// /// The `period` argument allows the specification of a re-activation period, where the operator /// will re-activate itself every so often. The `None` argument instructs the operator not to /// re-activate itself. - fn replay_core>(self, scope: &mut S, period: Option) -> StreamCore; + fn replay_core>(self, scope: &mut S, period: Option) -> OwnedStream; } impl Replay for I @@ -67,7 +67,7 @@ where I : IntoIterator, ::Item: EventIterator+'static, { - fn replay_core>(self, scope: &mut S, period: Option) -> StreamCore{ + fn replay_core>(self, scope: &mut S, period: Option) -> OwnedStream{ let mut builder = OperatorBuilder::new("Replay".to_owned(), scope.clone()); diff --git a/timely/src/dataflow/operators/core/concat.rs b/timely/src/dataflow/operators/core/concat.rs index 89848f6890..779b7ba44e 100644 --- a/timely/src/dataflow/operators/core/concat.rs +++ b/timely/src/dataflow/operators/core/concat.rs @@ -1,12 +1,12 @@ //! Merges the contents of multiple streams. -use crate::{Container, Data}; +use crate::Container; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{StreamCore, Scope}; +use crate::dataflow::{OwnedStream, StreamLike, Scope}; /// Merge the contents of two streams. -pub trait Concat { +pub trait Concat> { /// Merge the contents of two streams. /// /// # Examples @@ -15,22 +15,22 @@ pub trait Concat { /// /// timely::example(|scope| { /// - /// let stream = (0..10).to_stream(scope); + /// let stream = (0..10).to_stream(scope).tee(); /// stream.concat(&stream) /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn concat(&self, _: &StreamCore) -> StreamCore; + fn concat(self, other: S) -> OwnedStream; } -impl Concat for StreamCore { - fn concat(&self, other: &StreamCore) -> StreamCore { - self.scope().concatenate([self.clone(), other.clone()]) +impl> Concat for S { + fn concat(self, other: S) -> OwnedStream { + self.scope().concatenate([self, other]) } } /// Merge the contents of multiple streams. -pub trait Concatenate { +pub trait Concatenate> { /// Merge the contents of multiple streams. /// /// # Examples @@ -47,25 +47,24 @@ pub trait Concatenate { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn concatenate(&self, sources: I) -> StreamCore + fn concatenate(self, sources: I) -> OwnedStream where - I: IntoIterator>; + I: IntoIterator; } -impl Concatenate for StreamCore { - fn concatenate(&self, sources: I) -> StreamCore +impl Concatenate> for OwnedStream { + fn concatenate(self, sources: I) -> OwnedStream where - I: IntoIterator> + I: IntoIterator> { - let clone = self.clone(); - self.scope().concatenate(Some(clone).into_iter().chain(sources)) + self.scope().concatenate(Some(self).into_iter().chain(sources)) } } -impl Concatenate for G { - fn concatenate(&self, sources: I) -> StreamCore +impl> Concatenate for &G { + fn concatenate(self, sources: I) -> OwnedStream where - I: IntoIterator> + I: IntoIterator { // create an operator builder. @@ -73,7 +72,7 @@ impl Concatenate for G { let mut builder = OperatorBuilder::new("Concatenate".to_string(), self.clone()); // create new input handles for each input stream. - let mut handles = sources.into_iter().map(|s| builder.new_input(&s, Pipeline)).collect::>(); + let mut handles = sources.into_iter().map(|s| builder.new_input(s, Pipeline)).collect::>(); // create one output handle for the concatenated results. let (mut output, result) = builder.new_output(); diff --git a/timely/src/dataflow/operators/core/enterleave.rs b/timely/src/dataflow/operators/core/enterleave.rs index 16464a7f60..3ccf303e8e 100644 --- a/timely/src/dataflow/operators/core/enterleave.rs +++ b/timely/src/dataflow/operators/core/enterleave.rs @@ -27,11 +27,11 @@ use crate::progress::timestamp::Refines; use crate::progress::{Source, Target}; use crate::{Container, Data}; use crate::communication::Push; -use crate::dataflow::channels::pushers::{Counter, Tee}; +use crate::dataflow::channels::pushers::{Counter, PushOwned}; use crate::dataflow::channels::Message; use crate::worker::AsWorker; -use crate::dataflow::{StreamCore, Scope}; +use crate::dataflow::{Scope, OwnedStream, StreamLike}; use crate::dataflow::scopes::Child; /// Extension trait to move a `Stream` into a child of its current `Scope`. @@ -50,15 +50,15 @@ pub trait Enter, C: Container> { /// }); /// }); /// ``` - fn enter<'a>(&self, _: &Child<'a, G, T>) -> StreamCore, C>; + fn enter<'a>(self, _: &Child<'a, G, T>) -> OwnedStream, C>; } -impl, C: Data+Container> Enter for StreamCore { - fn enter<'a>(&self, scope: &Child<'a, G, T>) -> StreamCore, C> { +impl, C: Data+Container, S: StreamLike> Enter for S { + fn enter<'a>(self, scope: &Child<'a, G, T>) -> OwnedStream, C> { use crate::scheduling::Scheduler; - let (targets, registrar) = Tee::::new(); + let (targets, registrar) = PushOwned::::new(); let ingress = IngressNub { targets: Counter::new(targets), phantom: PhantomData, @@ -76,7 +76,7 @@ impl, C: Data+Container> Enter, C: Data+Container> Enter { +pub trait Leave { /// Moves a `Stream` to the parent of its current `Scope`. /// /// # Examples @@ -100,20 +100,20 @@ pub trait Leave { /// }); /// }); /// ``` - fn leave(&self) -> StreamCore; + fn leave(self) -> OwnedStream; } -impl> Leave for StreamCore, C> { - fn leave(&self) -> StreamCore { +impl<'a, G: Scope, C: Container + 'static, T: Timestamp+Refines, S: StreamLike, C>> Leave for S { + fn leave(self) -> OwnedStream { let scope = self.scope(); let output = scope.subgraph.borrow_mut().new_output(); - let target = Target::new(0, output.port); - let (targets, registrar) = Tee::::new(); - let egress = EgressNub { targets, phantom: PhantomData }; + let (target, registrar) = PushOwned::::new(); + let egress = EgressNub { target, phantom: PhantomData }; let channel_id = scope.clone().new_identifier(); + let target = Target::new(0, output.port); if let Some(logger) = scope.logging() { let pusher = LogPusher::new(egress, channel_id, scope.index(), logger); self.connect_to(target, pusher, channel_id); @@ -121,7 +121,7 @@ impl> Leave> Leave, TContainer: Container + Data> { - targets: Counter>, - phantom: ::std::marker::PhantomData, + targets: Counter>, + phantom: PhantomData, activator: crate::scheduling::Activator, active: bool, } @@ -159,23 +159,23 @@ impl, TContainer: Container } -struct EgressNub, TContainer: Data> { - targets: Tee, +struct EgressNub, TContainer> { + target: PushOwned, phantom: PhantomData, } impl Push> for EgressNub -where TOuter: Timestamp, TInner: Timestamp+Refines, TContainer: Data { +where TOuter: Timestamp, TInner: Timestamp+Refines, TContainer: 'static { fn push(&mut self, message: &mut Option>) { if let Some(inner_message) = message { let data = ::std::mem::take(&mut inner_message.data); let mut outer_message = Some(Message::new(inner_message.time.clone().to_outer(), data, 0, 0)); - self.targets.push(&mut outer_message); + self.target.push(&mut outer_message); if let Some(outer_message) = outer_message { inner_message.data = outer_message.data; } } - else { self.targets.done(); } + else { self.target.done(); } } } diff --git a/timely/src/dataflow/operators/core/exchange.rs b/timely/src/dataflow/operators/core/exchange.rs index bb021a5d54..14660c7237 100644 --- a/timely/src/dataflow/operators/core/exchange.rs +++ b/timely/src/dataflow/operators/core/exchange.rs @@ -1,13 +1,14 @@ //! Exchange records between workers. +use crate::Container; use crate::ExchangeData; -use crate::container::{Container, SizableContainer, PushInto}; +use crate::container::{SizableContainer, PushInto}; use crate::dataflow::channels::pact::ExchangeCore; use crate::dataflow::operators::generic::operator::Operator; -use crate::dataflow::{Scope, StreamCore}; +use crate::dataflow::{Scope, OwnedStream, StreamLike}; /// Exchange records between workers. -pub trait Exchange { +pub trait Exchange { /// Exchange records between workers. /// /// The closure supplied should map a reference to a record to a `u64`, @@ -23,18 +24,18 @@ pub trait Exchange { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn exchange(&self, route: F) -> Self + fn exchange(self, route: F) -> OwnedStream where for<'a> F: FnMut(&C::Item<'a>) -> u64 + 'static; } -impl Exchange for StreamCore +impl Exchange for S where C: SizableContainer + ExchangeData + crate::dataflow::channels::ContainerBytes, C: for<'a> PushInto>, - + S: StreamLike { - fn exchange(&self, route: F) -> StreamCore + fn exchange(self, route: F) -> OwnedStream where for<'a> F: FnMut(&C::Item<'a>) -> u64 + 'static, { diff --git a/timely/src/dataflow/operators/core/feedback.rs b/timely/src/dataflow/operators/core/feedback.rs index e008b30291..0929b25ce1 100644 --- a/timely/src/dataflow/operators/core/feedback.rs +++ b/timely/src/dataflow/operators/core/feedback.rs @@ -3,11 +3,11 @@ use crate::{Container, Data}; use crate::container::CapacityContainerBuilder; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::channels::pushers::Tee; +use crate::dataflow::channels::pushers::PushOwned; use crate::dataflow::operators::generic::OutputWrapper; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::dataflow::scopes::child::Iterative; -use crate::dataflow::{StreamCore, Scope}; +use crate::dataflow::{Scope, OwnedStream, StreamLike}; use crate::order::Product; use crate::progress::frontier::Antichain; use crate::progress::{Timestamp, PathSummary}; @@ -30,13 +30,13 @@ pub trait Feedback { /// // circulate 0..10 for 100 iterations. /// let (handle, cycle) = scope.feedback(1); /// (0..10).to_stream(scope) - /// .concat(&cycle) + /// .concat(cycle) /// .inspect(|x| println!("seen: {:?}", x)) /// .branch_when(|t| t < &100).1 /// .connect_loop(handle); /// }); /// ``` - fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore); + fn feedback(&mut self, summary: ::Summary) -> (Handle, OwnedStream); } /// Creates a `StreamCore` and a `Handle` to later bind the source of that `StreamCore`. @@ -57,19 +57,19 @@ pub trait LoopVariable<'a, G: Scope, T: Timestamp> { /// scope.iterative::(|inner| { /// let (handle, cycle) = inner.loop_variable(1); /// (0..10).to_stream(inner) - /// .concat(&cycle) + /// .concat(cycle) /// .inspect(|x| println!("seen: {:?}", x)) /// .branch_when(|t| t.inner < 100).1 /// .connect_loop(handle); /// }); /// }); /// ``` - fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>); + fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, OwnedStream, C>); } impl Feedback for G { - fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore) { + fn feedback(&mut self, summary: ::Summary) -> (Handle, OwnedStream) { let mut builder = OperatorBuilder::new("Feedback".to_owned(), self.clone()); let (output, stream) = builder.new_output(); @@ -79,7 +79,7 @@ impl Feedback for G { } impl<'a, G: Scope, T: Timestamp> LoopVariable<'a, G, T> for Iterative<'a, G, T> { - fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>) { + fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, OwnedStream, C>) { self.feedback(Product::new(Default::default(), summary)) } } @@ -97,17 +97,17 @@ pub trait ConnectLoop { /// // circulate 0..10 for 100 iterations. /// let (handle, cycle) = scope.feedback(1); /// (0..10).to_stream(scope) - /// .concat(&cycle) + /// .concat(cycle) /// .inspect(|x| println!("seen: {:?}", x)) /// .branch_when(|t| t < &100).1 /// .connect_loop(handle); /// }); /// ``` - fn connect_loop(&self, handle: Handle); + fn connect_loop(self, handle: Handle); } -impl ConnectLoop for StreamCore { - fn connect_loop(&self, handle: Handle) { +impl> ConnectLoop for S { + fn connect_loop(self, handle: Handle) { let mut builder = handle.builder; let summary = handle.summary; @@ -131,8 +131,8 @@ impl ConnectLoop for StreamCore { /// A handle used to bind the source of a loop variable. #[derive(Debug)] -pub struct Handle { +pub struct Handle { builder: OperatorBuilder, summary: ::Summary, - output: OutputWrapper, Tee>, + output: OutputWrapper, PushOwned>, } diff --git a/timely/src/dataflow/operators/core/filter.rs b/timely/src/dataflow/operators/core/filter.rs index 80ef564b0a..f60eb39c62 100644 --- a/timely/src/dataflow/operators/core/filter.rs +++ b/timely/src/dataflow/operators/core/filter.rs @@ -2,11 +2,11 @@ use crate::container::{Container, SizableContainer, PushInto}; use crate::Data; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{Scope, StreamCore}; +use crate::dataflow::{OwnedStream, Scope, StreamLike}; use crate::dataflow::operators::generic::operator::Operator; /// Extension trait for filtering. -pub trait Filter { +pub trait Filter { /// Returns a new instance of `self` containing only records satisfying `predicate`. /// /// # Examples @@ -20,14 +20,14 @@ pub trait Filter { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn filter)->bool+'static>(&self, predicate: P) -> Self; + fn filter)->bool+'static>(self, predicate: P) -> OwnedStream; } -impl Filter for StreamCore +impl> Filter for S where for<'a> C: PushInto> { - fn filter)->bool+'static>(&self, mut predicate: P) -> StreamCore { + fn filter)->bool+'static>(self, mut predicate: P) -> OwnedStream { self.unary(Pipeline, "Filter", move |_,_| move |input, output| { input.for_each(|time, data| { if !data.is_empty() { diff --git a/timely/src/dataflow/operators/core/input.rs b/timely/src/dataflow/operators/core/input.rs index a1af533c22..d2c6e47818 100644 --- a/timely/src/dataflow/operators/core/input.rs +++ b/timely/src/dataflow/operators/core/input.rs @@ -13,8 +13,8 @@ use crate::progress::Source; use crate::{Container, Data}; use crate::communication::Push; -use crate::dataflow::{Scope, ScopeParent, StreamCore}; -use crate::dataflow::channels::pushers::{Tee, Counter}; +use crate::dataflow::{OwnedStream, Scope, ScopeParent}; +use crate::dataflow::channels::pushers::{Counter, PushOwned}; use crate::dataflow::channels::Message; @@ -60,7 +60,7 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore); + fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, OwnedStream); /// Create a new [StreamCore] and [Handle] through which to supply input. /// @@ -97,7 +97,10 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn new_input_with_builder(&mut self) -> (Handle<::Timestamp, CB>, StreamCore); + fn new_input_with_builder(&mut self) -> (Handle<::Timestamp, CB>, OwnedStream) + where + CB::Container: Clone, + ; /// Create a new stream from a supplied interactive handle. /// @@ -130,25 +133,34 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn input_from(&mut self, handle: &mut Handle<::Timestamp, CB>) -> StreamCore; + fn input_from(&mut self, handle: &mut Handle<::Timestamp, CB>) -> OwnedStream + where + CB::Container: Clone, + ; } use crate::order::TotalOrder; impl Input for G where ::Timestamp: TotalOrder { - fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore) { + fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, OwnedStream) { let mut handle = Handle::new(); let stream = self.input_from(&mut handle); (handle, stream) } - fn new_input_with_builder(&mut self) -> (Handle<::Timestamp, CB>, StreamCore) { + fn new_input_with_builder(&mut self) -> (Handle<::Timestamp, CB>, OwnedStream) + where + CB::Container: Clone, + { let mut handle = Handle::new_with_builder(); let stream = self.input_from(&mut handle); (handle, stream) } - fn input_from(&mut self, handle: &mut Handle<::Timestamp, CB>) -> StreamCore { - let (output, registrar) = Tee::<::Timestamp, CB::Container>::new(); + fn input_from(&mut self, handle: &mut Handle<::Timestamp, CB>) -> OwnedStream + where + CB::Container: Clone, + { + let (output, registrar) = PushOwned::<::Timestamp, CB::Container>::new(); let counter = Counter::new(output); let produced = counter.produced().clone(); @@ -172,7 +184,7 @@ impl Input for G where ::Timestamp: TotalOrder { copies, }), index); - StreamCore::new(Source::new(index, 0), registrar, self.clone()) + OwnedStream::new(Source::new(index, 0), registrar, self.clone()) } } @@ -216,10 +228,13 @@ impl Operate for Operator { /// A handle to an input `StreamCore`, used to introduce data to a timely dataflow computation. #[derive(Debug)] -pub struct Handle { +pub struct Handle +where + CB::Container: Clone, +{ activate: Vec, progress: Vec>>>, - pushers: Vec>>, + pushers: Vec>>, builder: CB, buffer: CB::Container, now_at: T, @@ -265,7 +280,10 @@ impl Handle> { } } -impl Handle { +impl Handle +where + CB::Container: Clone, +{ /// Allocates a new input handle, from which one can create timely streams. /// /// # Examples @@ -332,7 +350,7 @@ impl Handle { /// } /// }); /// ``` - pub fn to_stream(&mut self, scope: &mut G) -> StreamCore + pub fn to_stream(&mut self, scope: &mut G) -> OwnedStream where T: TotalOrder, G: Scope, @@ -342,7 +360,7 @@ impl Handle { fn register( &mut self, - pusher: Counter>, + pusher: Counter>, progress: Rc>>, ) { // flush current contents, so new registrant does not see existing data. @@ -381,7 +399,7 @@ impl Handle { fn send_container( container: &mut CB::Container, buffer: &mut CB::Container, - pushers: &mut [Counter>], + pushers: &mut [Counter>], now_at: &T ) { for index in 0 .. pushers.len() { @@ -486,6 +504,7 @@ impl PushInto for Handle where T: Timestamp, CB: ContainerBuilder + PushInto, + CB::Container: Clone, { #[inline] fn push_into(&mut self, item: D) { @@ -494,7 +513,10 @@ where } } -impl Handle { +impl Handle +where + CB::Container: Clone, +{ /// Sends one record into the corresponding timely dataflow `Stream`, at the current epoch. /// /// # Examples @@ -528,13 +550,19 @@ impl Handle { } } -impl Default for Handle { +impl Default for Handle +where + CB::Container: Clone, +{ fn default() -> Self { Self::new_with_builder() } } -impl Drop for Handle { +impl Drop for Handle +where + CB::Container: Clone, +{ fn drop(&mut self) { self.close_epoch(); } diff --git a/timely/src/dataflow/operators/core/inspect.rs b/timely/src/dataflow/operators/core/inspect.rs index b8c41f97bd..b578e11d42 100644 --- a/timely/src/dataflow/operators/core/inspect.rs +++ b/timely/src/dataflow/operators/core/inspect.rs @@ -2,7 +2,7 @@ use crate::{Container, Data}; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{Scope, StreamCore}; +use crate::dataflow::{Scope, OwnedStream, StreamLike}; use crate::dataflow::operators::generic::Operator; /// Methods to inspect records and batches of records on a stream. @@ -18,7 +18,7 @@ pub trait Inspect: InspectCore + Sized { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn inspect(&self, mut func: F) -> Self + fn inspect(self, mut func: F) -> OwnedStream where F: for<'a> FnMut(C::ItemRef<'a>) + 'static, { @@ -38,7 +38,7 @@ pub trait Inspect: InspectCore + Sized { /// .inspect_time(|t, x| println!("seen at: {:?}\t{:?}", t, x)); /// }); /// ``` - fn inspect_time(&self, mut func: F) -> Self + fn inspect_time(self, mut func: F) -> OwnedStream where F: for<'a> FnMut(&G::Timestamp, C::ItemRef<'a>) + 'static, { @@ -60,7 +60,7 @@ pub trait Inspect: InspectCore + Sized { /// .inspect_batch(|t,xs| println!("seen at: {:?}\t{:?} records", t, xs.len())); /// }); /// ``` - fn inspect_batch(&self, mut func: impl FnMut(&G::Timestamp, &C)+'static) -> Self { + fn inspect_batch(self, mut func: impl FnMut(&G::Timestamp, &C)+'static) -> OwnedStream { self.inspect_core(move |event| { if let Ok((time, data)) = event { func(time, data); @@ -87,11 +87,11 @@ pub trait Inspect: InspectCore + Sized { /// }); /// }); /// ``` - fn inspect_core(&self, func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; + fn inspect_core(self, func: F) -> OwnedStream where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl Inspect for StreamCore { - fn inspect_core(&self, func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>) + 'static { +impl> Inspect for S { + fn inspect_core(self, func: F) -> OwnedStream where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>) + 'static { self.inspect_container(func) } } @@ -117,12 +117,12 @@ pub trait InspectCore { /// }); /// }); /// ``` - fn inspect_container(&self, func: F) -> StreamCore where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; + fn inspect_container(self, func: F) -> OwnedStream where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl InspectCore for StreamCore { +impl> InspectCore for S { - fn inspect_container(&self, mut func: F) -> StreamCore + fn inspect_container(self, mut func: F) -> OwnedStream where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static { use crate::progress::timestamp::Timestamp; diff --git a/timely/src/dataflow/operators/core/map.rs b/timely/src/dataflow/operators/core/map.rs index 8af70e4a49..8f0a4390f2 100644 --- a/timely/src/dataflow/operators/core/map.rs +++ b/timely/src/dataflow/operators/core/map.rs @@ -2,12 +2,12 @@ use crate::container::{Container, SizableContainer, PushInto}; use crate::Data; -use crate::dataflow::{Scope, StreamCore}; +use crate::dataflow::{OwnedStream, Scope, StreamLike}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::operator::Operator; /// Extension trait for `Stream`. -pub trait Map { +pub trait Map : Sized { /// Consumes each element of the stream and yields a new element. /// /// # Examples @@ -22,7 +22,7 @@ pub trait Map { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn map(&self, mut logic: L) -> StreamCore + fn map(self, mut logic: L) -> OwnedStream where C2: SizableContainer + PushInto + Data, L: FnMut(C::Item<'_>)->D2 + 'static, @@ -43,7 +43,7 @@ pub trait Map { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn flat_map(&self, logic: L) -> StreamCore + fn flat_map(self, logic: L) -> OwnedStream where I: IntoIterator, C2: SizableContainer + PushInto + Data, @@ -51,11 +51,11 @@ pub trait Map { ; } -impl Map for StreamCore { +impl> Map for S { // TODO : This would be more robust if it captured an iterator and then pulled an appropriate // TODO : number of elements from the iterator. This would allow iterators that produce many // TODO : records without taking arbitrarily long and arbitrarily much memory. - fn flat_map(&self, mut logic: L) -> StreamCore + fn flat_map(self, mut logic: L) -> OwnedStream where I: IntoIterator, C2: SizableContainer + PushInto + Data, diff --git a/timely/src/dataflow/operators/core/ok_err.rs b/timely/src/dataflow/operators/core/ok_err.rs index fd7887053a..559b623a64 100644 --- a/timely/src/dataflow/operators/core/ok_err.rs +++ b/timely/src/dataflow/operators/core/ok_err.rs @@ -4,7 +4,7 @@ use crate::container::{Container, SizableContainer, PushInto}; use crate::Data; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; -use crate::dataflow::{Scope, StreamCore}; +use crate::dataflow::{Scope, OwnedStream, StreamLike}; /// Extension trait for `Stream`. pub trait OkErr { @@ -28,10 +28,7 @@ pub trait OkErr { /// odd.container::>().inspect(|x| println!("odd: {:?}", x)); /// }); /// ``` - fn ok_err( - &self, - logic: L, - ) -> (StreamCore, StreamCore) + fn ok_err(self, logic: L) -> (OwnedStream, OwnedStream) where C1: SizableContainer + PushInto + Data, C2: SizableContainer + PushInto + Data, @@ -39,11 +36,8 @@ pub trait OkErr { ; } -impl OkErr for StreamCore { - fn ok_err( - &self, - mut logic: L, - ) -> (StreamCore, StreamCore) +impl> OkErr for S { + fn ok_err(self, mut logic: L) -> (OwnedStream, OwnedStream) where C1: SizableContainer + PushInto + Data, C2: SizableContainer + PushInto + Data, diff --git a/timely/src/dataflow/operators/core/probe.rs b/timely/src/dataflow/operators/core/probe.rs index c7fdced335..8c95b43004 100644 --- a/timely/src/dataflow/operators/core/probe.rs +++ b/timely/src/dataflow/operators/core/probe.rs @@ -12,7 +12,7 @@ use crate::dataflow::channels::pullers::Counter as PullCounter; use crate::dataflow::operators::generic::builder_raw::OperatorBuilder; -use crate::dataflow::{StreamCore, Scope}; +use crate::dataflow::{Scope, OwnedStream, StreamLike}; use crate::{Container, Data}; /// Monitors progress at a `Stream`. @@ -44,7 +44,7 @@ pub trait Probe { /// } /// }).unwrap(); /// ``` - fn probe(&self) -> Handle; + fn probe(self) -> Handle; /// Inserts a progress probe in a stream. /// @@ -76,18 +76,18 @@ pub trait Probe { /// } /// }).unwrap(); /// ``` - fn probe_with(&self, handle: &Handle) -> StreamCore; + fn probe_with(self, handle: &Handle) -> OwnedStream; } -impl Probe for StreamCore { - fn probe(&self) -> Handle { +impl> Probe for S { + fn probe(self) -> Handle { // the frontier is shared state; scope updates, handle reads. let handle = Handle::::new(); self.probe_with(&handle); handle } - fn probe_with(&self, handle: &Handle) -> StreamCore { + fn probe_with(self, handle: &Handle) -> OwnedStream { let mut builder = OperatorBuilder::new("Probe".to_owned(), self.scope()); let mut input = PullCounter::new(builder.new_input(self, Pipeline)); diff --git a/timely/src/dataflow/operators/core/rc.rs b/timely/src/dataflow/operators/core/rc.rs index fdc68b9d48..4eebb763a7 100644 --- a/timely/src/dataflow/operators/core/rc.rs +++ b/timely/src/dataflow/operators/core/rc.rs @@ -2,12 +2,12 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::Operator; -use crate::dataflow::{Scope, StreamCore}; +use crate::dataflow::{OwnedStream, Scope, StreamLike}; use crate::{Container, Data}; use std::rc::Rc; /// Convert a stream into a stream of shared containers -pub trait SharedStream { +pub trait SharedStream { /// Convert a stream into a stream of shared data /// /// # Examples @@ -21,11 +21,11 @@ pub trait SharedStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn shared(&self) -> StreamCore>; + fn shared(self) -> OwnedStream>; } -impl SharedStream for StreamCore { - fn shared(&self) -> StreamCore> { +impl> SharedStream for S { + fn shared(self) -> OwnedStream> { self.unary(Pipeline, "Shared", move |_, _| { move |input, output| { input.for_each(|time, data| { @@ -48,7 +48,7 @@ mod test { #[test] fn test_shared() { let output = crate::example(|scope| { - let shared = vec![Ok(0), Err(())].to_stream(scope).container::>().shared(); + let shared = vec![Ok(0), Err(())].to_stream(scope).container::>().shared().tee(); scope .concatenate([ shared.unary(Pipeline, "read shared 1", |_, _| { diff --git a/timely/src/dataflow/operators/core/reclock.rs b/timely/src/dataflow/operators/core/reclock.rs index e74bedbc4e..1f891a824f 100644 --- a/timely/src/dataflow/operators/core/reclock.rs +++ b/timely/src/dataflow/operators/core/reclock.rs @@ -2,12 +2,12 @@ use crate::{Container, Data}; use crate::order::PartialOrder; -use crate::dataflow::{Scope, StreamCore}; +use crate::dataflow::{Scope, OwnedStream, StreamLike}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::operator::Operator; /// Extension trait for reclocking a stream. -pub trait Reclock { +pub trait Reclock { /// Delays records until an input is observed on the `clock` input. /// /// The source stream is buffered until a record is seen on the clock input, @@ -35,7 +35,7 @@ pub trait Reclock { /// .map(|_| ()); /// /// // reclock the data. - /// data.reclock(&clock) + /// data.reclock(clock) /// .capture() /// }); /// @@ -45,11 +45,11 @@ pub trait Reclock { /// assert_eq!(extracted[1], (5, vec![4,5])); /// assert_eq!(extracted[2], (8, vec![6,7,8])); /// ``` - fn reclock(&self, clock: &StreamCore) -> Self; + fn reclock>(self, clock: CS) -> OwnedStream; } -impl Reclock for StreamCore { - fn reclock(&self, clock: &StreamCore) -> StreamCore { +impl> Reclock for S { + fn reclock>(self, clock: CS) -> OwnedStream { let mut stash = vec![]; diff --git a/timely/src/dataflow/operators/core/to_stream.rs b/timely/src/dataflow/operators/core/to_stream.rs index cfc6f429b7..4df6a2162c 100644 --- a/timely/src/dataflow/operators/core/to_stream.rs +++ b/timely/src/dataflow/operators/core/to_stream.rs @@ -3,9 +3,9 @@ use crate::container::{CapacityContainerBuilder, ContainerBuilder, SizableContainer, PushInto}; use crate::{Container, Data}; use crate::dataflow::operators::generic::operator::source; -use crate::dataflow::{StreamCore, Scope}; +use crate::dataflow::{Scope, OwnedStream}; -/// Converts to a timely [StreamCore], using a container builder. +/// Converts to a timely stream, using a container builder. pub trait ToStreamBuilder { /// Converts to a timely [StreamCore], using the supplied container builder type. /// @@ -28,11 +28,11 @@ pub trait ToStreamBuilder { /// /// assert_eq!(data1.extract(), data2.extract()); /// ``` - fn to_stream_with_builder(self, scope: &mut S) -> StreamCore; + fn to_stream_with_builder(self, scope: &mut S) -> OwnedStream; } impl ToStreamBuilder for I where CB: PushInto { - fn to_stream_with_builder(self, scope: &mut S) -> StreamCore { + fn to_stream_with_builder(self, scope: &mut S) -> OwnedStream { source::<_, CB, _, _>(scope, "ToStreamBuilder", |capability, info| { @@ -78,11 +78,11 @@ pub trait ToStream { /// /// assert_eq!(data1.extract(), data2.extract()); /// ``` - fn to_stream(self, scope: &mut S) -> StreamCore; + fn to_stream(self, scope: &mut S) -> OwnedStream; } impl ToStream for I where C: PushInto { - fn to_stream(self, scope: &mut S) -> StreamCore { + fn to_stream(self, scope: &mut S) -> OwnedStream { ToStreamBuilder::>::to_stream_with_builder(self, scope) } } diff --git a/timely/src/dataflow/operators/core/unordered_input.rs b/timely/src/dataflow/operators/core/unordered_input.rs index aacd89ca9f..8160ef906d 100644 --- a/timely/src/dataflow/operators/core/unordered_input.rs +++ b/timely/src/dataflow/operators/core/unordered_input.rs @@ -12,16 +12,16 @@ use crate::progress::{Operate, operate::SharedProgress, Timestamp}; use crate::progress::Source; use crate::progress::ChangeBatch; -use crate::dataflow::channels::pushers::{Counter, Tee}; +use crate::dataflow::channels::pushers::{Counter, PushOwned}; use crate::dataflow::channels::pushers::buffer::{Buffer as PushBuffer, AutoflushSession}; use crate::dataflow::operators::{ActivateCapability, Capability}; -use crate::dataflow::{Scope, StreamCore}; +use crate::dataflow::{OwnedStream, Scope}; /// Create a new `Stream` and `Handle` through which to supply input. pub trait UnorderedInput { - /// Create a new capability-based [StreamCore] and [UnorderedHandle] through which to supply input. This + /// Create a new capability-based stream and [UnorderedHandle] through which to supply input. This /// input supports multiple open epochs (timestamps) at the same time. /// /// The `new_unordered_input_core` method returns `((HandleCore, Capability), StreamCore)` where the `StreamCore` can be used @@ -76,13 +76,13 @@ pub trait UnorderedInput { /// assert_eq!(extract[i], (i, vec![i])); /// } /// ``` - fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), StreamCore); + fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream); } impl UnorderedInput for G { - fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), StreamCore) { + fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream) { - let (output, registrar) = Tee::::new(); + let (output, registrar) = PushOwned::::new(); let internal = Rc::new(RefCell::new(ChangeBatch::new())); // let produced = Rc::new(RefCell::new(ChangeBatch::new())); let cap = Capability::new(G::Timestamp::minimum(), internal.clone()); @@ -106,7 +106,7 @@ impl UnorderedInput for G { peers, }), index); - ((helper, cap), StreamCore::new(Source::new(index, 0), registrar, self.clone())) + ((helper, cap), OwnedStream::new(Source::new(index, 0), registrar, self.clone())) } } @@ -148,11 +148,11 @@ impl Operate for UnorderedOperator { /// A handle to an input [StreamCore], used to introduce data to a timely dataflow computation. #[derive(Debug)] pub struct UnorderedHandle { - buffer: PushBuffer>>, + buffer: PushBuffer>>, } impl UnorderedHandle { - fn new(pusher: Counter>) -> UnorderedHandle { + fn new(pusher: Counter>) -> UnorderedHandle { UnorderedHandle { buffer: PushBuffer::new(pusher), } @@ -160,7 +160,7 @@ impl UnorderedHandle { /// Allocates a new automatically flushing session based on the supplied capability. #[inline] - pub fn session_with_builder(&mut self, cap: ActivateCapability) -> ActivateOnDrop>>> { + pub fn session_with_builder(&mut self, cap: ActivateCapability) -> ActivateOnDrop>>> { ActivateOnDrop::new(self.buffer.autoflush_session_with_builder(cap.capability.clone()), cap.address.clone(), cap.activations.clone()) } } @@ -168,7 +168,7 @@ impl UnorderedHandle { impl UnorderedHandle> { /// Allocates a new automatically flushing session based on the supplied capability. #[inline] - pub fn session(&mut self, cap: ActivateCapability) -> ActivateOnDrop, Counter>>> { + pub fn session(&mut self, cap: ActivateCapability) -> ActivateOnDrop, Counter>>> { self.session_with_builder(cap) } } diff --git a/timely/src/dataflow/operators/count.rs b/timely/src/dataflow/operators/count.rs index 89e650b2b8..452d9275c9 100644 --- a/timely/src/dataflow/operators/count.rs +++ b/timely/src/dataflow/operators/count.rs @@ -3,11 +3,11 @@ use std::collections::HashMap; use crate::Data; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{Stream, Scope}; +use crate::dataflow::{OwnedStream, Scope, StreamLike}; use crate::dataflow::operators::generic::operator::Operator; /// Accumulates records within a timestamp. -pub trait Accumulate { +pub trait Accumulate: Sized { /// Accumulates records within a timestamp. /// /// # Examples @@ -25,7 +25,7 @@ pub trait Accumulate { /// let extracted = captured.extract(); /// assert_eq!(extracted, vec![(0, vec![45])]); /// ``` - fn accumulate(&self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> Stream; + fn accumulate(self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> OwnedStream>; /// Counts the number of records observed at each time. /// /// # Examples @@ -43,13 +43,13 @@ pub trait Accumulate { /// let extracted = captured.extract(); /// assert_eq!(extracted, vec![(0, vec![10])]); /// ``` - fn count(&self) -> Stream { + fn count(self) -> OwnedStream> { self.accumulate(0, |sum, data| *sum += data.len()) } } -impl Accumulate for Stream { - fn accumulate(&self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> Stream { +impl>> Accumulate for S { + fn accumulate(self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> OwnedStream> { let mut accums = HashMap::new(); self.unary_notify(Pipeline, "Accumulate", vec![], move |input, output, notificator| { diff --git a/timely/src/dataflow/operators/delay.rs b/timely/src/dataflow/operators/delay.rs index 8638bfd088..555acc7237 100644 --- a/timely/src/dataflow/operators/delay.rs +++ b/timely/src/dataflow/operators/delay.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use crate::Data; use crate::order::{PartialOrder, TotalOrder}; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{Stream, Scope}; +use crate::dataflow::{OwnedStream, StreamLike, Scope}; use crate::dataflow::operators::generic::operator::Operator; /// Methods to advance the timestamps of records or batches of records. @@ -36,7 +36,7 @@ pub trait Delay { /// }); /// }); /// ``` - fn delayG::Timestamp+'static>(&self, func: L) -> Self; + fn delayG::Timestamp+'static>(self, func: L) -> OwnedStream>; /// Advances the timestamp of records using a supplied function. /// @@ -63,7 +63,7 @@ pub trait Delay { /// }); /// }); /// ``` - fn delay_totalG::Timestamp+'static>(&self, func: L) -> Self + fn delay_totalG::Timestamp+'static>(self, func: L) -> OwnedStream> where G::Timestamp: TotalOrder; /// Advances the timestamp of batches of records using a supplied function. @@ -91,11 +91,11 @@ pub trait Delay { /// }); /// }); /// ``` - fn delay_batchG::Timestamp+'static>(&self, func: L) -> Self; + fn delay_batchG::Timestamp+'static>(self, func: L) -> OwnedStream>; } -impl Delay for Stream { - fn delayG::Timestamp+'static>(&self, mut func: L) -> Self { +impl>> Delay for S { + fn delayG::Timestamp+'static>(self, mut func: L) -> OwnedStream> { let mut elements = HashMap::new(); self.unary_notify(Pipeline, "Delay", vec![], move |input, output, notificator| { input.for_each(|time, data| { @@ -117,13 +117,13 @@ impl Delay for Stream { }) } - fn delay_totalG::Timestamp+'static>(&self, func: L) -> Self + fn delay_totalG::Timestamp+'static>(self, func: L) -> OwnedStream> where G::Timestamp: TotalOrder { self.delay(func) } - fn delay_batchG::Timestamp+'static>(&self, mut func: L) -> Self { + fn delay_batchG::Timestamp+'static>(self, mut func: L) -> OwnedStream> { let mut elements = HashMap::new(); self.unary_notify(Pipeline, "Delay", vec![], move |input, output, notificator| { input.for_each(|time, data| { diff --git a/timely/src/dataflow/operators/filter.rs b/timely/src/dataflow/operators/filter.rs index c2486e9eb9..d4c89e6106 100644 --- a/timely/src/dataflow/operators/filter.rs +++ b/timely/src/dataflow/operators/filter.rs @@ -2,11 +2,11 @@ use crate::Data; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{Stream, Scope}; +use crate::dataflow::{OwnedStream, StreamLike, Scope}; use crate::dataflow::operators::generic::operator::Operator; /// Extension trait for filtering. -pub trait Filter { +pub trait Filter { /// Returns a new instance of `self` containing only records satisfying `predicate`. /// /// # Examples @@ -19,11 +19,11 @@ pub trait Filter { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn filterbool+'static>(&self, predicate: P) -> Self; + fn filterbool+'static>(self, predicate: P) -> OwnedStream>; } -impl Filter for Stream { - fn filterbool+'static>(&self, mut predicate: P) -> Stream { +impl>> Filter for S { + fn filterbool+'static>(self, mut predicate: P) -> OwnedStream> { self.unary(Pipeline, "Filter", move |_,_| move |input, output| { input.for_each(|time, data| { data.retain(|x| predicate(x)); diff --git a/timely/src/dataflow/operators/flow_controlled.rs b/timely/src/dataflow/operators/flow_controlled.rs index 0d91ef7f95..13d7bc2f1c 100644 --- a/timely/src/dataflow/operators/flow_controlled.rs +++ b/timely/src/dataflow/operators/flow_controlled.rs @@ -5,7 +5,7 @@ use crate::order::{PartialOrder, TotalOrder}; use crate::progress::timestamp::Timestamp; use crate::dataflow::operators::generic::operator::source; use crate::dataflow::operators::probe::Handle; -use crate::dataflow::{Stream, Scope}; +use crate::dataflow::{OwnedStream, Scope}; /// Output of the input reading function for iterator_source. pub struct IteratorSourceInput, I: IntoIterator> { @@ -81,7 +81,7 @@ pub fn iterator_source< name: &str, mut input_f: F, probe: Handle, - ) -> Stream where G::Timestamp: TotalOrder { + ) -> OwnedStream> where G::Timestamp: TotalOrder { let mut target = G::Timestamp::minimum(); source(scope, name, |cap, info| { diff --git a/timely/src/dataflow/operators/generic/builder_raw.rs b/timely/src/dataflow/operators/generic/builder_raw.rs index 8ba131612a..a557eb0f9a 100644 --- a/timely/src/dataflow/operators/generic/builder_raw.rs +++ b/timely/src/dataflow/operators/generic/builder_raw.rs @@ -14,9 +14,9 @@ use crate::progress::{Source, Target}; use crate::progress::{Timestamp, Operate, operate::SharedProgress, Antichain}; use crate::Container; -use crate::dataflow::{StreamCore, Scope}; -use crate::dataflow::channels::pushers::Tee; +use crate::dataflow::{OwnedStream, StreamLike, Scope}; use crate::dataflow::channels::pact::ParallelizationContract; +use crate::dataflow::channels::pushers::PushOwned; use crate::dataflow::operators::generic::operator_info::OperatorInfo; /// Contains type-free information about the operator properties. @@ -104,7 +104,7 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input(&mut self, stream: &StreamCore, pact: P) -> P::Puller + pub fn new_input>(&mut self, stream: S, pact: P) -> P::Puller where P: ParallelizationContract { let connection = vec![Antichain::from_elem(Default::default()); self.shape.outputs]; @@ -112,7 +112,7 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input_connection(&mut self, stream: &StreamCore, pact: P, connection: Vec::Summary>>) -> P::Puller + pub fn new_input_connection>(&mut self, stream: S, pact: P, connection: Vec::Summary>>) -> P::Puller where P: ParallelizationContract { @@ -130,18 +130,18 @@ impl OperatorBuilder { } /// Adds a new output to a generic operator builder, returning the `Push` implementor to use. - pub fn new_output(&mut self) -> (Tee, StreamCore) { + pub fn new_output(&mut self) -> (PushOwned, OwnedStream) { let connection = vec![Antichain::from_elem(Default::default()); self.shape.inputs]; self.new_output_connection(connection) } /// Adds a new output to a generic operator builder, returning the `Push` implementor to use. - pub fn new_output_connection(&mut self, connection: Vec::Summary>>) -> (Tee, StreamCore) { + pub fn new_output_connection(&mut self, connection: Vec::Summary>>) -> (PushOwned, OwnedStream) { - let (targets, registrar) = Tee::::new(); + let (target, registrar) = PushOwned::new(); let source = Source::new(self.index, self.shape.outputs); - let stream = StreamCore::new(source, registrar, self.scope.clone()); + let stream = OwnedStream::new(source, registrar, self.scope.clone()); self.shape.outputs += 1; assert_eq!(self.shape.inputs, connection.len()); @@ -149,7 +149,7 @@ impl OperatorBuilder { summary.push(entry); } - (targets, stream) + (target, stream) } /// Creates an operator implementation from supplied logic constructor. diff --git a/timely/src/dataflow/operators/generic/builder_rc.rs b/timely/src/dataflow/operators/generic/builder_rc.rs index 752b3be2e7..8b09cfc2cb 100644 --- a/timely/src/dataflow/operators/generic/builder_rc.rs +++ b/timely/src/dataflow/operators/generic/builder_rc.rs @@ -10,12 +10,12 @@ use crate::progress::frontier::{Antichain, MutableAntichain}; use crate::Container; use crate::container::ContainerBuilder; -use crate::dataflow::{Scope, StreamCore}; -use crate::dataflow::channels::pushers::Tee; +use crate::dataflow::{Scope, OwnedStream, StreamLike}; use crate::dataflow::channels::pushers::Counter as PushCounter; use crate::dataflow::channels::pushers::buffer::Buffer as PushBuffer; use crate::dataflow::channels::pact::ParallelizationContract; use crate::dataflow::channels::pullers::Counter as PullCounter; +use crate::dataflow::channels::pushers::PushOwned; use crate::dataflow::operators::capability::Capability; use crate::dataflow::operators::generic::handles::{InputHandleCore, new_input_handle, OutputWrapper}; use crate::dataflow::operators::generic::operator_info::OperatorInfo; @@ -60,10 +60,10 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input(&mut self, stream: &StreamCore, pact: P) -> InputHandleCore + pub fn new_input>(&mut self, stream: S, pact: P) -> InputHandleCore where - P: ParallelizationContract { - + P: ParallelizationContract + { let connection = (0..self.builder.shape().outputs()).map(|_| Antichain::from_elem(Default::default())).collect(); self.new_input_connection(stream, pact, connection) } @@ -76,7 +76,7 @@ impl OperatorBuilder { /// /// Commonly the connections are either the unit summary, indicating the same timestamp might be produced as output, or an empty /// antichain indicating that there is no connection from the input to the output. - pub fn new_input_connection(&mut self, stream: &StreamCore, pact: P, connection: Vec::Summary>>) -> InputHandleCore + pub fn new_input_connection>(&mut self, stream: S, pact: P, connection: Vec::Summary>>) -> InputHandleCore where P: ParallelizationContract { @@ -93,8 +93,8 @@ impl OperatorBuilder { } /// Adds a new output to a generic operator builder, returning the `Push` implementor to use. - pub fn new_output(&mut self) -> (OutputWrapper>, StreamCore) { - let connection = (0..self.builder.shape().inputs()).map(|_| Antichain::from_elem(Default::default())).collect(); + pub fn new_output(&mut self) -> (OutputWrapper>, OwnedStream) { + let connection = vec![Antichain::from_elem(Default::default()); self.builder.shape().inputs()]; self.new_output_connection(connection) } @@ -110,8 +110,8 @@ impl OperatorBuilder { &mut self, connection: Vec::Summary>> ) -> ( - OutputWrapper>, - StreamCore + OutputWrapper>, + OwnedStream ) { let (tee, stream) = self.builder.new_output_connection(connection.clone()); diff --git a/timely/src/dataflow/operators/generic/handles.rs b/timely/src/dataflow/operators/generic/handles.rs index 47811a6590..b147d8bbf9 100644 --- a/timely/src/dataflow/operators/generic/handles.rs +++ b/timely/src/dataflow/operators/generic/handles.rs @@ -15,7 +15,7 @@ use crate::dataflow::channels::pushers::Counter as PushCounter; use crate::dataflow::channels::pushers::buffer::{Buffer, Session}; use crate::dataflow::channels::Message; use crate::communication::{Push, Pull}; -use crate::{Container, Data}; +use crate::Container; use crate::container::{ContainerBuilder, CapacityContainerBuilder}; use crate::logging::TimelyLogger as Logger; @@ -235,7 +235,7 @@ impl<'a, T: Timestamp, CB: ContainerBuilder, P: Push>> } } -impl<'a, T: Timestamp, C: Container + Data, P: Push>> OutputHandleCore<'a, T, CapacityContainerBuilder, P> { +impl<'a, T: Timestamp, C: Container + 'static, P: Push>> OutputHandleCore<'a, T, CapacityContainerBuilder, P> { /// Obtains a session that can send data at the timestamp associated with capability `cap`. /// /// In order to send data at a future timestamp, obtain a capability for the new timestamp diff --git a/timely/src/dataflow/operators/generic/notificator.rs b/timely/src/dataflow/operators/generic/notificator.rs index 905fdef4fd..900d136b50 100644 --- a/timely/src/dataflow/operators/generic/notificator.rs +++ b/timely/src/dataflow/operators/generic/notificator.rs @@ -195,7 +195,7 @@ fn notificator_delivers_notifications_in_topo_order() { /// let (mut in1, mut in2) = worker.dataflow::(|scope| { /// let (in1_handle, in1) = scope.new_input(); /// let (in2_handle, in2) = scope.new_input(); -/// in1.binary_frontier(&in2, Pipeline, Pipeline, "example", |mut _default_cap, _info| { +/// in1.binary_frontier(in2, Pipeline, Pipeline, "example", |mut _default_cap, _info| { /// let mut notificator = FrontierNotificator::default(); /// let mut stash = HashMap::new(); /// move |input1, input2, output| { diff --git a/timely/src/dataflow/operators/generic/operator.rs b/timely/src/dataflow/operators/generic/operator.rs index 247cb5258c..6ff529e105 100644 --- a/timely/src/dataflow/operators/generic/operator.rs +++ b/timely/src/dataflow/operators/generic/operator.rs @@ -1,13 +1,13 @@ //! Methods to construct generic streaming and blocking unary operators. -use crate::dataflow::channels::pushers::Tee; +use crate::dataflow::channels::pushers::PushOwned; use crate::dataflow::channels::pact::ParallelizationContract; use crate::dataflow::operators::generic::handles::{InputHandleCore, FrontieredInputHandleCore, OutputHandleCore}; use crate::dataflow::operators::capability::Capability; -use crate::dataflow::{Scope, StreamCore}; +use crate::dataflow::{Scope, OwnedStream, StreamLike}; use super::builder_rc::OperatorBuilder; use crate::dataflow::operators::generic::OperatorInfo; @@ -55,12 +55,12 @@ pub trait Operator { /// }); /// } /// ``` - fn unary_frontier(&self, pact: P, name: &str, constructor: B) -> StreamCore + fn unary_frontier(self, pact: P, name: &str, constructor: B) -> OwnedStream where CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut FrontieredInputHandleCore, - &mut OutputHandleCore>)+'static, + &mut OutputHandleCore>)+'static, P: ParallelizationContract; /// Creates a new dataflow operator that partitions its input stream by a parallelization @@ -92,10 +92,10 @@ pub trait Operator { /// ``` fn unary_notify, - &mut OutputHandleCore>, + &mut OutputHandleCore>, &mut Notificator)+'static, P: ParallelizationContract> - (&self, pact: P, name: &str, init: impl IntoIterator, logic: L) -> StreamCore; + (self, pact: P, name: &str, init: impl IntoIterator, logic: L) -> OwnedStream; /// Creates a new dataflow operator that partitions its input stream by a parallelization /// strategy `pact`, and repeatedly invokes `logic`, the function returned by the function passed as `constructor`. @@ -123,12 +123,12 @@ pub trait Operator { /// }); /// }); /// ``` - fn unary(&self, pact: P, name: &str, constructor: B) -> StreamCore + fn unary(self, pact: P, name: &str, constructor: B) -> OwnedStream where CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut InputHandleCore, - &mut OutputHandleCore>)+'static, + &mut OutputHandleCore>)+'static, P: ParallelizationContract; /// Creates a new dataflow operator that partitions its input streams by a parallelization @@ -146,7 +146,7 @@ pub trait Operator { /// let (mut in1, mut in2) = worker.dataflow::(|scope| { /// let (in1_handle, in1) = scope.new_input(); /// let (in2_handle, in2) = scope.new_input(); - /// in1.binary_frontier(&in2, Pipeline, Pipeline, "example", |mut _default_cap, _info| { + /// in1.binary_frontier(in2, Pipeline, Pipeline, "example", |mut _default_cap, _info| { /// let mut notificator = FrontierNotificator::default(); /// let mut stash = HashMap::new(); /// move |input1, input2, output| { @@ -179,14 +179,15 @@ pub trait Operator { /// } /// }).unwrap(); /// ``` - fn binary_frontier(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore + fn binary_frontier(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream where C2: Container + Data, CB: ContainerBuilder, + O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut FrontieredInputHandleCore, &mut FrontieredInputHandleCore, - &mut OutputHandleCore>)+'static, + &mut OutputHandleCore>)+'static, P1: ParallelizationContract, P2: ParallelizationContract; @@ -206,7 +207,7 @@ pub trait Operator { /// let (in1_handle, in1) = scope.new_input(); /// let (in2_handle, in2) = scope.new_input(); /// - /// in1.binary_notify(&in2, Pipeline, Pipeline, "example", None, move |input1, input2, output, notificator| { + /// in1.binary_notify(in2, Pipeline, Pipeline, "example", None, move |input1, input2, output, notificator| { /// input1.for_each(|time, data| { /// output.session(&time).give_container(data); /// notificator.notify_at(time.retain()); @@ -233,13 +234,14 @@ pub trait Operator { /// ``` fn binary_notify, L: FnMut(&mut InputHandleCore, &mut InputHandleCore, - &mut OutputHandleCore>, + &mut OutputHandleCore>, &mut Notificator)+'static, P1: ParallelizationContract, P2: ParallelizationContract> - (&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, init: impl IntoIterator, logic: L) -> StreamCore; + (self, other: O, pact1: P1, pact2: P2, name: &str, init: impl IntoIterator, logic: L) -> OwnedStream; /// Creates a new dataflow operator that partitions its input streams by a parallelization /// strategy `pact`, and repeatedly invokes `logic`, the function returned by the function passed as `constructor`. @@ -255,7 +257,7 @@ pub trait Operator { /// timely::example(|scope| { /// let stream2 = (0u64..10).to_stream(scope); /// (0u64..10).to_stream(scope) - /// .binary(&stream2, Pipeline, Pipeline, "example", |default_cap, _info| { + /// .binary(stream2, Pipeline, Pipeline, "example", |default_cap, _info| { /// let mut cap = Some(default_cap.delayed(&12)); /// move |input1, input2, output| { /// if let Some(ref c) = cap.take() { @@ -271,14 +273,15 @@ pub trait Operator { /// }).inspect(|x| println!("{:?}", x)); /// }); /// ``` - fn binary(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore + fn binary(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream where C2: Container + Data, CB: ContainerBuilder, + O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut InputHandleCore, &mut InputHandleCore, - &mut OutputHandleCore>)+'static, + &mut OutputHandleCore>)+'static, P1: ParallelizationContract, P2: ParallelizationContract; @@ -305,20 +308,20 @@ pub trait Operator { /// }); /// }); /// ``` - fn sink(&self, pact: P, name: &str, logic: L) + fn sink(self, pact: P, name: &str, logic: L) where L: FnMut(&mut FrontieredInputHandleCore)+'static, P: ParallelizationContract; } -impl Operator for StreamCore { +impl> Operator for S { - fn unary_frontier(&self, pact: P, name: &str, constructor: B) -> StreamCore + fn unary_frontier(self, pact: P, name: &str, constructor: B) -> OwnedStream where CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut FrontieredInputHandleCore, - &mut OutputHandleCore>)+'static, + &mut OutputHandleCore>)+'static, P: ParallelizationContract { let mut builder = OperatorBuilder::new(name.to_owned(), self.scope()); @@ -343,18 +346,18 @@ impl Operator for StreamCore { fn unary_notify, - &mut OutputHandleCore>, + &mut OutputHandleCore>, &mut Notificator)+'static, P: ParallelizationContract> - (&self, pact: P, name: &str, init: impl IntoIterator, mut logic: L) -> StreamCore { + (self, pact: P, name: &str, init: impl IntoIterator, mut logic: L) -> OwnedStream { + let logging = self.scope().logging(); self.unary_frontier(pact, name, move |capability, _info| { let mut notificator = FrontierNotificator::default(); for time in init { notificator.notify_at(capability.delayed(&time)); } - let logging = self.scope().logging(); move |input, output| { let frontier = &[input.frontier()]; let notificator = &mut Notificator::new(frontier, &mut notificator, &logging); @@ -363,12 +366,12 @@ impl Operator for StreamCore { }) } - fn unary(&self, pact: P, name: &str, constructor: B) -> StreamCore + fn unary(self, pact: P, name: &str, constructor: B) -> OwnedStream where CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut InputHandleCore, - &mut OutputHandleCore>)+'static, + &mut OutputHandleCore>)+'static, P: ParallelizationContract { let mut builder = OperatorBuilder::new(name.to_owned(), self.scope()); @@ -391,14 +394,15 @@ impl Operator for StreamCore { stream } - fn binary_frontier(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore + fn binary_frontier(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream where C2: Container + Data, CB: ContainerBuilder, + O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut FrontieredInputHandleCore, &mut FrontieredInputHandleCore, - &mut OutputHandleCore>)+'static, + &mut OutputHandleCore>)+'static, P1: ParallelizationContract, P2: ParallelizationContract { @@ -426,21 +430,22 @@ impl Operator for StreamCore { fn binary_notify, L: FnMut(&mut InputHandleCore, &mut InputHandleCore, - &mut OutputHandleCore>, + &mut OutputHandleCore>, &mut Notificator)+'static, P1: ParallelizationContract, P2: ParallelizationContract> - (&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, init: impl IntoIterator, mut logic: L) -> StreamCore { + (self, other: O, pact1: P1, pact2: P2, name: &str, init: impl IntoIterator, mut logic: L) -> OwnedStream { + let logging = self.scope().logging(); self.binary_frontier(other, pact1, pact2, name, |capability, _info| { let mut notificator = FrontierNotificator::default(); for time in init { notificator.notify_at(capability.delayed(&time)); } - let logging = self.scope().logging(); move |input1, input2, output| { let frontiers = &[input1.frontier(), input2.frontier()]; let notificator = &mut Notificator::new(frontiers, &mut notificator, &logging); @@ -451,14 +456,15 @@ impl Operator for StreamCore { } - fn binary(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore + fn binary(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream where C2: Container + Data, CB: ContainerBuilder, + O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut InputHandleCore, &mut InputHandleCore, - &mut OutputHandleCore>)+'static, + &mut OutputHandleCore>)+'static, P1: ParallelizationContract, P2: ParallelizationContract { @@ -483,7 +489,7 @@ impl Operator for StreamCore { stream } - fn sink(&self, pact: P, name: &str, mut logic: L) + fn sink(self, pact: P, name: &str, mut logic: L) where L: FnMut(&mut FrontieredInputHandleCore)+'static, P: ParallelizationContract { @@ -542,11 +548,11 @@ impl Operator for StreamCore { /// .inspect(|x| println!("number: {:?}", x)); /// }); /// ``` -pub fn source(scope: &G, name: &str, constructor: B) -> StreamCore +pub fn source(scope: &G, name: &str, constructor: B) -> OwnedStream where CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, - L: FnMut(&mut OutputHandleCore>)+'static { + L: FnMut(&mut OutputHandleCore>)+'static { let mut builder = OperatorBuilder::new(name.to_owned(), scope.clone()); let operator_info = builder.operator_info(); @@ -586,7 +592,7 @@ where /// /// }); /// ``` -pub fn empty(scope: &G) -> StreamCore { +pub fn empty(scope: &G) -> OwnedStream { source::<_, CapacityContainerBuilder, _, _>(scope, "Empty", |_capability, _info| |_output| { // drop capability, do nothing }) diff --git a/timely/src/dataflow/operators/input.rs b/timely/src/dataflow/operators/input.rs index 30d243ab4f..5d7bb9b4cc 100644 --- a/timely/src/dataflow/operators/input.rs +++ b/timely/src/dataflow/operators/input.rs @@ -2,7 +2,7 @@ use crate::Data; use crate::container::CapacityContainerBuilder; -use crate::dataflow::{Stream, ScopeParent, Scope}; +use crate::dataflow::{ScopeParent, Scope, OwnedStream}; use crate::dataflow::operators::core::{Input as InputCore}; // TODO : This is an exogenous input, but it would be nice to wrap a Subgraph in something @@ -47,7 +47,7 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn new_input(&mut self) -> (Handle<::Timestamp, D>, Stream); + fn new_input(&mut self) -> (Handle<::Timestamp, D>, OwnedStream>); /// Create a new stream from a supplied interactive handle. /// @@ -79,16 +79,16 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> Stream; + fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> OwnedStream>; } use crate::order::TotalOrder; impl Input for G where ::Timestamp: TotalOrder { - fn new_input(&mut self) -> (Handle<::Timestamp, D>, Stream) { + fn new_input(&mut self) -> (Handle<::Timestamp, D>, OwnedStream>) { InputCore::new_input(self) } - fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> Stream { + fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> OwnedStream> { InputCore::input_from(self, handle) } } diff --git a/timely/src/dataflow/operators/map.rs b/timely/src/dataflow/operators/map.rs index 5adab6e108..2ff9b81d7b 100644 --- a/timely/src/dataflow/operators/map.rs +++ b/timely/src/dataflow/operators/map.rs @@ -1,13 +1,13 @@ //! Extension methods for `Stream` based on record-by-record transformation. use crate::Data; -use crate::dataflow::{Stream, Scope}; +use crate::dataflow::{OwnedStream, StreamLike, Scope}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::operator::Operator; use crate::dataflow::operators::core::{Map as MapCore}; /// Extension trait for `Stream`. -pub trait Map { +pub trait Map : Sized { /// Consumes each element of the stream and yields a new element. /// /// # Examples @@ -20,7 +20,7 @@ pub trait Map { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn mapD2+'static>(&self, mut logic: L) -> Stream { + fn mapD2+'static>(self, mut logic: L) -> OwnedStream> { self.flat_map(move |x| std::iter::once(logic(x))) } /// Updates each element of the stream and yields the element, re-using memory where possible. @@ -35,7 +35,7 @@ pub trait Map { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn map_in_place(&self, logic: L) -> Stream; + fn map_in_place(self, logic: L) -> OwnedStream>; /// Consumes each element of the stream and yields some number of new elements. /// /// # Examples @@ -48,11 +48,11 @@ pub trait Map { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn flat_mapI+'static>(&self, logic: L) -> Stream where I::Item: Data; + fn flat_mapI+'static>(self, logic: L) -> OwnedStream> where I::Item: Data; } -impl Map for Stream { - fn map_in_place(&self, mut logic: L) -> Stream { +impl>> Map for S { + fn map_in_place(self, mut logic: L) -> OwnedStream> { self.unary(Pipeline, "MapInPlace", move |_,_| move |input, output| { input.for_each(|time, data| { for datum in data.iter_mut() { logic(datum); } @@ -63,7 +63,7 @@ impl Map for Stream { // TODO : This would be more robust if it captured an iterator and then pulled an appropriate // TODO : number of elements from the iterator. This would allow iterators that produce many // TODO : records without taking arbitrarily long and arbitrarily much memory. - fn flat_mapI+'static>(&self, logic: L) -> Stream where I::Item: Data { + fn flat_mapI+'static>(self, logic: L) -> OwnedStream> where I::Item: Data { MapCore::flat_map(self, logic) } } diff --git a/timely/src/dataflow/operators/partition.rs b/timely/src/dataflow/operators/partition.rs index 39fc427cca..8661e15907 100644 --- a/timely/src/dataflow/operators/partition.rs +++ b/timely/src/dataflow/operators/partition.rs @@ -2,7 +2,7 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; -use crate::dataflow::{Scope, Stream}; +use crate::dataflow::{Scope, OwnedStream, StreamLike}; use crate::Data; /// Partition a stream of records into multiple streams. @@ -16,17 +16,24 @@ pub trait Partition (u64, D2)> { /// timely::example(|scope| { /// let streams = (0..10).to_stream(scope) /// .partition(3, |x| (x % 3, x)); - /// - /// streams[0].inspect(|x| println!("seen 0: {:?}", x)); - /// streams[1].inspect(|x| println!("seen 1: {:?}", x)); - /// streams[2].inspect(|x| println!("seen 2: {:?}", x)); + /// for (index, stream) in streams.into_iter().enumerate() { + /// assert!(index < 3); + /// stream.inspect(move |x| println!("seen {index}: {:?}", x)); + /// } /// }); /// ``` - fn partition(&self, parts: u64, route: F) -> Vec>; + fn partition(self, parts: u64, route: F) -> Vec>>; } -impl(u64, D2)+'static> Partition for Stream { - fn partition(&self, parts: u64, route: F) -> Vec> { +impl Partition for S +where + G: Scope, + D: Data, + D2: Data, + F: Fn(D) -> (u64, D2) + 'static, + S: StreamLike>, +{ + fn partition(self, parts: u64, route: F) -> Vec>> { let mut builder = OperatorBuilder::new("Partition".to_owned(), self.scope()); let mut input = builder.new_input(self, Pipeline); diff --git a/timely/src/dataflow/operators/result.rs b/timely/src/dataflow/operators/result.rs index 6dddf8a710..4a1ee3ec37 100644 --- a/timely/src/dataflow/operators/result.rs +++ b/timely/src/dataflow/operators/result.rs @@ -2,10 +2,10 @@ use crate::Data; use crate::dataflow::operators::Map; -use crate::dataflow::{Scope, Stream}; +use crate::dataflow::{Scope, StreamLike, OwnedStream}; /// Extension trait for `Stream`. -pub trait ResultStream { +pub trait ResultStream { /// Returns a new instance of `self` containing only `ok` records. /// /// # Examples @@ -18,7 +18,7 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn ok(&self) -> Stream; + fn ok(self) -> OwnedStream>; /// Returns a new instance of `self` containing only `err` records. /// @@ -32,7 +32,7 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn err(&self) -> Stream; + fn err(self) -> OwnedStream>; /// Returns a new instance of `self` applying `logic` on all `Ok` records. /// @@ -46,7 +46,7 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn map_ok T2 + 'static>(&self, logic: L) -> Stream>; + fn map_ok T2 + 'static>(self, logic: L) -> OwnedStream>>; /// Returns a new instance of `self` applying `logic` on all `Err` records. /// @@ -60,7 +60,7 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn map_err E2 + 'static>(&self, logic: L) -> Stream>; + fn map_err E2 + 'static>(self, logic: L) -> OwnedStream>>; /// Returns a new instance of `self` applying `logic` on all `Ok` records, passes through `Err` /// records. @@ -76,9 +76,9 @@ pub trait ResultStream { /// }); /// ``` fn and_then Result + 'static>( - &self, + self, logic: L, - ) -> Stream>; + ) -> OwnedStream>>; /// Returns a new instance of `self` applying `logic` on all `Ok` records. /// @@ -92,31 +92,31 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn unwrap_or_else T + 'static>(&self, logic: L) -> Stream; + fn unwrap_or_else T + 'static>(self, logic: L) -> OwnedStream>; } -impl ResultStream for Stream> { - fn ok(&self) -> Stream { +impl>>> ResultStream for S { + fn ok(self) -> OwnedStream> { self.flat_map(Result::ok) } - fn err(&self) -> Stream { + fn err(self) -> OwnedStream> { self.flat_map(Result::err) } - fn map_ok T2 + 'static>(&self, mut logic: L) -> Stream> { + fn map_ok T2 + 'static>(self, mut logic: L) -> OwnedStream>> { self.map(move |r| r.map(&mut logic)) } - fn map_err E2 + 'static>(&self, mut logic: L) -> Stream> { + fn map_err E2 + 'static>(self, mut logic: L) -> OwnedStream>> { self.map(move |r| r.map_err(&mut logic)) } - fn and_then Result + 'static>(&self, mut logic: L) -> Stream> { + fn and_then Result + 'static>(self, mut logic: L) -> OwnedStream>> { self.map(move |r| r.and_then(&mut logic)) } - fn unwrap_or_else T + 'static>(&self, mut logic: L) -> Stream { + fn unwrap_or_else T + 'static>(self, mut logic: L) -> OwnedStream> { self.map(move |r| r.unwrap_or_else(&mut logic)) } } diff --git a/timely/src/dataflow/operators/to_stream.rs b/timely/src/dataflow/operators/to_stream.rs index a85677dc05..53aad3f2f4 100644 --- a/timely/src/dataflow/operators/to_stream.rs +++ b/timely/src/dataflow/operators/to_stream.rs @@ -1,11 +1,11 @@ //! Conversion to the `Stream` type from iterators. use crate::Data; -use crate::dataflow::{Stream, Scope}; +use crate::dataflow::{Scope, OwnedStream}; use crate::dataflow::operators::core::{ToStream as ToStreamCore}; /// Converts to a timely `Stream`. -pub trait ToStream { +pub trait ToStream { /// Converts to a timely `Stream`. /// /// # Examples @@ -22,11 +22,11 @@ pub trait ToStream { /// /// assert_eq!(data1.extract(), data2.extract()); /// ``` - fn to_stream(self, scope: &mut S) -> Stream; + fn to_stream(self, scope: &mut S) -> OwnedStream>; } impl ToStream for I where I::Item: Data { - fn to_stream(self, scope: &mut S) -> Stream { + fn to_stream(self, scope: &mut S) -> OwnedStream> { ToStreamCore::to_stream(self, scope) } } diff --git a/timely/src/dataflow/operators/unordered_input.rs b/timely/src/dataflow/operators/unordered_input.rs index 76fb4329da..9572949c83 100644 --- a/timely/src/dataflow/operators/unordered_input.rs +++ b/timely/src/dataflow/operators/unordered_input.rs @@ -5,7 +5,7 @@ use crate::Data; use crate::container::CapacityContainerBuilder; use crate::dataflow::operators::{ActivateCapability}; use crate::dataflow::operators::core::{UnorderedInput as UnorderedInputCore, UnorderedHandle as UnorderedHandleCore}; -use crate::dataflow::{Stream, Scope}; +use crate::dataflow::{Scope, OwnedStream}; /// Create a new `Stream` and `Handle` through which to supply input. pub trait UnorderedInput { @@ -62,12 +62,12 @@ pub trait UnorderedInput { /// assert_eq!(extract[i], (i, vec![i])); /// } /// ``` - fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), Stream); + fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream>); } impl UnorderedInput for G { - fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), Stream) { + fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream>) { UnorderedInputCore::new_unordered_input(self) } } diff --git a/timely/src/dataflow/stream.rs b/timely/src/dataflow/stream.rs index d071cac450..9a20d7a722 100644 --- a/timely/src/dataflow/stream.rs +++ b/timely/src/dataflow/stream.rs @@ -12,8 +12,20 @@ use crate::dataflow::channels::pushers::tee::TeeHelper; use crate::dataflow::channels::Message; use std::fmt::{self, Debug}; use crate::Container; +use crate::dataflow::channels::pushers::{PushOwned, Tee}; -// use dataflow::scopes::root::loggers::CHANNELS_Q; +/// Common behavior for all streams. Streams belong to a scope and carry data. +/// +/// The main purpose of this is to allow connecting different stream kinds to a pusher. +pub trait StreamLike { + /// Connects the stream to a destination. + /// + /// The destination is described both by a `Target`, for progress tracking information, and a `P: Push` where the + /// records should actually be sent. The identifier is unique to the edge and is used only for logging purposes. + fn connect_to>+'static>(self, target: Target, pusher: P, identifier: usize); + /// The scope containing the stream. + fn scope(&self) -> S; +} /// Abstraction of a stream of `C: Container` records timestamped with `S::Timestamp`. /// @@ -44,16 +56,39 @@ impl Clone for StreamCore { } } +/// A stream that owns a single pusher. +pub struct OwnedStream { + /// The progress identifier of the stream's data source. + name: Source, + /// The `Scope` containing the stream. + scope: S, + /// The single pusher interested in the stream's output, if any. + port: PushOwned, +} + +impl OwnedStream { + /// Allocates an `OwnedStream` from a supplied `Source` name and rendezvous point within a scope. + pub fn new(name: Source, port: PushOwned, scope: S) -> Self { + Self { name, port, scope } + } + + /// Convert the stream into a `StreamCore` that can be cloned. Requires elements to be `Clone`. + /// Consumes this stream. + pub fn tee(self) -> StreamCore where C: Clone { + let (target, registrar) = Tee::new(); + self.port.set(target); + StreamCore::new(self.name, registrar, self.scope) + } + + /// Allows the assertion of a container type, for the benefit of type inference. + pub fn container(self) -> >::Stream where Self: AsStream { self.as_stream() } +} + /// A stream batching data in vectors. pub type Stream = StreamCore>; -impl StreamCore { - /// Connects the stream to a destination. - /// - /// The destination is described both by a `Target`, for progress tracking information, and a `P: Push` where the - /// records should actually be sent. The identifier is unique to the edge and is used only for logging purposes. - pub fn connect_to>+'static>(&self, target: Target, pusher: P, identifier: usize) { - +impl StreamLike for &StreamCore { + fn connect_to>+'static>(self, target: Target, pusher: P, identifier: usize) { let mut logging = self.scope().logging(); logging.as_mut().map(|l| l.log(crate::logging::ChannelsEvent { id: identifier, @@ -65,6 +100,32 @@ impl StreamCore { self.scope.add_edge(self.name, target); self.ports.add_pusher(pusher); } + + fn scope(&self) -> S { + self.scope.clone() + } +} + +impl StreamLike for OwnedStream { + fn connect_to> + 'static>(self, target: Target, pusher: P, identifier: usize) { + let mut logging = self.scope().logging(); + logging.as_mut().map(|l| l.log(crate::logging::ChannelsEvent { + id: identifier, + scope_addr: self.scope.addr().to_vec(), + source: (self.name.node, self.name.port), + target: (target.node, target.port), + })); + + self.scope.add_edge(self.name, target); + self.port.set(pusher); + } + + fn scope(&self) -> S { + self.scope.clone() + } +} + +impl StreamCore { /// Allocates a `Stream` from a supplied `Source` name and rendezvous point. pub fn new(source: Source, output: TeeHelper, scope: S) -> Self { Self { name: source, ports: output, scope } @@ -75,16 +136,24 @@ impl StreamCore { pub fn scope(&self) -> S { self.scope.clone() } /// Allows the assertion of a container type, for the benefit of type inference. - pub fn container(self) -> StreamCore where Self: AsStream { self.as_stream() } + pub fn container(self) -> >::Stream where Self: AsStream { self.as_stream() } } /// A type that can be translated to a [StreamCore]. pub trait AsStream { - /// Translate `self` to a [StreamCore]. - fn as_stream(self) -> StreamCore; + /// The type of the stream. + type Stream; + /// Translate `self` to a stream. + fn as_stream(self) -> Self::Stream; } impl AsStream for StreamCore { + type Stream = StreamCore; + fn as_stream(self) -> Self { self } +} + +impl AsStream for OwnedStream { + type Stream = OwnedStream; fn as_stream(self) -> Self { self } } @@ -99,3 +168,28 @@ where .finish() } } + +#[cfg(test)] +mod tests { + use crate::dataflow::channels::pact::Pipeline; + use crate::dataflow::operators::{Operator, ToStream}; + + #[derive(Debug, Eq, PartialEq)] + struct NotClone; + + #[test] + fn test_non_clone_stream() { + crate::example(|scope| { + let _ = [NotClone] + .to_stream(scope) + .sink(Pipeline, "check non-clone", |input| { + while let Some((_time, data)) = input.next() { + for datum in data.drain(..) { + assert_eq!(datum, &NotClone); + } + } + }); + }); + } + +} From 6300718b6f4d833e6e85f0c6c02f9435cdc034fd Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Mon, 9 Dec 2024 21:56:53 -0500 Subject: [PATCH 2/5] Remove Data trait Signed-off-by: Moritz Hoffmann --- timely/src/dataflow/channels/pact.rs | 3 +-- .../src/dataflow/channels/pushers/exchange.rs | 4 ++-- timely/src/dataflow/channels/pushers/owned.rs | 3 +-- timely/src/dataflow/channels/pushers/tee.rs | 4 ++-- .../dataflow/operators/aggregation/aggregate.rs | 6 +++--- .../operators/aggregation/state_machine.rs | 6 +++--- timely/src/dataflow/operators/branch.rs | 8 ++++---- timely/src/dataflow/operators/broadcast.rs | 2 +- .../src/dataflow/operators/core/enterleave.rs | 8 ++++---- timely/src/dataflow/operators/core/feedback.rs | 6 +++--- timely/src/dataflow/operators/core/filter.rs | 3 +-- timely/src/dataflow/operators/core/input.rs | 8 ++++---- timely/src/dataflow/operators/core/inspect.rs | 6 +++--- timely/src/dataflow/operators/core/map.rs | 9 ++++----- timely/src/dataflow/operators/core/ok_err.rs | 11 +++++------ timely/src/dataflow/operators/core/probe.rs | 6 ++---- timely/src/dataflow/operators/core/rc.rs | 6 +++--- timely/src/dataflow/operators/core/reclock.rs | 8 ++++---- timely/src/dataflow/operators/core/to_stream.rs | 4 ++-- .../dataflow/operators/core/unordered_input.rs | 4 ++-- timely/src/dataflow/operators/count.rs | 9 ++++----- timely/src/dataflow/operators/delay.rs | 5 ++--- timely/src/dataflow/operators/filter.rs | 5 ++--- .../src/dataflow/operators/flow_controlled.rs | 5 ++--- .../src/dataflow/operators/generic/operator.rs | 16 ++++++++-------- timely/src/dataflow/operators/input.rs | 9 ++++----- timely/src/dataflow/operators/map.rs | 11 +++++------ timely/src/dataflow/operators/partition.rs | 7 +++---- timely/src/dataflow/operators/result.rs | 17 ++++++++--------- timely/src/dataflow/operators/to_stream.rs | 3 +-- .../src/dataflow/operators/unordered_input.rs | 6 ++---- timely/src/dataflow/stream.rs | 2 +- timely/src/lib.rs | 10 ++-------- timely/src/logging.rs | 2 +- timely/src/synchronization/sequence.rs | 4 ++-- 35 files changed, 101 insertions(+), 125 deletions(-) diff --git a/timely/src/dataflow/channels/pact.rs b/timely/src/dataflow/channels/pact.rs index e93cb3b68c..fe04f8cd1e 100644 --- a/timely/src/dataflow/channels/pact.rs +++ b/timely/src/dataflow/channels/pact.rs @@ -18,7 +18,6 @@ use crate::dataflow::channels::Message; use crate::logging::{TimelyLogger as Logger, MessagesEvent}; use crate::progress::Timestamp; use crate::worker::AsWorker; -use crate::Data; /// A `ParallelizationContract` allocates paired `Push` and `Pull` implementors. pub trait ParallelizationContract { @@ -84,7 +83,7 @@ impl ParallelizationContract for where CB: ContainerBuilder, CB: for<'a> PushInto<::Item<'a>>, - CB::Container: Data + Send + SizableContainer + crate::dataflow::channels::ContainerBytes, + CB::Container: Send + SizableContainer + crate::dataflow::channels::ContainerBytes, for<'a> H: FnMut(&::Item<'a>) -> u64 { type Pusher = ExchangePusher>>>, H>; diff --git a/timely/src/dataflow/channels/pushers/exchange.rs b/timely/src/dataflow/channels/pushers/exchange.rs index a38f87aa53..89b1ddfa06 100644 --- a/timely/src/dataflow/channels/pushers/exchange.rs +++ b/timely/src/dataflow/channels/pushers/exchange.rs @@ -3,7 +3,7 @@ use crate::communication::Push; use crate::container::{ContainerBuilder, SizableContainer, PushInto}; use crate::dataflow::channels::Message; -use crate::{Container, Data}; +use crate::Container; // TODO : Software write combining /// Distributes records among target pushees according to a distribution function. @@ -50,7 +50,7 @@ where } } -impl Push> for Exchange +impl Push> for Exchange where CB: ContainerBuilder, CB::Container: SizableContainer, diff --git a/timely/src/dataflow/channels/pushers/owned.rs b/timely/src/dataflow/channels/pushers/owned.rs index 4286a40784..1034516ebe 100644 --- a/timely/src/dataflow/channels/pushers/owned.rs +++ b/timely/src/dataflow/channels/pushers/owned.rs @@ -5,7 +5,6 @@ use std::fmt; use std::rc::Rc; use timely_communication::Push; -use crate::{Data, Container}; use crate::dataflow::channels::Message; /// A pusher that can bind to a single downstream pusher. @@ -37,7 +36,7 @@ impl Clone for PushOwned { } } -impl Push> for PushOwned { +impl Push> for PushOwned { #[inline] fn push(&mut self, message: &mut Option>) { let mut pusher = self.0.borrow_mut(); diff --git a/timely/src/dataflow/channels/pushers/tee.rs b/timely/src/dataflow/channels/pushers/tee.rs index e9251c24aa..e12a9178b2 100644 --- a/timely/src/dataflow/channels/pushers/tee.rs +++ b/timely/src/dataflow/channels/pushers/tee.rs @@ -7,7 +7,7 @@ use std::rc::Rc; use crate::dataflow::channels::Message; use crate::communication::Push; -use crate::{Container, Data}; +use crate::Container; type PushList = Rc>>>>>; @@ -17,7 +17,7 @@ pub struct Tee { shared: PushList, } -impl Push> for Tee { +impl Push> for Tee { #[inline] fn push(&mut self, message: &mut Option>) { let mut pushers = self.shared.borrow_mut(); diff --git a/timely/src/dataflow/operators/aggregation/aggregate.rs b/timely/src/dataflow/operators/aggregation/aggregate.rs index 68be9b8956..68a82b3fcd 100644 --- a/timely/src/dataflow/operators/aggregation/aggregate.rs +++ b/timely/src/dataflow/operators/aggregation/aggregate.rs @@ -2,7 +2,7 @@ use std::hash::Hash; use std::collections::HashMap; -use crate::{Data, ExchangeData}; +use crate::ExchangeData; use crate::dataflow::{Scope, StreamLike, OwnedStream}; use crate::dataflow::operators::generic::operator::Operator; use crate::dataflow::channels::pact::Exchange; @@ -60,7 +60,7 @@ pub trait Aggregate { /// .inspect(|x| assert!(*x == (0, 5) || *x == (1, 5))); /// }); /// ``` - fn aggregateR+'static, H: Fn(&K)->u64+'static>( + fn aggregateR+'static, H: Fn(&K)->u64+'static>( self, fold: F, emit: E, @@ -75,7 +75,7 @@ where S: StreamLike>, { - fn aggregateR+'static, H: Fn(&K)->u64+'static>( + fn aggregateR+'static, H: Fn(&K)->u64+'static>( self, fold: F, emit: E, diff --git a/timely/src/dataflow/operators/aggregation/state_machine.rs b/timely/src/dataflow/operators/aggregation/state_machine.rs index 33a6a59c04..eb65fdc4dc 100644 --- a/timely/src/dataflow/operators/aggregation/state_machine.rs +++ b/timely/src/dataflow/operators/aggregation/state_machine.rs @@ -2,7 +2,7 @@ use std::hash::Hash; use std::collections::HashMap; -use crate::{Data, ExchangeData}; +use crate::ExchangeData; use crate::dataflow::{OwnedStream, Scope, StreamLike}; use crate::dataflow::operators::generic::operator::Operator; use crate::dataflow::channels::pact::Exchange; @@ -46,7 +46,7 @@ pub trait StateMachine { /// }); /// ``` fn state_machine< - R: Data, // output type + R: 'static, // output type D: Default+'static, // per-key state (data) I: IntoIterator, // type of output iterator F: Fn(&K, V, &mut D)->(bool, I)+'static, // state update logic @@ -62,7 +62,7 @@ where S: StreamLike>, { fn state_machine< - R: Data, // output type + R: 'static, // output type D: Default+'static, // per-key state (data) I: IntoIterator, // type of output iterator F: Fn(&K, V, &mut D)->(bool, I)+'static, // state update logic diff --git a/timely/src/dataflow/operators/branch.rs b/timely/src/dataflow/operators/branch.rs index 50a71477a9..0ec288e618 100644 --- a/timely/src/dataflow/operators/branch.rs +++ b/timely/src/dataflow/operators/branch.rs @@ -3,10 +3,10 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::dataflow::{Scope, OwnedStream, StreamLike}; -use crate::{Container, Data}; +use crate::Container; /// Extension trait for `Stream`. -pub trait Branch { +pub trait Branch { /// Takes one input stream and splits it into two output streams. /// For each record, the supplied closure is called with a reference to /// the data and its time. If it returns `true`, the record will be sent @@ -34,7 +34,7 @@ pub trait Branch { ) -> (OwnedStream>, OwnedStream>); } -impl>> Branch for S { +impl>> Branch for S { fn branch( self, condition: impl Fn(&G::Timestamp, &D) -> bool + 'static, @@ -92,7 +92,7 @@ pub trait BranchWhen: Sized { fn branch_when(self, condition: impl Fn(&G::Timestamp) -> bool + 'static) -> (OwnedStream, OwnedStream); } -impl> BranchWhen for S { +impl> BranchWhen for S { fn branch_when(self, condition: impl Fn(&G::Timestamp) -> bool + 'static) -> (OwnedStream, OwnedStream) { let mut builder = OperatorBuilder::new("Branch".to_owned(), self.scope()); diff --git a/timely/src/dataflow/operators/broadcast.rs b/timely/src/dataflow/operators/broadcast.rs index 5daa0b11de..435573969c 100644 --- a/timely/src/dataflow/operators/broadcast.rs +++ b/timely/src/dataflow/operators/broadcast.rs @@ -21,7 +21,7 @@ pub trait Broadcast { fn broadcast(self) -> OwnedStream>; } -impl>> Broadcast for S { +impl>> Broadcast for S { fn broadcast(self) -> OwnedStream> { // NOTE: Simplified implementation due to underlying motion diff --git a/timely/src/dataflow/operators/core/enterleave.rs b/timely/src/dataflow/operators/core/enterleave.rs index 3ccf303e8e..34ac6745cb 100644 --- a/timely/src/dataflow/operators/core/enterleave.rs +++ b/timely/src/dataflow/operators/core/enterleave.rs @@ -25,7 +25,7 @@ use crate::logging::{TimelyLogger, MessagesEvent}; use crate::progress::Timestamp; use crate::progress::timestamp::Refines; use crate::progress::{Source, Target}; -use crate::{Container, Data}; +use crate::Container; use crate::communication::Push; use crate::dataflow::channels::pushers::{Counter, PushOwned}; use crate::dataflow::channels::Message; @@ -53,7 +53,7 @@ pub trait Enter, C: Container> { fn enter<'a>(self, _: &Child<'a, G, T>) -> OwnedStream, C>; } -impl, C: Data+Container, S: StreamLike> Enter for S { +impl, C: Container + 'static, S: StreamLike> Enter for S { fn enter<'a>(self, scope: &Child<'a, G, T>) -> OwnedStream, C> { use crate::scheduling::Scheduler; @@ -130,14 +130,14 @@ impl<'a, G: Scope, C: Container + 'static, T: Timestamp+Refines, S } -struct IngressNub, TContainer: Container + Data> { +struct IngressNub, TContainer: Container> { targets: Counter>, phantom: PhantomData, activator: crate::scheduling::Activator, active: bool, } -impl, TContainer: Container + Data> Push> for IngressNub { +impl, TContainer: Container> Push> for IngressNub { fn push(&mut self, element: &mut Option>) { if let Some(outer_message) = element { let data = ::std::mem::take(&mut outer_message.data); diff --git a/timely/src/dataflow/operators/core/feedback.rs b/timely/src/dataflow/operators/core/feedback.rs index 0929b25ce1..ee90bd3d42 100644 --- a/timely/src/dataflow/operators/core/feedback.rs +++ b/timely/src/dataflow/operators/core/feedback.rs @@ -1,6 +1,6 @@ //! Create cycles in a timely dataflow graph. -use crate::{Container, Data}; +use crate::Container; use crate::container::CapacityContainerBuilder; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::channels::pushers::PushOwned; @@ -85,7 +85,7 @@ impl<'a, G: Scope, T: Timestamp> LoopVariable<'a, G, T> for Iterative<'a, G, T> } /// Connect a `Stream` to the input of a loop variable. -pub trait ConnectLoop { +pub trait ConnectLoop { /// Connect a `Stream` to be the input of a loop variable. /// /// # Examples @@ -106,7 +106,7 @@ pub trait ConnectLoop { fn connect_loop(self, handle: Handle); } -impl> ConnectLoop for S { +impl> ConnectLoop for S { fn connect_loop(self, handle: Handle) { let mut builder = handle.builder; diff --git a/timely/src/dataflow/operators/core/filter.rs b/timely/src/dataflow/operators/core/filter.rs index f60eb39c62..38294b624f 100644 --- a/timely/src/dataflow/operators/core/filter.rs +++ b/timely/src/dataflow/operators/core/filter.rs @@ -1,6 +1,5 @@ //! Filters a stream by a predicate. use crate::container::{Container, SizableContainer, PushInto}; -use crate::Data; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{OwnedStream, Scope, StreamLike}; use crate::dataflow::operators::generic::operator::Operator; @@ -23,7 +22,7 @@ pub trait Filter { fn filter)->bool+'static>(self, predicate: P) -> OwnedStream; } -impl> Filter for S +impl> Filter for S where for<'a> C: PushInto> { diff --git a/timely/src/dataflow/operators/core/input.rs b/timely/src/dataflow/operators/core/input.rs index d2c6e47818..a8c6c08e61 100644 --- a/timely/src/dataflow/operators/core/input.rs +++ b/timely/src/dataflow/operators/core/input.rs @@ -11,7 +11,7 @@ use crate::progress::frontier::Antichain; use crate::progress::{Operate, operate::SharedProgress, Timestamp, ChangeBatch}; use crate::progress::Source; -use crate::{Container, Data}; +use crate::Container; use crate::communication::Push; use crate::dataflow::{OwnedStream, Scope, ScopeParent}; use crate::dataflow::channels::pushers::{Counter, PushOwned}; @@ -60,7 +60,7 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, OwnedStream); + fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, OwnedStream); /// Create a new [StreamCore] and [Handle] through which to supply input. /// @@ -141,7 +141,7 @@ pub trait Input : Scope { use crate::order::TotalOrder; impl Input for G where ::Timestamp: TotalOrder { - fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, OwnedStream) { + fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, OwnedStream) { let mut handle = Handle::new(); let stream = self.input_from(&mut handle); (handle, stream) @@ -240,7 +240,7 @@ where now_at: T, } -impl Handle> { +impl Handle> { /// Allocates a new input handle, from which one can create timely streams. /// /// # Examples diff --git a/timely/src/dataflow/operators/core/inspect.rs b/timely/src/dataflow/operators/core/inspect.rs index b578e11d42..09428465b4 100644 --- a/timely/src/dataflow/operators/core/inspect.rs +++ b/timely/src/dataflow/operators/core/inspect.rs @@ -1,6 +1,6 @@ //! Extension trait and implementation for observing and action on streamed data. -use crate::{Container, Data}; +use crate::Container; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{Scope, OwnedStream, StreamLike}; use crate::dataflow::operators::generic::Operator; @@ -90,7 +90,7 @@ pub trait Inspect: InspectCore + Sized { fn inspect_core(self, func: F) -> OwnedStream where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl> Inspect for S { +impl> Inspect for S { fn inspect_core(self, func: F) -> OwnedStream where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>) + 'static { self.inspect_container(func) } @@ -120,7 +120,7 @@ pub trait InspectCore { fn inspect_container(self, func: F) -> OwnedStream where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl> InspectCore for S { +impl> InspectCore for S { fn inspect_container(self, mut func: F) -> OwnedStream where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static diff --git a/timely/src/dataflow/operators/core/map.rs b/timely/src/dataflow/operators/core/map.rs index 8f0a4390f2..f560a0c853 100644 --- a/timely/src/dataflow/operators/core/map.rs +++ b/timely/src/dataflow/operators/core/map.rs @@ -1,7 +1,6 @@ //! Extension methods for `StreamCore` based on record-by-record transformation. use crate::container::{Container, SizableContainer, PushInto}; -use crate::Data; use crate::dataflow::{OwnedStream, Scope, StreamLike}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::operator::Operator; @@ -24,7 +23,7 @@ pub trait Map : Sized { /// ``` fn map(self, mut logic: L) -> OwnedStream where - C2: SizableContainer + PushInto + Data, + C2: SizableContainer + PushInto + 'static, L: FnMut(C::Item<'_>)->D2 + 'static, { self.flat_map(move |x| std::iter::once(logic(x))) @@ -46,19 +45,19 @@ pub trait Map : Sized { fn flat_map(self, logic: L) -> OwnedStream where I: IntoIterator, - C2: SizableContainer + PushInto + Data, + C2: SizableContainer + PushInto + 'static, L: FnMut(C::Item<'_>)->I + 'static, ; } -impl> Map for S { +impl> Map for S { // TODO : This would be more robust if it captured an iterator and then pulled an appropriate // TODO : number of elements from the iterator. This would allow iterators that produce many // TODO : records without taking arbitrarily long and arbitrarily much memory. fn flat_map(self, mut logic: L) -> OwnedStream where I: IntoIterator, - C2: SizableContainer + PushInto + Data, + C2: SizableContainer + PushInto + 'static, L: FnMut(C::Item<'_>)->I + 'static, { self.unary(Pipeline, "FlatMap", move |_,_| move |input, output| { diff --git a/timely/src/dataflow/operators/core/ok_err.rs b/timely/src/dataflow/operators/core/ok_err.rs index 559b623a64..6ba7cffcd9 100644 --- a/timely/src/dataflow/operators/core/ok_err.rs +++ b/timely/src/dataflow/operators/core/ok_err.rs @@ -1,7 +1,6 @@ //! Operators that separate one stream into two streams based on some condition use crate::container::{Container, SizableContainer, PushInto}; -use crate::Data; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::dataflow::{Scope, OwnedStream, StreamLike}; @@ -30,17 +29,17 @@ pub trait OkErr { /// ``` fn ok_err(self, logic: L) -> (OwnedStream, OwnedStream) where - C1: SizableContainer + PushInto + Data, - C2: SizableContainer + PushInto + Data, + C1: SizableContainer + PushInto + 'static, + C2: SizableContainer + PushInto + 'static, L: FnMut(C::Item<'_>) -> Result+'static ; } -impl> OkErr for S { +impl> OkErr for S { fn ok_err(self, mut logic: L) -> (OwnedStream, OwnedStream) where - C1: SizableContainer + PushInto + Data, - C2: SizableContainer + PushInto + Data, + C1: SizableContainer + PushInto + 'static, + C2: SizableContainer + PushInto + 'static, L: FnMut(C::Item<'_>) -> Result+'static { let mut builder = OperatorBuilder::new("OkErr".to_owned(), self.scope()); diff --git a/timely/src/dataflow/operators/core/probe.rs b/timely/src/dataflow/operators/core/probe.rs index 8c95b43004..25989c6529 100644 --- a/timely/src/dataflow/operators/core/probe.rs +++ b/timely/src/dataflow/operators/core/probe.rs @@ -10,10 +10,8 @@ use crate::dataflow::channels::pushers::buffer::Buffer as PushBuffer; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::channels::pullers::Counter as PullCounter; use crate::dataflow::operators::generic::builder_raw::OperatorBuilder; - - use crate::dataflow::{Scope, OwnedStream, StreamLike}; -use crate::{Container, Data}; +use crate::Container; /// Monitors progress at a `Stream`. pub trait Probe { @@ -79,7 +77,7 @@ pub trait Probe { fn probe_with(self, handle: &Handle) -> OwnedStream; } -impl> Probe for S { +impl> Probe for S { fn probe(self) -> Handle { // the frontier is shared state; scope updates, handle reads. diff --git a/timely/src/dataflow/operators/core/rc.rs b/timely/src/dataflow/operators/core/rc.rs index 4eebb763a7..e5a3e297ca 100644 --- a/timely/src/dataflow/operators/core/rc.rs +++ b/timely/src/dataflow/operators/core/rc.rs @@ -1,10 +1,10 @@ //! Shared containers +use std::rc::Rc; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::Operator; use crate::dataflow::{OwnedStream, Scope, StreamLike}; -use crate::{Container, Data}; -use std::rc::Rc; +use crate::Container; /// Convert a stream into a stream of shared containers pub trait SharedStream { @@ -24,7 +24,7 @@ pub trait SharedStream { fn shared(self) -> OwnedStream>; } -impl> SharedStream for S { +impl> SharedStream for S { fn shared(self) -> OwnedStream> { self.unary(Pipeline, "Shared", move |_, _| { move |input, output| { diff --git a/timely/src/dataflow/operators/core/reclock.rs b/timely/src/dataflow/operators/core/reclock.rs index 1f891a824f..697b3ebdd7 100644 --- a/timely/src/dataflow/operators/core/reclock.rs +++ b/timely/src/dataflow/operators/core/reclock.rs @@ -1,6 +1,6 @@ //! Extension methods for `Stream` based on record-by-record transformation. -use crate::{Container, Data}; +use crate::Container; use crate::order::PartialOrder; use crate::dataflow::{Scope, OwnedStream, StreamLike}; use crate::dataflow::channels::pact::Pipeline; @@ -45,11 +45,11 @@ pub trait Reclock { /// assert_eq!(extracted[1], (5, vec![4,5])); /// assert_eq!(extracted[2], (8, vec![6,7,8])); /// ``` - fn reclock>(self, clock: CS) -> OwnedStream; + fn reclock>(self, clock: CS) -> OwnedStream; } -impl> Reclock for S { - fn reclock>(self, clock: CS) -> OwnedStream { +impl> Reclock for S { + fn reclock>(self, clock: CS) -> OwnedStream { let mut stash = vec![]; diff --git a/timely/src/dataflow/operators/core/to_stream.rs b/timely/src/dataflow/operators/core/to_stream.rs index 4df6a2162c..4c2b918463 100644 --- a/timely/src/dataflow/operators/core/to_stream.rs +++ b/timely/src/dataflow/operators/core/to_stream.rs @@ -1,7 +1,7 @@ //! Conversion to the `StreamCore` type from iterators. use crate::container::{CapacityContainerBuilder, ContainerBuilder, SizableContainer, PushInto}; -use crate::{Container, Data}; +use crate::Container; use crate::dataflow::operators::generic::operator::source; use crate::dataflow::{Scope, OwnedStream}; @@ -81,7 +81,7 @@ pub trait ToStream { fn to_stream(self, scope: &mut S) -> OwnedStream; } -impl ToStream for I where C: PushInto { +impl ToStream for I where C: PushInto { fn to_stream(self, scope: &mut S) -> OwnedStream { ToStreamBuilder::>::to_stream_with_builder(self, scope) } diff --git a/timely/src/dataflow/operators/core/unordered_input.rs b/timely/src/dataflow/operators/core/unordered_input.rs index 8160ef906d..6788a53e85 100644 --- a/timely/src/dataflow/operators/core/unordered_input.rs +++ b/timely/src/dataflow/operators/core/unordered_input.rs @@ -2,7 +2,7 @@ use std::rc::Rc; use std::cell::RefCell; -use crate::{Container, Data}; +use crate::Container; use crate::container::{ContainerBuilder, CapacityContainerBuilder}; use crate::scheduling::{Schedule, ActivateOnDrop}; @@ -165,7 +165,7 @@ impl UnorderedHandle { } } -impl UnorderedHandle> { +impl UnorderedHandle> { /// Allocates a new automatically flushing session based on the supplied capability. #[inline] pub fn session(&mut self, cap: ActivateCapability) -> ActivateOnDrop, Counter>>> { diff --git a/timely/src/dataflow/operators/count.rs b/timely/src/dataflow/operators/count.rs index 452d9275c9..625deddbc9 100644 --- a/timely/src/dataflow/operators/count.rs +++ b/timely/src/dataflow/operators/count.rs @@ -1,13 +1,12 @@ //! Counts the number of records at each time. use std::collections::HashMap; -use crate::Data; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{OwnedStream, Scope, StreamLike}; use crate::dataflow::operators::generic::operator::Operator; /// Accumulates records within a timestamp. -pub trait Accumulate: Sized { +pub trait Accumulate: Sized { /// Accumulates records within a timestamp. /// /// # Examples @@ -25,7 +24,7 @@ pub trait Accumulate: Sized { /// let extracted = captured.extract(); /// assert_eq!(extracted, vec![(0, vec![45])]); /// ``` - fn accumulate(self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> OwnedStream>; + fn accumulate(self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> OwnedStream>; /// Counts the number of records observed at each time. /// /// # Examples @@ -48,8 +47,8 @@ pub trait Accumulate: Sized { } } -impl>> Accumulate for S { - fn accumulate(self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> OwnedStream> { +impl>> Accumulate for S { + fn accumulate(self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> OwnedStream> { let mut accums = HashMap::new(); self.unary_notify(Pipeline, "Accumulate", vec![], move |input, output, notificator| { diff --git a/timely/src/dataflow/operators/delay.rs b/timely/src/dataflow/operators/delay.rs index 555acc7237..e506e63549 100644 --- a/timely/src/dataflow/operators/delay.rs +++ b/timely/src/dataflow/operators/delay.rs @@ -2,14 +2,13 @@ use std::collections::HashMap; -use crate::Data; use crate::order::{PartialOrder, TotalOrder}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{OwnedStream, StreamLike, Scope}; use crate::dataflow::operators::generic::operator::Operator; /// Methods to advance the timestamps of records or batches of records. -pub trait Delay { +pub trait Delay { /// Advances the timestamp of records using a supplied function. /// @@ -94,7 +93,7 @@ pub trait Delay { fn delay_batchG::Timestamp+'static>(self, func: L) -> OwnedStream>; } -impl>> Delay for S { +impl>> Delay for S { fn delayG::Timestamp+'static>(self, mut func: L) -> OwnedStream> { let mut elements = HashMap::new(); self.unary_notify(Pipeline, "Delay", vec![], move |input, output, notificator| { diff --git a/timely/src/dataflow/operators/filter.rs b/timely/src/dataflow/operators/filter.rs index d4c89e6106..6f6461213d 100644 --- a/timely/src/dataflow/operators/filter.rs +++ b/timely/src/dataflow/operators/filter.rs @@ -1,12 +1,11 @@ //! Filters a stream by a predicate. -use crate::Data; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{OwnedStream, StreamLike, Scope}; use crate::dataflow::operators::generic::operator::Operator; /// Extension trait for filtering. -pub trait Filter { +pub trait Filter { /// Returns a new instance of `self` containing only records satisfying `predicate`. /// /// # Examples @@ -22,7 +21,7 @@ pub trait Filter { fn filterbool+'static>(self, predicate: P) -> OwnedStream>; } -impl>> Filter for S { +impl>> Filter for S { fn filterbool+'static>(self, mut predicate: P) -> OwnedStream> { self.unary(Pipeline, "Filter", move |_,_| move |input, output| { input.for_each(|time, data| { diff --git a/timely/src/dataflow/operators/flow_controlled.rs b/timely/src/dataflow/operators/flow_controlled.rs index 13d7bc2f1c..1c87193484 100644 --- a/timely/src/dataflow/operators/flow_controlled.rs +++ b/timely/src/dataflow/operators/flow_controlled.rs @@ -1,6 +1,5 @@ //! Methods to construct flow-controlled sources. -use crate::Data; use crate::order::{PartialOrder, TotalOrder}; use crate::progress::timestamp::Timestamp; use crate::dataflow::operators::generic::operator::source; @@ -8,7 +7,7 @@ use crate::dataflow::operators::probe::Handle; use crate::dataflow::{OwnedStream, Scope}; /// Output of the input reading function for iterator_source. -pub struct IteratorSourceInput, I: IntoIterator> { +pub struct IteratorSourceInput, I: IntoIterator> { /// Lower bound on timestamps that can be emitted by this input in the future. pub lower_bound: T, /// Any `T: IntoIterator` of new input data in the form (time, data): time must be @@ -73,7 +72,7 @@ pub struct IteratorSourceInput, I: I /// ``` pub fn iterator_source< G: Scope, - D: Data, + D: 'static, DI: IntoIterator, I: IntoIterator, F: FnMut(&G::Timestamp)->Option>+'static>( diff --git a/timely/src/dataflow/operators/generic/operator.rs b/timely/src/dataflow/operators/generic/operator.rs index 6ff529e105..33b619b77f 100644 --- a/timely/src/dataflow/operators/generic/operator.rs +++ b/timely/src/dataflow/operators/generic/operator.rs @@ -12,7 +12,7 @@ use crate::dataflow::{Scope, OwnedStream, StreamLike}; use super::builder_rc::OperatorBuilder; use crate::dataflow::operators::generic::OperatorInfo; use crate::dataflow::operators::generic::notificator::{Notificator, FrontierNotificator}; -use crate::{Container, Data}; +use crate::Container; use crate::container::{ContainerBuilder, CapacityContainerBuilder}; /// Methods to construct generic streaming and blocking operators. @@ -181,7 +181,7 @@ pub trait Operator { /// ``` fn binary_frontier(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream where - C2: Container + Data, + C2: Container + 'static, CB: ContainerBuilder, O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, @@ -232,7 +232,7 @@ pub trait Operator { /// } /// }).unwrap(); /// ``` - fn binary_notify, L: FnMut(&mut InputHandleCore, @@ -275,7 +275,7 @@ pub trait Operator { /// ``` fn binary(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream where - C2: Container + Data, + C2: Container + 'static, CB: ContainerBuilder, O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, @@ -314,7 +314,7 @@ pub trait Operator { P: ParallelizationContract; } -impl> Operator for S { +impl> Operator for S { fn unary_frontier(self, pact: P, name: &str, constructor: B) -> OwnedStream where @@ -396,7 +396,7 @@ impl> Operator for S fn binary_frontier(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream where - C2: Container + Data, + C2: Container + 'static, CB: ContainerBuilder, O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, @@ -428,7 +428,7 @@ impl> Operator for S stream } - fn binary_notify, L: FnMut(&mut InputHandleCore, @@ -458,7 +458,7 @@ impl> Operator for S fn binary(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream where - C2: Container + Data, + C2: Container + 'static, CB: ContainerBuilder, O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, diff --git a/timely/src/dataflow/operators/input.rs b/timely/src/dataflow/operators/input.rs index 5d7bb9b4cc..e87881739e 100644 --- a/timely/src/dataflow/operators/input.rs +++ b/timely/src/dataflow/operators/input.rs @@ -1,6 +1,5 @@ //! Create new `Streams` connected to external inputs. -use crate::Data; use crate::container::CapacityContainerBuilder; use crate::dataflow::{ScopeParent, Scope, OwnedStream}; use crate::dataflow::operators::core::{Input as InputCore}; @@ -47,7 +46,7 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn new_input(&mut self) -> (Handle<::Timestamp, D>, OwnedStream>); + fn new_input(&mut self) -> (Handle<::Timestamp, D>, OwnedStream>); /// Create a new stream from a supplied interactive handle. /// @@ -79,16 +78,16 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> OwnedStream>; + fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> OwnedStream>; } use crate::order::TotalOrder; impl Input for G where ::Timestamp: TotalOrder { - fn new_input(&mut self) -> (Handle<::Timestamp, D>, OwnedStream>) { + fn new_input(&mut self) -> (Handle<::Timestamp, D>, OwnedStream>) { InputCore::new_input(self) } - fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> OwnedStream> { + fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> OwnedStream> { InputCore::input_from(self, handle) } } diff --git a/timely/src/dataflow/operators/map.rs b/timely/src/dataflow/operators/map.rs index 2ff9b81d7b..5aee3396a4 100644 --- a/timely/src/dataflow/operators/map.rs +++ b/timely/src/dataflow/operators/map.rs @@ -1,13 +1,12 @@ //! Extension methods for `Stream` based on record-by-record transformation. -use crate::Data; use crate::dataflow::{OwnedStream, StreamLike, Scope}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::operator::Operator; use crate::dataflow::operators::core::{Map as MapCore}; /// Extension trait for `Stream`. -pub trait Map : Sized { +pub trait Map : Sized { /// Consumes each element of the stream and yields a new element. /// /// # Examples @@ -20,7 +19,7 @@ pub trait Map : Sized { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn mapD2+'static>(self, mut logic: L) -> OwnedStream> { + fn mapD2+'static>(self, mut logic: L) -> OwnedStream> { self.flat_map(move |x| std::iter::once(logic(x))) } /// Updates each element of the stream and yields the element, re-using memory where possible. @@ -48,10 +47,10 @@ pub trait Map : Sized { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn flat_mapI+'static>(self, logic: L) -> OwnedStream> where I::Item: Data; + fn flat_mapI+'static>(self, logic: L) -> OwnedStream> where I::Item: 'static; } -impl>> Map for S { +impl>> Map for S { fn map_in_place(self, mut logic: L) -> OwnedStream> { self.unary(Pipeline, "MapInPlace", move |_,_| move |input, output| { input.for_each(|time, data| { @@ -63,7 +62,7 @@ impl>> Map for S { // TODO : This would be more robust if it captured an iterator and then pulled an appropriate // TODO : number of elements from the iterator. This would allow iterators that produce many // TODO : records without taking arbitrarily long and arbitrarily much memory. - fn flat_mapI+'static>(self, logic: L) -> OwnedStream> where I::Item: Data { + fn flat_mapI+'static>(self, logic: L) -> OwnedStream> where I::Item: 'static { MapCore::flat_map(self, logic) } } diff --git a/timely/src/dataflow/operators/partition.rs b/timely/src/dataflow/operators/partition.rs index 8661e15907..c796f9ba73 100644 --- a/timely/src/dataflow/operators/partition.rs +++ b/timely/src/dataflow/operators/partition.rs @@ -3,10 +3,9 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::dataflow::{Scope, OwnedStream, StreamLike}; -use crate::Data; /// Partition a stream of records into multiple streams. -pub trait Partition (u64, D2)> { +pub trait Partition (u64, D2)> { /// Produces `parts` output streams, containing records produced and assigned by `route`. /// /// # Examples @@ -28,8 +27,8 @@ pub trait Partition (u64, D2)> { impl Partition for S where G: Scope, - D: Data, - D2: Data, + D: 'static, + D2: 'static, F: Fn(D) -> (u64, D2) + 'static, S: StreamLike>, { diff --git a/timely/src/dataflow/operators/result.rs b/timely/src/dataflow/operators/result.rs index 4a1ee3ec37..907b768161 100644 --- a/timely/src/dataflow/operators/result.rs +++ b/timely/src/dataflow/operators/result.rs @@ -1,11 +1,10 @@ //! Extension methods for `Stream` containing `Result`s. -use crate::Data; use crate::dataflow::operators::Map; use crate::dataflow::{Scope, StreamLike, OwnedStream}; /// Extension trait for `Stream`. -pub trait ResultStream { +pub trait ResultStream { /// Returns a new instance of `self` containing only `ok` records. /// /// # Examples @@ -46,7 +45,7 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn map_ok T2 + 'static>(self, logic: L) -> OwnedStream>>; + fn map_ok T2 + 'static>(self, logic: L) -> OwnedStream>>; /// Returns a new instance of `self` applying `logic` on all `Err` records. /// @@ -60,7 +59,7 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn map_err E2 + 'static>(self, logic: L) -> OwnedStream>>; + fn map_err E2 + 'static>(self, logic: L) -> OwnedStream>>; /// Returns a new instance of `self` applying `logic` on all `Ok` records, passes through `Err` /// records. @@ -75,7 +74,7 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn and_then Result + 'static>( + fn and_then Result + 'static>( self, logic: L, ) -> OwnedStream>>; @@ -95,7 +94,7 @@ pub trait ResultStream { fn unwrap_or_else T + 'static>(self, logic: L) -> OwnedStream>; } -impl>>> ResultStream for S { +impl>>> ResultStream for S { fn ok(self) -> OwnedStream> { self.flat_map(Result::ok) } @@ -104,15 +103,15 @@ impl>>> ResultStre self.flat_map(Result::err) } - fn map_ok T2 + 'static>(self, mut logic: L) -> OwnedStream>> { + fn map_ok T2 + 'static>(self, mut logic: L) -> OwnedStream>> { self.map(move |r| r.map(&mut logic)) } - fn map_err E2 + 'static>(self, mut logic: L) -> OwnedStream>> { + fn map_err E2 + 'static>(self, mut logic: L) -> OwnedStream>> { self.map(move |r| r.map_err(&mut logic)) } - fn and_then Result + 'static>(self, mut logic: L) -> OwnedStream>> { + fn and_then Result + 'static>(self, mut logic: L) -> OwnedStream>> { self.map(move |r| r.and_then(&mut logic)) } diff --git a/timely/src/dataflow/operators/to_stream.rs b/timely/src/dataflow/operators/to_stream.rs index 53aad3f2f4..28667301cc 100644 --- a/timely/src/dataflow/operators/to_stream.rs +++ b/timely/src/dataflow/operators/to_stream.rs @@ -1,6 +1,5 @@ //! Conversion to the `Stream` type from iterators. -use crate::Data; use crate::dataflow::{Scope, OwnedStream}; use crate::dataflow::operators::core::{ToStream as ToStreamCore}; @@ -25,7 +24,7 @@ pub trait ToStream { fn to_stream(self, scope: &mut S) -> OwnedStream>; } -impl ToStream for I where I::Item: Data { +impl ToStream for I { fn to_stream(self, scope: &mut S) -> OwnedStream> { ToStreamCore::to_stream(self, scope) } diff --git a/timely/src/dataflow/operators/unordered_input.rs b/timely/src/dataflow/operators/unordered_input.rs index 9572949c83..c57ef0a74f 100644 --- a/timely/src/dataflow/operators/unordered_input.rs +++ b/timely/src/dataflow/operators/unordered_input.rs @@ -1,7 +1,5 @@ //! Create new `Streams` connected to external inputs. -use crate::Data; - use crate::container::CapacityContainerBuilder; use crate::dataflow::operators::{ActivateCapability}; use crate::dataflow::operators::core::{UnorderedInput as UnorderedInputCore, UnorderedHandle as UnorderedHandleCore}; @@ -62,12 +60,12 @@ pub trait UnorderedInput { /// assert_eq!(extract[i], (i, vec![i])); /// } /// ``` - fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream>); + fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream>); } impl UnorderedInput for G { - fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream>) { + fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream>) { UnorderedInputCore::new_unordered_input(self) } } diff --git a/timely/src/dataflow/stream.rs b/timely/src/dataflow/stream.rs index 9a20d7a722..01aad1eeec 100644 --- a/timely/src/dataflow/stream.rs +++ b/timely/src/dataflow/stream.rs @@ -185,7 +185,7 @@ mod tests { .sink(Pipeline, "check non-clone", |input| { while let Some((_time, data)) = input.next() { for datum in data.drain(..) { - assert_eq!(datum, &NotClone); + assert_eq!(datum, NotClone); } } }); diff --git a/timely/src/lib.rs b/timely/src/lib.rs index 8be417d47b..419276b37a 100644 --- a/timely/src/lib.rs +++ b/timely/src/lib.rs @@ -98,18 +98,12 @@ pub mod logging; pub mod scheduling; -/// A composite trait for types usable as data in timely dataflow. -/// -/// The `Data` trait is necessary for all types that go along timely dataflow channels. -pub trait Data: Clone+'static { } -impl Data for T { } - /// A composite trait for types usable on exchange channels in timely dataflow. /// /// The `ExchangeData` trait extends `Data` with any requirements imposed by the `timely_communication` /// `Data` trait, which describes requirements for communication along channels. -pub trait ExchangeData: Data + encoding::Data { } -impl ExchangeData for T { } +pub trait ExchangeData: encoding::Data { } +impl ExchangeData for T { } #[doc = include_str!("../../README.md")] #[cfg(doctest)] diff --git a/timely/src/logging.rs b/timely/src/logging.rs index 6043f2d16d..0164fff1b1 100644 --- a/timely/src/logging.rs +++ b/timely/src/logging.rs @@ -101,7 +101,7 @@ pub trait ProgressEventTimestamp: std::fmt::Debug + std::any::Any { /// `"std::option::Option"`. fn type_name(&self) -> &'static str; } -impl ProgressEventTimestamp for T { +impl ProgressEventTimestamp for T { fn as_any(&self) -> &dyn std::any::Any { self } fn type_name(&self) -> &'static str { std::any::type_name::() } diff --git a/timely/src/synchronization/sequence.rs b/timely/src/synchronization/sequence.rs index 37c61c3f9f..883e34d9cf 100644 --- a/timely/src/synchronization/sequence.rs +++ b/timely/src/synchronization/sequence.rs @@ -52,7 +52,7 @@ pub struct Sequencer { recv: Rc>>, // sequenced items. } -impl Sequencer { +impl Sequencer { /// Creates a new Sequencer. /// @@ -230,4 +230,4 @@ impl Drop for Sequencer { .expect("Sequencer.activator unavailable") .activate() } -} \ No newline at end of file +} From 0fdc20c260d84d9c8018216c14943199c11994d9 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Tue, 10 Dec 2024 14:03:43 -0500 Subject: [PATCH 3/5] Simpler container calls Signed-off-by: Moritz Hoffmann --- timely/examples/wordcount.rs | 2 -- timely/src/dataflow/channels/pushers/owned.rs | 10 ++-------- timely/src/dataflow/stream.rs | 10 +++------- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/timely/examples/wordcount.rs b/timely/examples/wordcount.rs index bfb0b01ca4..4830953ad0 100644 --- a/timely/examples/wordcount.rs +++ b/timely/examples/wordcount.rs @@ -17,13 +17,11 @@ fn main() { // create a new input, exchange data, and inspect its output worker.dataflow::(|scope| { input.to_stream(scope) - .container::>() .flat_map(|(text, diff): (String, i64)| text.split_whitespace() .map(move |word| (word.to_owned(), diff)) .collect::>() ) - .container::>() .unary_frontier(exchange, "WordCount", |_capability, _info| { let mut queues = HashMap::new(); diff --git a/timely/src/dataflow/channels/pushers/owned.rs b/timely/src/dataflow/channels/pushers/owned.rs index 1034516ebe..ff926958a9 100644 --- a/timely/src/dataflow/channels/pushers/owned.rs +++ b/timely/src/dataflow/channels/pushers/owned.rs @@ -14,8 +14,8 @@ impl PushOwned { /// Create a new `PushOwned`. Similarly to `Tee`, it returns a pair where either element /// can be used as pusher or registrar. pub fn new() -> (Self, Self) { - let zelf = Self(Rc::new(RefCell::new(None))); - (zelf.clone(), zelf) + let shared = Rc::new(RefCell::new(None)); + (Self(Rc::clone(&shared)), Self(shared)) } /// Set the downstream pusher. @@ -30,12 +30,6 @@ impl fmt::Debug for PushOwned { } } -impl Clone for PushOwned { - fn clone(&self) -> Self { - Self(Rc::clone(&self.0)) - } -} - impl Push> for PushOwned { #[inline] fn push(&mut self, message: &mut Option>) { diff --git a/timely/src/dataflow/stream.rs b/timely/src/dataflow/stream.rs index 01aad1eeec..18b39b529d 100644 --- a/timely/src/dataflow/stream.rs +++ b/timely/src/dataflow/stream.rs @@ -81,7 +81,7 @@ impl OwnedStream { } /// Allows the assertion of a container type, for the benefit of type inference. - pub fn container(self) -> >::Stream where Self: AsStream { self.as_stream() } + pub fn container(self) -> OwnedStream where Self: AsStream { self.as_stream() } } /// A stream batching data in vectors. @@ -136,24 +136,20 @@ impl StreamCore { pub fn scope(&self) -> S { self.scope.clone() } /// Allows the assertion of a container type, for the benefit of type inference. - pub fn container(self) -> >::Stream where Self: AsStream { self.as_stream() } + pub fn container(self) -> StreamCore where Self: AsStream { self.as_stream() } } /// A type that can be translated to a [StreamCore]. pub trait AsStream { - /// The type of the stream. - type Stream; /// Translate `self` to a stream. - fn as_stream(self) -> Self::Stream; + fn as_stream(self) -> Self; } impl AsStream for StreamCore { - type Stream = StreamCore; fn as_stream(self) -> Self { self } } impl AsStream for OwnedStream { - type Stream = OwnedStream; fn as_stream(self) -> Self { self } } From 9df628b4707597fb1704da4b91c67123e31e9ba6 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Tue, 10 Dec 2024 20:10:46 -0500 Subject: [PATCH 4/5] Remove StreamLike trait Signed-off-by: Moritz Hoffmann --- mdbook/src/chapter_2/chapter_2_3.md | 3 +- timely/examples/unionfind.rs | 8 +- timely/examples/wordcount.rs | 2 + timely/src/dataflow/channels/pushers/owned.rs | 4 +- timely/src/dataflow/mod.rs | 2 +- .../operators/aggregation/aggregate.rs | 14 +- .../operators/aggregation/state_machine.rs | 14 +- timely/src/dataflow/operators/branch.rs | 18 +-- timely/src/dataflow/operators/broadcast.rs | 10 +- .../operators/core/capture/capture.rs | 12 +- .../dataflow/operators/core/capture/replay.rs | 12 +- timely/src/dataflow/operators/core/concat.rs | 33 ++--- .../src/dataflow/operators/core/enterleave.rs | 20 +-- .../src/dataflow/operators/core/exchange.rs | 14 +- .../src/dataflow/operators/core/feedback.rs | 12 +- timely/src/dataflow/operators/core/filter.rs | 10 +- timely/src/dataflow/operators/core/input.rs | 18 +-- timely/src/dataflow/operators/core/inspect.rs | 20 +-- timely/src/dataflow/operators/core/map.rs | 12 +- timely/src/dataflow/operators/core/ok_err.rs | 14 +- timely/src/dataflow/operators/core/probe.rs | 8 +- timely/src/dataflow/operators/core/rc.rs | 15 ++- timely/src/dataflow/operators/core/reclock.rs | 10 +- .../src/dataflow/operators/core/to_stream.rs | 12 +- .../operators/core/unordered_input.rs | 10 +- timely/src/dataflow/operators/count.rs | 10 +- timely/src/dataflow/operators/delay.rs | 16 +-- timely/src/dataflow/operators/filter.rs | 10 +- .../src/dataflow/operators/flow_controlled.rs | 4 +- .../dataflow/operators/generic/builder_raw.rs | 12 +- .../dataflow/operators/generic/builder_rc.rs | 10 +- .../dataflow/operators/generic/operator.rs | 38 +++--- timely/src/dataflow/operators/input.rs | 10 +- timely/src/dataflow/operators/map.rs | 16 +-- timely/src/dataflow/operators/partition.rs | 15 +-- timely/src/dataflow/operators/result.rs | 30 ++--- timely/src/dataflow/operators/to_stream.rs | 6 +- .../src/dataflow/operators/unordered_input.rs | 6 +- timely/src/dataflow/stream.rs | 122 +++++++----------- 39 files changed, 285 insertions(+), 327 deletions(-) diff --git a/mdbook/src/chapter_2/chapter_2_3.md b/mdbook/src/chapter_2/chapter_2_3.md index abc3d3afe0..46b4d2b162 100644 --- a/mdbook/src/chapter_2/chapter_2_3.md +++ b/mdbook/src/chapter_2/chapter_2_3.md @@ -169,7 +169,8 @@ fn main() { let streams = (0..10).to_stream(scope) .partition(3, |x| (x % 3, x)); - scope.concatenate(streams) + scope.clone() + .concatenate(streams) .inspect(|x| println!("seen: {:?}", x)); }); } diff --git a/timely/examples/unionfind.rs b/timely/examples/unionfind.rs index a0855d4b2c..5e827049b8 100644 --- a/timely/examples/unionfind.rs +++ b/timely/examples/unionfind.rs @@ -47,12 +47,12 @@ fn main() { }).unwrap(); // asserts error-free execution; } -trait UnionFind { - fn union_find(self) -> OwnedStream>; +trait UnionFind { + fn union_find(self) -> Self; } -impl>> UnionFind for S { - fn union_find(self) -> OwnedStream> { +impl UnionFind for Stream { + fn union_find(self) -> Stream { self.unary(Pipeline, "UnionFind", |_,_| { diff --git a/timely/examples/wordcount.rs b/timely/examples/wordcount.rs index 4830953ad0..bfb0b01ca4 100644 --- a/timely/examples/wordcount.rs +++ b/timely/examples/wordcount.rs @@ -17,11 +17,13 @@ fn main() { // create a new input, exchange data, and inspect its output worker.dataflow::(|scope| { input.to_stream(scope) + .container::>() .flat_map(|(text, diff): (String, i64)| text.split_whitespace() .map(move |word| (word.to_owned(), diff)) .collect::>() ) + .container::>() .unary_frontier(exchange, "WordCount", |_capability, _info| { let mut queues = HashMap::new(); diff --git a/timely/src/dataflow/channels/pushers/owned.rs b/timely/src/dataflow/channels/pushers/owned.rs index ff926958a9..6775357243 100644 --- a/timely/src/dataflow/channels/pushers/owned.rs +++ b/timely/src/dataflow/channels/pushers/owned.rs @@ -19,7 +19,9 @@ impl PushOwned { } /// Set the downstream pusher. - pub fn set> + 'static>(self, pusher: P) { + /// + /// Consumes `Self` as only a single pusher can be set. + pub fn set_pusher> + 'static>(self, pusher: P) { *self.0.borrow_mut() = Some(Box::new(pusher)); } } diff --git a/timely/src/dataflow/mod.rs b/timely/src/dataflow/mod.rs index f263451014..4ec963eb17 100644 --- a/timely/src/dataflow/mod.rs +++ b/timely/src/dataflow/mod.rs @@ -13,7 +13,7 @@ //! }); //! ``` -pub use self::stream::{StreamCore, Stream, StreamLike, OwnedStream}; +pub use self::stream::{StreamTee, Stream, StreamCore}; pub use self::scopes::{Scope, ScopeParent}; pub use self::operators::core::input::Handle as InputHandleCore; diff --git a/timely/src/dataflow/operators/aggregation/aggregate.rs b/timely/src/dataflow/operators/aggregation/aggregate.rs index 68a82b3fcd..63ad6b07a5 100644 --- a/timely/src/dataflow/operators/aggregation/aggregate.rs +++ b/timely/src/dataflow/operators/aggregation/aggregate.rs @@ -3,7 +3,7 @@ use std::hash::Hash; use std::collections::HashMap; use crate::ExchangeData; -use crate::dataflow::{Scope, StreamLike, OwnedStream}; +use crate::dataflow::{Stream, Scope}; use crate::dataflow::operators::generic::operator::Operator; use crate::dataflow::channels::pact::Exchange; @@ -64,22 +64,16 @@ pub trait Aggregate { self, fold: F, emit: E, - hash: H) -> OwnedStream> where S::Timestamp: Eq; + hash: H) -> Stream where S::Timestamp: Eq; } -impl Aggregate for S -where - G: Scope, - K: ExchangeData + Hash + Eq + Clone, - V: ExchangeData, - S: StreamLike>, -{ +impl Aggregate for Stream { fn aggregateR+'static, H: Fn(&K)->u64+'static>( self, fold: F, emit: E, - hash: H) -> OwnedStream> where G::Timestamp: Eq { + hash: H) -> Stream where S::Timestamp: Eq { let mut aggregates = HashMap::new(); self.unary_notify(Exchange::new(move |(k, _)| hash(k)), "Aggregate", vec![], move |input, output, notificator| { diff --git a/timely/src/dataflow/operators/aggregation/state_machine.rs b/timely/src/dataflow/operators/aggregation/state_machine.rs index eb65fdc4dc..b75ab174b7 100644 --- a/timely/src/dataflow/operators/aggregation/state_machine.rs +++ b/timely/src/dataflow/operators/aggregation/state_machine.rs @@ -3,7 +3,7 @@ use std::hash::Hash; use std::collections::HashMap; use crate::ExchangeData; -use crate::dataflow::{OwnedStream, Scope, StreamLike}; +use crate::dataflow::{Stream, Scope}; use crate::dataflow::operators::generic::operator::Operator; use crate::dataflow::channels::pact::Exchange; @@ -51,23 +51,17 @@ pub trait StateMachine { I: IntoIterator, // type of output iterator F: Fn(&K, V, &mut D)->(bool, I)+'static, // state update logic H: Fn(&K)->u64+'static, // "hash" function for keys - >(self, fold: F, hash: H) -> OwnedStream> where S::Timestamp : Hash+Eq ; + >(self, fold: F, hash: H) -> Stream where S::Timestamp : Hash+Eq ; } -impl StateMachine for S -where - G: Scope, - K: ExchangeData + Hash + Eq + Clone, - V: ExchangeData, - S: StreamLike>, -{ +impl StateMachine for Stream { fn state_machine< R: 'static, // output type D: Default+'static, // per-key state (data) I: IntoIterator, // type of output iterator F: Fn(&K, V, &mut D)->(bool, I)+'static, // state update logic H: Fn(&K)->u64+'static, // "hash" function for keys - >(self, fold: F, hash: H) -> OwnedStream> where G::Timestamp : Hash+Eq { + >(self, fold: F, hash: H) -> Stream where S::Timestamp : Hash+Eq { let mut pending: HashMap<_, Vec<(K, V)>> = HashMap::new(); // times -> (keys -> state) let mut states = HashMap::new(); // keys -> state diff --git a/timely/src/dataflow/operators/branch.rs b/timely/src/dataflow/operators/branch.rs index 0ec288e618..05a1396fd3 100644 --- a/timely/src/dataflow/operators/branch.rs +++ b/timely/src/dataflow/operators/branch.rs @@ -2,7 +2,7 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; -use crate::dataflow::{Scope, OwnedStream, StreamLike}; +use crate::dataflow::{Scope, Stream, StreamCore}; use crate::Container; /// Extension trait for `Stream`. @@ -31,14 +31,14 @@ pub trait Branch { fn branch( self, condition: impl Fn(&S::Timestamp, &D) -> bool + 'static, - ) -> (OwnedStream>, OwnedStream>); + ) -> (Stream, Stream); } -impl>> Branch for S { +impl Branch for Stream { fn branch( self, - condition: impl Fn(&G::Timestamp, &D) -> bool + 'static, - ) -> (OwnedStream>, OwnedStream>) { + condition: impl Fn(&S::Timestamp, &D) -> bool + 'static, + ) -> (Stream, Stream) { let mut builder = OperatorBuilder::new("Branch".to_owned(), self.scope()); let mut input = builder.new_input(self, Pipeline); @@ -69,7 +69,7 @@ impl>> Branch for S { } /// Extension trait for `Stream`. -pub trait BranchWhen: Sized { +pub trait BranchWhen: Sized { /// Takes one input stream and splits it into two output streams. /// For each time, the supplied closure is called. If it returns `true`, /// the records for that will be sent to the second returned stream, otherwise @@ -89,11 +89,11 @@ pub trait BranchWhen: Sized { /// after_five.inspect(|x| println!("Times 5 and later: {:?}", x)); /// }); /// ``` - fn branch_when(self, condition: impl Fn(&G::Timestamp) -> bool + 'static) -> (OwnedStream, OwnedStream); + fn branch_when(self, condition: impl Fn(&T) -> bool + 'static) -> (Self, Self); } -impl> BranchWhen for S { - fn branch_when(self, condition: impl Fn(&G::Timestamp) -> bool + 'static) -> (OwnedStream, OwnedStream) { +impl BranchWhen for StreamCore { + fn branch_when(self, condition: impl Fn(&S::Timestamp) -> bool + 'static) -> (Self, Self) { let mut builder = OperatorBuilder::new("Branch".to_owned(), self.scope()); let mut input = builder.new_input(self, Pipeline); diff --git a/timely/src/dataflow/operators/broadcast.rs b/timely/src/dataflow/operators/broadcast.rs index 435573969c..8a1979d12a 100644 --- a/timely/src/dataflow/operators/broadcast.rs +++ b/timely/src/dataflow/operators/broadcast.rs @@ -1,11 +1,11 @@ //! Broadcast records to all workers. use crate::ExchangeData; -use crate::dataflow::{OwnedStream, StreamLike, Scope}; +use crate::dataflow::{Stream, Scope}; use crate::dataflow::operators::{Map, Exchange}; /// Broadcast records to all workers. -pub trait Broadcast { +pub trait Broadcast { /// Broadcast records to all workers. /// /// # Examples @@ -18,11 +18,11 @@ pub trait Broadcast { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn broadcast(self) -> OwnedStream>; + fn broadcast(self) -> Self; } -impl>> Broadcast for S { - fn broadcast(self) -> OwnedStream> { +impl Broadcast for Stream { + fn broadcast(self) -> Stream { // NOTE: Simplified implementation due to underlying motion // in timely dataflow internals. Optimize once they have diff --git a/timely/src/dataflow/operators/core/capture/capture.rs b/timely/src/dataflow/operators/core/capture/capture.rs index 077b80a962..f6abe5346c 100644 --- a/timely/src/dataflow/operators/core/capture/capture.rs +++ b/timely/src/dataflow/operators/core/capture/capture.rs @@ -5,7 +5,7 @@ //! and there are several default implementations, including a linked-list, Rust's MPSC //! queue, and a binary serializer wrapping any `W: Write`. -use crate::dataflow::{Scope, StreamLike}; +use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::channels::pullers::Counter as PullCounter; use crate::dataflow::operators::generic::builder_raw::OperatorBuilder; @@ -17,7 +17,7 @@ use crate::progress::Timestamp; use super::{Event, EventPusher}; /// Capture a stream of timestamped data for later replay. -pub trait Capture : Sized { +pub trait Capture : Sized { /// Captures a stream of timestamped data for later replay. /// /// # Examples @@ -103,18 +103,18 @@ pub trait Capture : Sized { /// /// assert_eq!(recv0.extract()[0].1, (0..10).collect::>()); /// ``` - fn capture_into+'static>(self, pusher: P); + fn capture_into+'static>(self, pusher: P); /// Captures a stream using Rust's MPSC channels. - fn capture(self) -> ::std::sync::mpsc::Receiver> { + fn capture(self) -> ::std::sync::mpsc::Receiver> { let (send, recv) = ::std::sync::mpsc::channel(); self.capture_into(send); recv } } -impl> Capture for S { - fn capture_into+'static>(self, mut event_pusher: P) { +impl Capture for StreamCore { + fn capture_into+'static>(self, mut event_pusher: P) { let mut builder = OperatorBuilder::new("Capture".to_owned(), self.scope()); let mut input = PullCounter::new(builder.new_input(self, Pipeline)); diff --git a/timely/src/dataflow/operators/core/capture/replay.rs b/timely/src/dataflow/operators/core/capture/replay.rs index ea939fe971..74a94c36fc 100644 --- a/timely/src/dataflow/operators/core/capture/replay.rs +++ b/timely/src/dataflow/operators/core/capture/replay.rs @@ -38,7 +38,7 @@ //! allowing the replay to occur in a timely dataflow computation with more or fewer workers //! than that in which the stream was captured. -use crate::dataflow::{Scope, OwnedStream}; +use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pushers::Counter as PushCounter; use crate::dataflow::channels::pushers::buffer::Buffer as PushBuffer; use crate::dataflow::operators::generic::builder_raw::OperatorBuilder; @@ -50,16 +50,16 @@ use crate::Container; /// Replay a capture stream into a scope with the same timestamp. pub trait Replay : Sized { - /// Replays `self` into the provided scope, as a `OwnedStream`. - fn replay_into>(self, scope: &mut S) -> OwnedStream { + /// Replays `self` into the provided scope, as a `StreamCore`. + fn replay_into>(self, scope: &mut S) -> StreamCore { self.replay_core(scope, Some(std::time::Duration::new(0, 0))) } - /// Replays `self` into the provided scope, as a `OwnedStream`. + /// Replays `self` into the provided scope, as a `StreamCore`. /// /// The `period` argument allows the specification of a re-activation period, where the operator /// will re-activate itself every so often. The `None` argument instructs the operator not to /// re-activate itself. - fn replay_core>(self, scope: &mut S, period: Option) -> OwnedStream; + fn replay_core>(self, scope: &mut S, period: Option) -> StreamCore; } impl Replay for I @@ -67,7 +67,7 @@ where I : IntoIterator, ::Item: EventIterator+'static, { - fn replay_core>(self, scope: &mut S, period: Option) -> OwnedStream{ + fn replay_core>(self, scope: &mut S, period: Option) -> StreamCore{ let mut builder = OperatorBuilder::new("Replay".to_owned(), scope.clone()); diff --git a/timely/src/dataflow/operators/core/concat.rs b/timely/src/dataflow/operators/core/concat.rs index 779b7ba44e..cbb5c5d47f 100644 --- a/timely/src/dataflow/operators/core/concat.rs +++ b/timely/src/dataflow/operators/core/concat.rs @@ -3,10 +3,10 @@ use crate::Container; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{OwnedStream, StreamLike, Scope}; +use crate::dataflow::{StreamCore, Scope}; /// Merge the contents of two streams. -pub trait Concat> { +pub trait Concat { /// Merge the contents of two streams. /// /// # Examples @@ -16,21 +16,21 @@ pub trait Concat> { /// timely::example(|scope| { /// /// let stream = (0..10).to_stream(scope).tee(); - /// stream.concat(&stream) + /// stream.owned().concat(stream.owned()) /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn concat(self, other: S) -> OwnedStream; + fn concat(self, other: StreamCore) -> StreamCore; } -impl> Concat for S { - fn concat(self, other: S) -> OwnedStream { +impl Concat for StreamCore { + fn concat(self, other: StreamCore) -> StreamCore { self.scope().concatenate([self, other]) } } /// Merge the contents of multiple streams. -pub trait Concatenate> { +pub trait Concatenate { /// Merge the contents of multiple streams. /// /// # Examples @@ -43,28 +43,29 @@ pub trait Concatenate> { /// (0..10).to_stream(scope), /// (0..10).to_stream(scope)]; /// - /// scope.concatenate(streams) + /// scope.clone() + /// .concatenate(streams) /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn concatenate(self, sources: I) -> OwnedStream + fn concatenate(self, sources: I) -> StreamCore where - I: IntoIterator; + I: IntoIterator>; } -impl Concatenate> for OwnedStream { - fn concatenate(self, sources: I) -> OwnedStream +impl Concatenate for StreamCore { + fn concatenate(self, sources: I) -> StreamCore where - I: IntoIterator> + I: IntoIterator> { self.scope().concatenate(Some(self).into_iter().chain(sources)) } } -impl> Concatenate for &G { - fn concatenate(self, sources: I) -> OwnedStream +impl Concatenate for G { + fn concatenate(self, sources: I) -> StreamCore where - I: IntoIterator + I: IntoIterator> { // create an operator builder. diff --git a/timely/src/dataflow/operators/core/enterleave.rs b/timely/src/dataflow/operators/core/enterleave.rs index 34ac6745cb..33d9a14530 100644 --- a/timely/src/dataflow/operators/core/enterleave.rs +++ b/timely/src/dataflow/operators/core/enterleave.rs @@ -31,7 +31,7 @@ use crate::dataflow::channels::pushers::{Counter, PushOwned}; use crate::dataflow::channels::Message; use crate::worker::AsWorker; -use crate::dataflow::{Scope, OwnedStream, StreamLike}; +use crate::dataflow::{StreamCore, Scope}; use crate::dataflow::scopes::Child; /// Extension trait to move a `Stream` into a child of its current `Scope`. @@ -50,11 +50,11 @@ pub trait Enter, C: Container> { /// }); /// }); /// ``` - fn enter<'a>(self, _: &Child<'a, G, T>) -> OwnedStream, C>; + fn enter<'a>(self, _: &Child<'a, G, T>) -> StreamCore, C>; } -impl, C: Container + 'static, S: StreamLike> Enter for S { - fn enter<'a>(self, scope: &Child<'a, G, T>) -> OwnedStream, C> { +impl, C: Container+'static> Enter for StreamCore { + fn enter<'a>(self, scope: &Child<'a, G, T>) -> StreamCore, C> { use crate::scheduling::Scheduler; @@ -76,7 +76,7 @@ impl, C: Container + 'static, S: St self.connect_to(input, ingress, channel_id); } - OwnedStream::new( + StreamCore::new( Source::new(0, input.port), registrar, scope.clone(), @@ -85,7 +85,7 @@ impl, C: Container + 'static, S: St } /// Extension trait to move a `Stream` to the parent of its current `Scope`. -pub trait Leave { +pub trait Leave { /// Moves a `Stream` to the parent of its current `Scope`. /// /// # Examples @@ -100,11 +100,11 @@ pub trait Leave { /// }); /// }); /// ``` - fn leave(self) -> OwnedStream; + fn leave(self) -> StreamCore; } -impl<'a, G: Scope, C: Container + 'static, T: Timestamp+Refines, S: StreamLike, C>> Leave for S { - fn leave(self) -> OwnedStream { +impl> Leave for StreamCore, C> { + fn leave(self) -> StreamCore { let scope = self.scope(); @@ -121,7 +121,7 @@ impl<'a, G: Scope, C: Container + 'static, T: Timestamp+Refines, S self.connect_to(target, egress, channel_id); } - OwnedStream::new( + StreamCore::new( output, registrar, scope.parent, diff --git a/timely/src/dataflow/operators/core/exchange.rs b/timely/src/dataflow/operators/core/exchange.rs index 14660c7237..403d106390 100644 --- a/timely/src/dataflow/operators/core/exchange.rs +++ b/timely/src/dataflow/operators/core/exchange.rs @@ -1,14 +1,13 @@ //! Exchange records between workers. -use crate::Container; use crate::ExchangeData; -use crate::container::{SizableContainer, PushInto}; +use crate::container::{Container, SizableContainer, PushInto}; use crate::dataflow::channels::pact::ExchangeCore; use crate::dataflow::operators::generic::operator::Operator; -use crate::dataflow::{Scope, OwnedStream, StreamLike}; +use crate::dataflow::{Scope, StreamCore}; /// Exchange records between workers. -pub trait Exchange { +pub trait Exchange { /// Exchange records between workers. /// /// The closure supplied should map a reference to a record to a `u64`, @@ -24,18 +23,17 @@ pub trait Exchange { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn exchange(self, route: F) -> OwnedStream + fn exchange(self, route: F) -> Self where for<'a> F: FnMut(&C::Item<'a>) -> u64 + 'static; } -impl Exchange for S +impl Exchange for StreamCore where C: SizableContainer + ExchangeData + crate::dataflow::channels::ContainerBytes, C: for<'a> PushInto>, - S: StreamLike { - fn exchange(self, route: F) -> OwnedStream + fn exchange(self, route: F) -> StreamCore where for<'a> F: FnMut(&C::Item<'a>) -> u64 + 'static, { diff --git a/timely/src/dataflow/operators/core/feedback.rs b/timely/src/dataflow/operators/core/feedback.rs index ee90bd3d42..a2a296b78c 100644 --- a/timely/src/dataflow/operators/core/feedback.rs +++ b/timely/src/dataflow/operators/core/feedback.rs @@ -7,7 +7,7 @@ use crate::dataflow::channels::pushers::PushOwned; use crate::dataflow::operators::generic::OutputWrapper; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::dataflow::scopes::child::Iterative; -use crate::dataflow::{Scope, OwnedStream, StreamLike}; +use crate::dataflow::{StreamCore, Scope}; use crate::order::Product; use crate::progress::frontier::Antichain; use crate::progress::{Timestamp, PathSummary}; @@ -36,7 +36,7 @@ pub trait Feedback { /// .connect_loop(handle); /// }); /// ``` - fn feedback(&mut self, summary: ::Summary) -> (Handle, OwnedStream); + fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore); } /// Creates a `StreamCore` and a `Handle` to later bind the source of that `StreamCore`. @@ -64,12 +64,12 @@ pub trait LoopVariable<'a, G: Scope, T: Timestamp> { /// }); /// }); /// ``` - fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, OwnedStream, C>); + fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>); } impl Feedback for G { - fn feedback(&mut self, summary: ::Summary) -> (Handle, OwnedStream) { + fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore) { let mut builder = OperatorBuilder::new("Feedback".to_owned(), self.clone()); let (output, stream) = builder.new_output(); @@ -79,7 +79,7 @@ impl Feedback for G { } impl<'a, G: Scope, T: Timestamp> LoopVariable<'a, G, T> for Iterative<'a, G, T> { - fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, OwnedStream, C>) { + fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>) { self.feedback(Product::new(Default::default(), summary)) } } @@ -106,7 +106,7 @@ pub trait ConnectLoop { fn connect_loop(self, handle: Handle); } -impl> ConnectLoop for S { +impl ConnectLoop for StreamCore { fn connect_loop(self, handle: Handle) { let mut builder = handle.builder; diff --git a/timely/src/dataflow/operators/core/filter.rs b/timely/src/dataflow/operators/core/filter.rs index 38294b624f..94ae71273b 100644 --- a/timely/src/dataflow/operators/core/filter.rs +++ b/timely/src/dataflow/operators/core/filter.rs @@ -1,11 +1,11 @@ //! Filters a stream by a predicate. use crate::container::{Container, SizableContainer, PushInto}; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{OwnedStream, Scope, StreamLike}; +use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::operators::generic::operator::Operator; /// Extension trait for filtering. -pub trait Filter { +pub trait Filter { /// Returns a new instance of `self` containing only records satisfying `predicate`. /// /// # Examples @@ -19,14 +19,14 @@ pub trait Filter { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn filter)->bool+'static>(self, predicate: P) -> OwnedStream; + fn filter)->bool+'static>(self, predicate: P) -> Self; } -impl> Filter for S +impl Filter for StreamCore where for<'a> C: PushInto> { - fn filter)->bool+'static>(self, mut predicate: P) -> OwnedStream { + fn filter)->bool+'static>(self, mut predicate: P) -> StreamCore { self.unary(Pipeline, "Filter", move |_,_| move |input, output| { input.for_each(|time, data| { if !data.is_empty() { diff --git a/timely/src/dataflow/operators/core/input.rs b/timely/src/dataflow/operators/core/input.rs index a8c6c08e61..21ecdf7035 100644 --- a/timely/src/dataflow/operators/core/input.rs +++ b/timely/src/dataflow/operators/core/input.rs @@ -13,7 +13,7 @@ use crate::progress::Source; use crate::Container; use crate::communication::Push; -use crate::dataflow::{OwnedStream, Scope, ScopeParent}; +use crate::dataflow::{Scope, ScopeParent, StreamCore}; use crate::dataflow::channels::pushers::{Counter, PushOwned}; use crate::dataflow::channels::Message; @@ -60,7 +60,7 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, OwnedStream); + fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore); /// Create a new [StreamCore] and [Handle] through which to supply input. /// @@ -97,7 +97,7 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn new_input_with_builder(&mut self) -> (Handle<::Timestamp, CB>, OwnedStream) + fn new_input_with_builder(&mut self) -> (Handle<::Timestamp, CB>, StreamCore) where CB::Container: Clone, ; @@ -133,7 +133,7 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn input_from(&mut self, handle: &mut Handle<::Timestamp, CB>) -> OwnedStream + fn input_from(&mut self, handle: &mut Handle<::Timestamp, CB>) -> StreamCore where CB::Container: Clone, ; @@ -141,13 +141,13 @@ pub trait Input : Scope { use crate::order::TotalOrder; impl Input for G where ::Timestamp: TotalOrder { - fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, OwnedStream) { + fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore) { let mut handle = Handle::new(); let stream = self.input_from(&mut handle); (handle, stream) } - fn new_input_with_builder(&mut self) -> (Handle<::Timestamp, CB>, OwnedStream) + fn new_input_with_builder(&mut self) -> (Handle<::Timestamp, CB>, StreamCore) where CB::Container: Clone, { @@ -156,7 +156,7 @@ impl Input for G where ::Timestamp: TotalOrder { (handle, stream) } - fn input_from(&mut self, handle: &mut Handle<::Timestamp, CB>) -> OwnedStream + fn input_from(&mut self, handle: &mut Handle<::Timestamp, CB>) -> StreamCore where CB::Container: Clone, { @@ -184,7 +184,7 @@ impl Input for G where ::Timestamp: TotalOrder { copies, }), index); - OwnedStream::new(Source::new(index, 0), registrar, self.clone()) + StreamCore::new(Source::new(index, 0), registrar, self.clone()) } } @@ -350,7 +350,7 @@ where /// } /// }); /// ``` - pub fn to_stream(&mut self, scope: &mut G) -> OwnedStream + pub fn to_stream(&mut self, scope: &mut G) -> StreamCore where T: TotalOrder, G: Scope, diff --git a/timely/src/dataflow/operators/core/inspect.rs b/timely/src/dataflow/operators/core/inspect.rs index 09428465b4..5e28eab55a 100644 --- a/timely/src/dataflow/operators/core/inspect.rs +++ b/timely/src/dataflow/operators/core/inspect.rs @@ -2,7 +2,7 @@ use crate::Container; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{Scope, OwnedStream, StreamLike}; +use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::operators::generic::Operator; /// Methods to inspect records and batches of records on a stream. @@ -18,7 +18,7 @@ pub trait Inspect: InspectCore + Sized { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn inspect(self, mut func: F) -> OwnedStream + fn inspect(self, mut func: F) -> Self where F: for<'a> FnMut(C::ItemRef<'a>) + 'static, { @@ -38,7 +38,7 @@ pub trait Inspect: InspectCore + Sized { /// .inspect_time(|t, x| println!("seen at: {:?}\t{:?}", t, x)); /// }); /// ``` - fn inspect_time(self, mut func: F) -> OwnedStream + fn inspect_time(self, mut func: F) -> Self where F: for<'a> FnMut(&G::Timestamp, C::ItemRef<'a>) + 'static, { @@ -60,7 +60,7 @@ pub trait Inspect: InspectCore + Sized { /// .inspect_batch(|t,xs| println!("seen at: {:?}\t{:?} records", t, xs.len())); /// }); /// ``` - fn inspect_batch(self, mut func: impl FnMut(&G::Timestamp, &C)+'static) -> OwnedStream { + fn inspect_batch(self, mut func: impl FnMut(&G::Timestamp, &C)+'static) -> Self { self.inspect_core(move |event| { if let Ok((time, data)) = event { func(time, data); @@ -87,11 +87,11 @@ pub trait Inspect: InspectCore + Sized { /// }); /// }); /// ``` - fn inspect_core(self, func: F) -> OwnedStream where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; + fn inspect_core(self, func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl> Inspect for S { - fn inspect_core(self, func: F) -> OwnedStream where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>) + 'static { +impl Inspect for StreamCore { + fn inspect_core(self, func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>) + 'static { self.inspect_container(func) } } @@ -117,12 +117,12 @@ pub trait InspectCore { /// }); /// }); /// ``` - fn inspect_container(self, func: F) -> OwnedStream where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; + fn inspect_container(self, func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl> InspectCore for S { +impl InspectCore for StreamCore { - fn inspect_container(self, mut func: F) -> OwnedStream + fn inspect_container(self, mut func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static { use crate::progress::timestamp::Timestamp; diff --git a/timely/src/dataflow/operators/core/map.rs b/timely/src/dataflow/operators/core/map.rs index f560a0c853..4b5bd7e6f8 100644 --- a/timely/src/dataflow/operators/core/map.rs +++ b/timely/src/dataflow/operators/core/map.rs @@ -1,12 +1,12 @@ //! Extension methods for `StreamCore` based on record-by-record transformation. use crate::container::{Container, SizableContainer, PushInto}; -use crate::dataflow::{OwnedStream, Scope, StreamLike}; +use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::operator::Operator; /// Extension trait for `Stream`. -pub trait Map : Sized { +pub trait Map : Sized { /// Consumes each element of the stream and yields a new element. /// /// # Examples @@ -21,7 +21,7 @@ pub trait Map : Sized { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn map(self, mut logic: L) -> OwnedStream + fn map(self, mut logic: L) -> StreamCore where C2: SizableContainer + PushInto + 'static, L: FnMut(C::Item<'_>)->D2 + 'static, @@ -42,7 +42,7 @@ pub trait Map : Sized { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn flat_map(self, logic: L) -> OwnedStream + fn flat_map(self, logic: L) -> StreamCore where I: IntoIterator, C2: SizableContainer + PushInto + 'static, @@ -50,11 +50,11 @@ pub trait Map : Sized { ; } -impl> Map for S { +impl Map for StreamCore { // TODO : This would be more robust if it captured an iterator and then pulled an appropriate // TODO : number of elements from the iterator. This would allow iterators that produce many // TODO : records without taking arbitrarily long and arbitrarily much memory. - fn flat_map(self, mut logic: L) -> OwnedStream + fn flat_map(self, mut logic: L) -> StreamCore where I: IntoIterator, C2: SizableContainer + PushInto + 'static, diff --git a/timely/src/dataflow/operators/core/ok_err.rs b/timely/src/dataflow/operators/core/ok_err.rs index 6ba7cffcd9..1b6aa74b11 100644 --- a/timely/src/dataflow/operators/core/ok_err.rs +++ b/timely/src/dataflow/operators/core/ok_err.rs @@ -3,7 +3,7 @@ use crate::container::{Container, SizableContainer, PushInto}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; -use crate::dataflow::{Scope, OwnedStream, StreamLike}; +use crate::dataflow::{Scope, StreamCore}; /// Extension trait for `Stream`. pub trait OkErr { @@ -27,7 +27,10 @@ pub trait OkErr { /// odd.container::>().inspect(|x| println!("odd: {:?}", x)); /// }); /// ``` - fn ok_err(self, logic: L) -> (OwnedStream, OwnedStream) + fn ok_err( + self, + logic: L, + ) -> (StreamCore, StreamCore) where C1: SizableContainer + PushInto + 'static, C2: SizableContainer + PushInto + 'static, @@ -35,8 +38,11 @@ pub trait OkErr { ; } -impl> OkErr for S { - fn ok_err(self, mut logic: L) -> (OwnedStream, OwnedStream) +impl OkErr for StreamCore { + fn ok_err( + self, + mut logic: L, + ) -> (StreamCore, StreamCore) where C1: SizableContainer + PushInto + 'static, C2: SizableContainer + PushInto + 'static, diff --git a/timely/src/dataflow/operators/core/probe.rs b/timely/src/dataflow/operators/core/probe.rs index 25989c6529..49909a4570 100644 --- a/timely/src/dataflow/operators/core/probe.rs +++ b/timely/src/dataflow/operators/core/probe.rs @@ -10,7 +10,7 @@ use crate::dataflow::channels::pushers::buffer::Buffer as PushBuffer; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::channels::pullers::Counter as PullCounter; use crate::dataflow::operators::generic::builder_raw::OperatorBuilder; -use crate::dataflow::{Scope, OwnedStream, StreamLike}; +use crate::dataflow::{StreamCore, Scope}; use crate::Container; /// Monitors progress at a `Stream`. @@ -74,10 +74,10 @@ pub trait Probe { /// } /// }).unwrap(); /// ``` - fn probe_with(self, handle: &Handle) -> OwnedStream; + fn probe_with(self, handle: &Handle) -> StreamCore; } -impl> Probe for S { +impl Probe for StreamCore { fn probe(self) -> Handle { // the frontier is shared state; scope updates, handle reads. @@ -85,7 +85,7 @@ impl> Probe for S { self.probe_with(&handle); handle } - fn probe_with(self, handle: &Handle) -> OwnedStream { + fn probe_with(self, handle: &Handle) -> StreamCore { let mut builder = OperatorBuilder::new("Probe".to_owned(), self.scope()); let mut input = PullCounter::new(builder.new_input(self, Pipeline)); diff --git a/timely/src/dataflow/operators/core/rc.rs b/timely/src/dataflow/operators/core/rc.rs index e5a3e297ca..b90d52493b 100644 --- a/timely/src/dataflow/operators/core/rc.rs +++ b/timely/src/dataflow/operators/core/rc.rs @@ -3,11 +3,11 @@ use std::rc::Rc; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::Operator; -use crate::dataflow::{OwnedStream, Scope, StreamLike}; +use crate::dataflow::{Scope, StreamCore}; use crate::Container; /// Convert a stream into a stream of shared containers -pub trait SharedStream { +pub trait SharedStream { /// Convert a stream into a stream of shared data /// /// # Examples @@ -21,11 +21,11 @@ pub trait SharedStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn shared(self) -> OwnedStream>; + fn shared(self) -> StreamCore>; } -impl> SharedStream for S { - fn shared(self) -> OwnedStream> { +impl SharedStream for StreamCore { + fn shared(self) -> StreamCore> { self.unary(Pipeline, "Shared", move |_, _| { move |input, output| { input.for_each(|time, data| { @@ -50,15 +50,16 @@ mod test { let output = crate::example(|scope| { let shared = vec![Ok(0), Err(())].to_stream(scope).container::>().shared().tee(); scope + .clone() .concatenate([ - shared.unary(Pipeline, "read shared 1", |_, _| { + shared.owned().unary(Pipeline, "read shared 1", |_, _| { move |input, output| { input.for_each(|time, data| { output.session(&time).give(data.as_ptr() as usize); }); } }), - shared.unary(Pipeline, "read shared 2", |_, _| { + shared.owned().unary(Pipeline, "read shared 2", |_, _| { move |input, output| { input.for_each(|time, data| { output.session(&time).give(data.as_ptr() as usize); diff --git a/timely/src/dataflow/operators/core/reclock.rs b/timely/src/dataflow/operators/core/reclock.rs index 697b3ebdd7..cd1bf39d60 100644 --- a/timely/src/dataflow/operators/core/reclock.rs +++ b/timely/src/dataflow/operators/core/reclock.rs @@ -2,12 +2,12 @@ use crate::Container; use crate::order::PartialOrder; -use crate::dataflow::{Scope, OwnedStream, StreamLike}; +use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::operator::Operator; /// Extension trait for reclocking a stream. -pub trait Reclock { +pub trait Reclock { /// Delays records until an input is observed on the `clock` input. /// /// The source stream is buffered until a record is seen on the clock input, @@ -45,11 +45,11 @@ pub trait Reclock { /// assert_eq!(extracted[1], (5, vec![4,5])); /// assert_eq!(extracted[2], (8, vec![6,7,8])); /// ``` - fn reclock>(self, clock: CS) -> OwnedStream; + fn reclock(self, clock: StreamCore) -> Self; } -impl> Reclock for S { - fn reclock>(self, clock: CS) -> OwnedStream { +impl Reclock for StreamCore { + fn reclock(self, clock: StreamCore) -> StreamCore { let mut stash = vec![]; diff --git a/timely/src/dataflow/operators/core/to_stream.rs b/timely/src/dataflow/operators/core/to_stream.rs index 4c2b918463..0a6fcf0eb8 100644 --- a/timely/src/dataflow/operators/core/to_stream.rs +++ b/timely/src/dataflow/operators/core/to_stream.rs @@ -3,9 +3,9 @@ use crate::container::{CapacityContainerBuilder, ContainerBuilder, SizableContainer, PushInto}; use crate::Container; use crate::dataflow::operators::generic::operator::source; -use crate::dataflow::{Scope, OwnedStream}; +use crate::dataflow::{StreamCore, Scope}; -/// Converts to a timely stream, using a container builder. +/// Converts to a timely [StreamCore], using a container builder. pub trait ToStreamBuilder { /// Converts to a timely [StreamCore], using the supplied container builder type. /// @@ -28,11 +28,11 @@ pub trait ToStreamBuilder { /// /// assert_eq!(data1.extract(), data2.extract()); /// ``` - fn to_stream_with_builder(self, scope: &mut S) -> OwnedStream; + fn to_stream_with_builder(self, scope: &mut S) -> StreamCore; } impl ToStreamBuilder for I where CB: PushInto { - fn to_stream_with_builder(self, scope: &mut S) -> OwnedStream { + fn to_stream_with_builder(self, scope: &mut S) -> StreamCore { source::<_, CB, _, _>(scope, "ToStreamBuilder", |capability, info| { @@ -78,11 +78,11 @@ pub trait ToStream { /// /// assert_eq!(data1.extract(), data2.extract()); /// ``` - fn to_stream(self, scope: &mut S) -> OwnedStream; + fn to_stream(self, scope: &mut S) -> StreamCore; } impl ToStream for I where C: PushInto { - fn to_stream(self, scope: &mut S) -> OwnedStream { + fn to_stream(self, scope: &mut S) -> StreamCore { ToStreamBuilder::>::to_stream_with_builder(self, scope) } } diff --git a/timely/src/dataflow/operators/core/unordered_input.rs b/timely/src/dataflow/operators/core/unordered_input.rs index 6788a53e85..a0f457098c 100644 --- a/timely/src/dataflow/operators/core/unordered_input.rs +++ b/timely/src/dataflow/operators/core/unordered_input.rs @@ -17,11 +17,11 @@ use crate::dataflow::channels::pushers::buffer::{Buffer as PushBuffer, Autoflush use crate::dataflow::operators::{ActivateCapability, Capability}; -use crate::dataflow::{OwnedStream, Scope}; +use crate::dataflow::{Scope, StreamCore}; /// Create a new `Stream` and `Handle` through which to supply input. pub trait UnorderedInput { - /// Create a new capability-based stream and [UnorderedHandle] through which to supply input. This + /// Create a new capability-based [StreamCore] and [UnorderedHandle] through which to supply input. This /// input supports multiple open epochs (timestamps) at the same time. /// /// The `new_unordered_input_core` method returns `((HandleCore, Capability), StreamCore)` where the `StreamCore` can be used @@ -76,11 +76,11 @@ pub trait UnorderedInput { /// assert_eq!(extract[i], (i, vec![i])); /// } /// ``` - fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream); + fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), StreamCore); } impl UnorderedInput for G { - fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream) { + fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), StreamCore) { let (output, registrar) = PushOwned::::new(); let internal = Rc::new(RefCell::new(ChangeBatch::new())); @@ -106,7 +106,7 @@ impl UnorderedInput for G { peers, }), index); - ((helper, cap), OwnedStream::new(Source::new(index, 0), registrar, self.clone())) + ((helper, cap), StreamCore::new(Source::new(index, 0), registrar, self.clone())) } } diff --git a/timely/src/dataflow/operators/count.rs b/timely/src/dataflow/operators/count.rs index 625deddbc9..9469bd81c5 100644 --- a/timely/src/dataflow/operators/count.rs +++ b/timely/src/dataflow/operators/count.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{OwnedStream, Scope, StreamLike}; +use crate::dataflow::{Stream, Scope}; use crate::dataflow::operators::generic::operator::Operator; /// Accumulates records within a timestamp. @@ -24,7 +24,7 @@ pub trait Accumulate: Sized { /// let extracted = captured.extract(); /// assert_eq!(extracted, vec![(0, vec![45])]); /// ``` - fn accumulate(self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> OwnedStream>; + fn accumulate(self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> Stream; /// Counts the number of records observed at each time. /// /// # Examples @@ -42,13 +42,13 @@ pub trait Accumulate: Sized { /// let extracted = captured.extract(); /// assert_eq!(extracted, vec![(0, vec![10])]); /// ``` - fn count(self) -> OwnedStream> { + fn count(self) -> Stream { self.accumulate(0, |sum, data| *sum += data.len()) } } -impl>> Accumulate for S { - fn accumulate(self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> OwnedStream> { +impl Accumulate for Stream { + fn accumulate(self, default: A, logic: impl Fn(&mut A, &mut Vec)+'static) -> Stream { let mut accums = HashMap::new(); self.unary_notify(Pipeline, "Accumulate", vec![], move |input, output, notificator| { diff --git a/timely/src/dataflow/operators/delay.rs b/timely/src/dataflow/operators/delay.rs index e506e63549..63785e74c1 100644 --- a/timely/src/dataflow/operators/delay.rs +++ b/timely/src/dataflow/operators/delay.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use crate::order::{PartialOrder, TotalOrder}; use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{OwnedStream, StreamLike, Scope}; +use crate::dataflow::{Stream, Scope}; use crate::dataflow::operators::generic::operator::Operator; /// Methods to advance the timestamps of records or batches of records. @@ -35,7 +35,7 @@ pub trait Delay { /// }); /// }); /// ``` - fn delayG::Timestamp+'static>(self, func: L) -> OwnedStream>; + fn delayG::Timestamp+'static>(self, func: L) -> Self; /// Advances the timestamp of records using a supplied function. /// @@ -62,7 +62,7 @@ pub trait Delay { /// }); /// }); /// ``` - fn delay_totalG::Timestamp+'static>(self, func: L) -> OwnedStream> + fn delay_totalG::Timestamp+'static>(self, func: L) -> Self where G::Timestamp: TotalOrder; /// Advances the timestamp of batches of records using a supplied function. @@ -90,11 +90,11 @@ pub trait Delay { /// }); /// }); /// ``` - fn delay_batchG::Timestamp+'static>(self, func: L) -> OwnedStream>; + fn delay_batchG::Timestamp+'static>(self, func: L) -> Self; } -impl>> Delay for S { - fn delayG::Timestamp+'static>(self, mut func: L) -> OwnedStream> { +impl Delay for Stream { + fn delayG::Timestamp+'static>(self, mut func: L) -> Self { let mut elements = HashMap::new(); self.unary_notify(Pipeline, "Delay", vec![], move |input, output, notificator| { input.for_each(|time, data| { @@ -116,13 +116,13 @@ impl>> Delay for S { }) } - fn delay_totalG::Timestamp+'static>(self, func: L) -> OwnedStream> + fn delay_totalG::Timestamp+'static>(self, func: L) -> Self where G::Timestamp: TotalOrder { self.delay(func) } - fn delay_batchG::Timestamp+'static>(self, mut func: L) -> OwnedStream> { + fn delay_batchG::Timestamp+'static>(self, mut func: L) -> Self { let mut elements = HashMap::new(); self.unary_notify(Pipeline, "Delay", vec![], move |input, output, notificator| { input.for_each(|time, data| { diff --git a/timely/src/dataflow/operators/filter.rs b/timely/src/dataflow/operators/filter.rs index 6f6461213d..ff7c4259ff 100644 --- a/timely/src/dataflow/operators/filter.rs +++ b/timely/src/dataflow/operators/filter.rs @@ -1,11 +1,11 @@ //! Filters a stream by a predicate. use crate::dataflow::channels::pact::Pipeline; -use crate::dataflow::{OwnedStream, StreamLike, Scope}; +use crate::dataflow::{Stream, Scope}; use crate::dataflow::operators::generic::operator::Operator; /// Extension trait for filtering. -pub trait Filter { +pub trait Filter { /// Returns a new instance of `self` containing only records satisfying `predicate`. /// /// # Examples @@ -18,11 +18,11 @@ pub trait Filter { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn filterbool+'static>(self, predicate: P) -> OwnedStream>; + fn filterbool+'static>(self, predicate: P) -> Self; } -impl>> Filter for S { - fn filterbool+'static>(self, mut predicate: P) -> OwnedStream> { +impl Filter for Stream { + fn filterbool+'static>(self, mut predicate: P) -> Stream { self.unary(Pipeline, "Filter", move |_,_| move |input, output| { input.for_each(|time, data| { data.retain(|x| predicate(x)); diff --git a/timely/src/dataflow/operators/flow_controlled.rs b/timely/src/dataflow/operators/flow_controlled.rs index 1c87193484..814bfbc000 100644 --- a/timely/src/dataflow/operators/flow_controlled.rs +++ b/timely/src/dataflow/operators/flow_controlled.rs @@ -4,7 +4,7 @@ use crate::order::{PartialOrder, TotalOrder}; use crate::progress::timestamp::Timestamp; use crate::dataflow::operators::generic::operator::source; use crate::dataflow::operators::probe::Handle; -use crate::dataflow::{OwnedStream, Scope}; +use crate::dataflow::{Stream, Scope}; /// Output of the input reading function for iterator_source. pub struct IteratorSourceInput, I: IntoIterator> { @@ -80,7 +80,7 @@ pub fn iterator_source< name: &str, mut input_f: F, probe: Handle, - ) -> OwnedStream> where G::Timestamp: TotalOrder { + ) -> Stream where G::Timestamp: TotalOrder { let mut target = G::Timestamp::minimum(); source(scope, name, |cap, info| { diff --git a/timely/src/dataflow/operators/generic/builder_raw.rs b/timely/src/dataflow/operators/generic/builder_raw.rs index a557eb0f9a..da91517af9 100644 --- a/timely/src/dataflow/operators/generic/builder_raw.rs +++ b/timely/src/dataflow/operators/generic/builder_raw.rs @@ -14,7 +14,7 @@ use crate::progress::{Source, Target}; use crate::progress::{Timestamp, Operate, operate::SharedProgress, Antichain}; use crate::Container; -use crate::dataflow::{OwnedStream, StreamLike, Scope}; +use crate::dataflow::{StreamCore, Scope}; use crate::dataflow::channels::pact::ParallelizationContract; use crate::dataflow::channels::pushers::PushOwned; use crate::dataflow::operators::generic::operator_info::OperatorInfo; @@ -104,7 +104,7 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input>(&mut self, stream: S, pact: P) -> P::Puller + pub fn new_input(&mut self, stream: StreamCore, pact: P) -> P::Puller where P: ParallelizationContract { let connection = vec![Antichain::from_elem(Default::default()); self.shape.outputs]; @@ -112,7 +112,7 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input_connection>(&mut self, stream: S, pact: P, connection: Vec::Summary>>) -> P::Puller + pub fn new_input_connection(&mut self, stream: StreamCore, pact: P, connection: Vec::Summary>>) -> P::Puller where P: ParallelizationContract { @@ -130,18 +130,18 @@ impl OperatorBuilder { } /// Adds a new output to a generic operator builder, returning the `Push` implementor to use. - pub fn new_output(&mut self) -> (PushOwned, OwnedStream) { + pub fn new_output(&mut self) -> (PushOwned, StreamCore) { let connection = vec![Antichain::from_elem(Default::default()); self.shape.inputs]; self.new_output_connection(connection) } /// Adds a new output to a generic operator builder, returning the `Push` implementor to use. - pub fn new_output_connection(&mut self, connection: Vec::Summary>>) -> (PushOwned, OwnedStream) { + pub fn new_output_connection(&mut self, connection: Vec::Summary>>) -> (PushOwned, StreamCore) { let (target, registrar) = PushOwned::new(); let source = Source::new(self.index, self.shape.outputs); - let stream = OwnedStream::new(source, registrar, self.scope.clone()); + let stream = StreamCore::new(source, registrar, self.scope.clone()); self.shape.outputs += 1; assert_eq!(self.shape.inputs, connection.len()); diff --git a/timely/src/dataflow/operators/generic/builder_rc.rs b/timely/src/dataflow/operators/generic/builder_rc.rs index 8b09cfc2cb..e20c594e06 100644 --- a/timely/src/dataflow/operators/generic/builder_rc.rs +++ b/timely/src/dataflow/operators/generic/builder_rc.rs @@ -10,7 +10,7 @@ use crate::progress::frontier::{Antichain, MutableAntichain}; use crate::Container; use crate::container::ContainerBuilder; -use crate::dataflow::{Scope, OwnedStream, StreamLike}; +use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pushers::Counter as PushCounter; use crate::dataflow::channels::pushers::buffer::Buffer as PushBuffer; use crate::dataflow::channels::pact::ParallelizationContract; @@ -60,7 +60,7 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input>(&mut self, stream: S, pact: P) -> InputHandleCore + pub fn new_input(&mut self, stream: StreamCore, pact: P) -> InputHandleCore where P: ParallelizationContract { @@ -76,7 +76,7 @@ impl OperatorBuilder { /// /// Commonly the connections are either the unit summary, indicating the same timestamp might be produced as output, or an empty /// antichain indicating that there is no connection from the input to the output. - pub fn new_input_connection>(&mut self, stream: S, pact: P, connection: Vec::Summary>>) -> InputHandleCore + pub fn new_input_connection(&mut self, stream: StreamCore, pact: P, connection: Vec::Summary>>) -> InputHandleCore where P: ParallelizationContract { @@ -93,7 +93,7 @@ impl OperatorBuilder { } /// Adds a new output to a generic operator builder, returning the `Push` implementor to use. - pub fn new_output(&mut self) -> (OutputWrapper>, OwnedStream) { + pub fn new_output(&mut self) -> (OutputWrapper>, StreamCore) { let connection = vec![Antichain::from_elem(Default::default()); self.builder.shape().inputs()]; self.new_output_connection(connection) } @@ -111,7 +111,7 @@ impl OperatorBuilder { connection: Vec::Summary>> ) -> ( OutputWrapper>, - OwnedStream + StreamCore ) { let (tee, stream) = self.builder.new_output_connection(connection.clone()); diff --git a/timely/src/dataflow/operators/generic/operator.rs b/timely/src/dataflow/operators/generic/operator.rs index 33b619b77f..300afddf61 100644 --- a/timely/src/dataflow/operators/generic/operator.rs +++ b/timely/src/dataflow/operators/generic/operator.rs @@ -7,7 +7,7 @@ use crate::dataflow::channels::pact::ParallelizationContract; use crate::dataflow::operators::generic::handles::{InputHandleCore, FrontieredInputHandleCore, OutputHandleCore}; use crate::dataflow::operators::capability::Capability; -use crate::dataflow::{Scope, OwnedStream, StreamLike}; +use crate::dataflow::{Scope, StreamCore}; use super::builder_rc::OperatorBuilder; use crate::dataflow::operators::generic::OperatorInfo; @@ -55,7 +55,7 @@ pub trait Operator { /// }); /// } /// ``` - fn unary_frontier(self, pact: P, name: &str, constructor: B) -> OwnedStream + fn unary_frontier(self, pact: P, name: &str, constructor: B) -> StreamCore where CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, @@ -95,7 +95,7 @@ pub trait Operator { &mut OutputHandleCore>, &mut Notificator)+'static, P: ParallelizationContract> - (self, pact: P, name: &str, init: impl IntoIterator, logic: L) -> OwnedStream; + (self, pact: P, name: &str, init: impl IntoIterator, logic: L) -> StreamCore; /// Creates a new dataflow operator that partitions its input stream by a parallelization /// strategy `pact`, and repeatedly invokes `logic`, the function returned by the function passed as `constructor`. @@ -123,7 +123,7 @@ pub trait Operator { /// }); /// }); /// ``` - fn unary(self, pact: P, name: &str, constructor: B) -> OwnedStream + fn unary(self, pact: P, name: &str, constructor: B) -> StreamCore where CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, @@ -179,11 +179,10 @@ pub trait Operator { /// } /// }).unwrap(); /// ``` - fn binary_frontier(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream + fn binary_frontier(self, other: StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where C2: Container + 'static, CB: ContainerBuilder, - O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut FrontieredInputHandleCore, &mut FrontieredInputHandleCore, @@ -234,14 +233,13 @@ pub trait Operator { /// ``` fn binary_notify, L: FnMut(&mut InputHandleCore, &mut InputHandleCore, &mut OutputHandleCore>, &mut Notificator)+'static, P1: ParallelizationContract, P2: ParallelizationContract> - (self, other: O, pact1: P1, pact2: P2, name: &str, init: impl IntoIterator, logic: L) -> OwnedStream; + (self, other: StreamCore, pact1: P1, pact2: P2, name: &str, init: impl IntoIterator, logic: L) -> StreamCore; /// Creates a new dataflow operator that partitions its input streams by a parallelization /// strategy `pact`, and repeatedly invokes `logic`, the function returned by the function passed as `constructor`. @@ -273,11 +271,10 @@ pub trait Operator { /// }).inspect(|x| println!("{:?}", x)); /// }); /// ``` - fn binary(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream + fn binary(self, other: StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where C2: Container + 'static, CB: ContainerBuilder, - O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut InputHandleCore, &mut InputHandleCore, @@ -314,9 +311,9 @@ pub trait Operator { P: ParallelizationContract; } -impl> Operator for S { +impl Operator for StreamCore { - fn unary_frontier(self, pact: P, name: &str, constructor: B) -> OwnedStream + fn unary_frontier(self, pact: P, name: &str, constructor: B) -> StreamCore where CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, @@ -349,7 +346,7 @@ impl> Operator fo &mut OutputHandleCore>, &mut Notificator)+'static, P: ParallelizationContract> - (self, pact: P, name: &str, init: impl IntoIterator, mut logic: L) -> OwnedStream { + (self, pact: P, name: &str, init: impl IntoIterator, mut logic: L) -> StreamCore { let logging = self.scope().logging(); self.unary_frontier(pact, name, move |capability, _info| { @@ -366,7 +363,7 @@ impl> Operator fo }) } - fn unary(self, pact: P, name: &str, constructor: B) -> OwnedStream + fn unary(self, pact: P, name: &str, constructor: B) -> StreamCore where CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, @@ -394,11 +391,10 @@ impl> Operator fo stream } - fn binary_frontier(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream + fn binary_frontier(self, other: StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where C2: Container + 'static, CB: ContainerBuilder, - O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut FrontieredInputHandleCore, &mut FrontieredInputHandleCore, @@ -430,14 +426,13 @@ impl> Operator fo fn binary_notify, L: FnMut(&mut InputHandleCore, &mut InputHandleCore, &mut OutputHandleCore>, &mut Notificator)+'static, P1: ParallelizationContract, P2: ParallelizationContract> - (self, other: O, pact1: P1, pact2: P2, name: &str, init: impl IntoIterator, mut logic: L) -> OwnedStream { + (self, other: StreamCore, pact1: P1, pact2: P2, name: &str, init: impl IntoIterator, mut logic: L) -> StreamCore { let logging = self.scope().logging(); self.binary_frontier(other, pact1, pact2, name, |capability, _info| { @@ -456,11 +451,10 @@ impl> Operator fo } - fn binary(self, other: O, pact1: P1, pact2: P2, name: &str, constructor: B) -> OwnedStream + fn binary(self, other: StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where C2: Container + 'static, CB: ContainerBuilder, - O: StreamLike, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut InputHandleCore, &mut InputHandleCore, @@ -548,7 +542,7 @@ impl> Operator fo /// .inspect(|x| println!("number: {:?}", x)); /// }); /// ``` -pub fn source(scope: &G, name: &str, constructor: B) -> OwnedStream +pub fn source(scope: &G, name: &str, constructor: B) -> StreamCore where CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, @@ -592,7 +586,7 @@ where /// /// }); /// ``` -pub fn empty(scope: &G) -> OwnedStream { +pub fn empty(scope: &G) -> StreamCore { source::<_, CapacityContainerBuilder, _, _>(scope, "Empty", |_capability, _info| |_output| { // drop capability, do nothing }) diff --git a/timely/src/dataflow/operators/input.rs b/timely/src/dataflow/operators/input.rs index e87881739e..ac1afc0452 100644 --- a/timely/src/dataflow/operators/input.rs +++ b/timely/src/dataflow/operators/input.rs @@ -1,7 +1,7 @@ //! Create new `Streams` connected to external inputs. use crate::container::CapacityContainerBuilder; -use crate::dataflow::{ScopeParent, Scope, OwnedStream}; +use crate::dataflow::{Stream, ScopeParent, Scope}; use crate::dataflow::operators::core::{Input as InputCore}; // TODO : This is an exogenous input, but it would be nice to wrap a Subgraph in something @@ -46,7 +46,7 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn new_input(&mut self) -> (Handle<::Timestamp, D>, OwnedStream>); + fn new_input(&mut self) -> (Handle<::Timestamp, D>, Stream); /// Create a new stream from a supplied interactive handle. /// @@ -78,16 +78,16 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> OwnedStream>; + fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> Stream; } use crate::order::TotalOrder; impl Input for G where ::Timestamp: TotalOrder { - fn new_input(&mut self) -> (Handle<::Timestamp, D>, OwnedStream>) { + fn new_input(&mut self) -> (Handle<::Timestamp, D>, Stream) { InputCore::new_input(self) } - fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> OwnedStream> { + fn input_from(&mut self, handle: &mut Handle<::Timestamp, D>) -> Stream { InputCore::input_from(self, handle) } } diff --git a/timely/src/dataflow/operators/map.rs b/timely/src/dataflow/operators/map.rs index 5aee3396a4..425b9f208d 100644 --- a/timely/src/dataflow/operators/map.rs +++ b/timely/src/dataflow/operators/map.rs @@ -1,12 +1,12 @@ //! Extension methods for `Stream` based on record-by-record transformation. -use crate::dataflow::{OwnedStream, StreamLike, Scope}; +use crate::dataflow::{Stream, Scope}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::operator::Operator; use crate::dataflow::operators::core::{Map as MapCore}; /// Extension trait for `Stream`. -pub trait Map : Sized { +pub trait Map : Sized { /// Consumes each element of the stream and yields a new element. /// /// # Examples @@ -19,7 +19,7 @@ pub trait Map : Sized { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn mapD2+'static>(self, mut logic: L) -> OwnedStream> { + fn mapD2+'static>(self, mut logic: L) -> Stream { self.flat_map(move |x| std::iter::once(logic(x))) } /// Updates each element of the stream and yields the element, re-using memory where possible. @@ -34,7 +34,7 @@ pub trait Map : Sized { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn map_in_place(self, logic: L) -> OwnedStream>; + fn map_in_place(self, logic: L) -> Stream; /// Consumes each element of the stream and yields some number of new elements. /// /// # Examples @@ -47,11 +47,11 @@ pub trait Map : Sized { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn flat_mapI+'static>(self, logic: L) -> OwnedStream> where I::Item: 'static; + fn flat_mapI+'static>(self, logic: L) -> Stream where I::Item: 'static; } -impl>> Map for S { - fn map_in_place(self, mut logic: L) -> OwnedStream> { +impl Map for Stream { + fn map_in_place(self, mut logic: L) -> Stream { self.unary(Pipeline, "MapInPlace", move |_,_| move |input, output| { input.for_each(|time, data| { for datum in data.iter_mut() { logic(datum); } @@ -62,7 +62,7 @@ impl>> Map for S { // TODO : This would be more robust if it captured an iterator and then pulled an appropriate // TODO : number of elements from the iterator. This would allow iterators that produce many // TODO : records without taking arbitrarily long and arbitrarily much memory. - fn flat_mapI+'static>(self, logic: L) -> OwnedStream> where I::Item: 'static { + fn flat_mapI+'static>(self, logic: L) -> Stream where I::Item: 'static { MapCore::flat_map(self, logic) } } diff --git a/timely/src/dataflow/operators/partition.rs b/timely/src/dataflow/operators/partition.rs index c796f9ba73..1077675923 100644 --- a/timely/src/dataflow/operators/partition.rs +++ b/timely/src/dataflow/operators/partition.rs @@ -2,7 +2,7 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; -use crate::dataflow::{Scope, OwnedStream, StreamLike}; +use crate::dataflow::{Scope, Stream}; /// Partition a stream of records into multiple streams. pub trait Partition (u64, D2)> { @@ -21,18 +21,11 @@ pub trait Partition (u64, D2)> { /// } /// }); /// ``` - fn partition(self, parts: u64, route: F) -> Vec>>; + fn partition(self, parts: u64, route: F) -> Vec>; } -impl Partition for S -where - G: Scope, - D: 'static, - D2: 'static, - F: Fn(D) -> (u64, D2) + 'static, - S: StreamLike>, -{ - fn partition(self, parts: u64, route: F) -> Vec>> { +impl(u64, D2)+'static> Partition for Stream { + fn partition(self, parts: u64, route: F) -> Vec> { let mut builder = OperatorBuilder::new("Partition".to_owned(), self.scope()); let mut input = builder.new_input(self, Pipeline); diff --git a/timely/src/dataflow/operators/result.rs b/timely/src/dataflow/operators/result.rs index 907b768161..b57124956f 100644 --- a/timely/src/dataflow/operators/result.rs +++ b/timely/src/dataflow/operators/result.rs @@ -1,10 +1,10 @@ //! Extension methods for `Stream` containing `Result`s. use crate::dataflow::operators::Map; -use crate::dataflow::{Scope, StreamLike, OwnedStream}; +use crate::dataflow::{Scope, Stream}; /// Extension trait for `Stream`. -pub trait ResultStream { +pub trait ResultStream { /// Returns a new instance of `self` containing only `ok` records. /// /// # Examples @@ -17,7 +17,7 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn ok(self) -> OwnedStream>; + fn ok(self) -> Stream; /// Returns a new instance of `self` containing only `err` records. /// @@ -31,7 +31,7 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn err(self) -> OwnedStream>; + fn err(self) -> Stream; /// Returns a new instance of `self` applying `logic` on all `Ok` records. /// @@ -45,7 +45,7 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn map_ok T2 + 'static>(self, logic: L) -> OwnedStream>>; + fn map_ok T2 + 'static>(self, logic: L) -> Stream>; /// Returns a new instance of `self` applying `logic` on all `Err` records. /// @@ -59,7 +59,7 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn map_err E2 + 'static>(self, logic: L) -> OwnedStream>>; + fn map_err E2 + 'static>(self, logic: L) -> Stream>; /// Returns a new instance of `self` applying `logic` on all `Ok` records, passes through `Err` /// records. @@ -77,7 +77,7 @@ pub trait ResultStream { fn and_then Result + 'static>( self, logic: L, - ) -> OwnedStream>>; + ) -> Stream>; /// Returns a new instance of `self` applying `logic` on all `Ok` records. /// @@ -91,31 +91,31 @@ pub trait ResultStream { /// .inspect(|x| println!("seen: {:?}", x)); /// }); /// ``` - fn unwrap_or_else T + 'static>(self, logic: L) -> OwnedStream>; + fn unwrap_or_else T + 'static>(self, logic: L) -> Stream; } -impl>>> ResultStream for S { - fn ok(self) -> OwnedStream> { +impl ResultStream for Stream> { + fn ok(self) -> Stream { self.flat_map(Result::ok) } - fn err(self) -> OwnedStream> { + fn err(self) -> Stream { self.flat_map(Result::err) } - fn map_ok T2 + 'static>(self, mut logic: L) -> OwnedStream>> { + fn map_ok T2 + 'static>(self, mut logic: L) -> Stream> { self.map(move |r| r.map(&mut logic)) } - fn map_err E2 + 'static>(self, mut logic: L) -> OwnedStream>> { + fn map_err E2 + 'static>(self, mut logic: L) -> Stream> { self.map(move |r| r.map_err(&mut logic)) } - fn and_then Result + 'static>(self, mut logic: L) -> OwnedStream>> { + fn and_then Result + 'static>(self, mut logic: L) -> Stream> { self.map(move |r| r.and_then(&mut logic)) } - fn unwrap_or_else T + 'static>(self, mut logic: L) -> OwnedStream> { + fn unwrap_or_else T + 'static>(self, mut logic: L) -> Stream { self.map(move |r| r.unwrap_or_else(&mut logic)) } } diff --git a/timely/src/dataflow/operators/to_stream.rs b/timely/src/dataflow/operators/to_stream.rs index 28667301cc..b7916fb1cf 100644 --- a/timely/src/dataflow/operators/to_stream.rs +++ b/timely/src/dataflow/operators/to_stream.rs @@ -1,6 +1,6 @@ //! Conversion to the `Stream` type from iterators. -use crate::dataflow::{Scope, OwnedStream}; +use crate::dataflow::{Stream, Scope}; use crate::dataflow::operators::core::{ToStream as ToStreamCore}; /// Converts to a timely `Stream`. @@ -21,11 +21,11 @@ pub trait ToStream { /// /// assert_eq!(data1.extract(), data2.extract()); /// ``` - fn to_stream(self, scope: &mut S) -> OwnedStream>; + fn to_stream(self, scope: &mut S) -> Stream; } impl ToStream for I { - fn to_stream(self, scope: &mut S) -> OwnedStream> { + fn to_stream(self, scope: &mut S) -> Stream { ToStreamCore::to_stream(self, scope) } } diff --git a/timely/src/dataflow/operators/unordered_input.rs b/timely/src/dataflow/operators/unordered_input.rs index c57ef0a74f..7a4f819967 100644 --- a/timely/src/dataflow/operators/unordered_input.rs +++ b/timely/src/dataflow/operators/unordered_input.rs @@ -3,7 +3,7 @@ use crate::container::CapacityContainerBuilder; use crate::dataflow::operators::{ActivateCapability}; use crate::dataflow::operators::core::{UnorderedInput as UnorderedInputCore, UnorderedHandle as UnorderedHandleCore}; -use crate::dataflow::{Scope, OwnedStream}; +use crate::dataflow::{Stream, Scope}; /// Create a new `Stream` and `Handle` through which to supply input. pub trait UnorderedInput { @@ -60,12 +60,12 @@ pub trait UnorderedInput { /// assert_eq!(extract[i], (i, vec![i])); /// } /// ``` - fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream>); + fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), Stream); } impl UnorderedInput for G { - fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), OwnedStream>) { + fn new_unordered_input(&mut self) -> ((UnorderedHandle, ActivateCapability), Stream) { UnorderedInputCore::new_unordered_input(self) } } diff --git a/timely/src/dataflow/stream.rs b/timely/src/dataflow/stream.rs index 18b39b529d..8118ece25b 100644 --- a/timely/src/dataflow/stream.rs +++ b/timely/src/dataflow/stream.rs @@ -4,34 +4,21 @@ //! operator output. Extension methods on the `Stream` type provide the appearance of higher-level //! declarative programming, while constructing a dataflow graph underneath. -use crate::progress::{Source, Target}; +use std::fmt::{self, Debug}; +use crate::Container; use crate::communication::Push; use crate::dataflow::Scope; -use crate::dataflow::channels::pushers::tee::TeeHelper; use crate::dataflow::channels::Message; -use std::fmt::{self, Debug}; -use crate::Container; +use crate::dataflow::channels::pushers::tee::TeeHelper; use crate::dataflow::channels::pushers::{PushOwned, Tee}; +use crate::progress::{Source, Target}; -/// Common behavior for all streams. Streams belong to a scope and carry data. -/// -/// The main purpose of this is to allow connecting different stream kinds to a pusher. -pub trait StreamLike { - /// Connects the stream to a destination. - /// - /// The destination is described both by a `Target`, for progress tracking information, and a `P: Push` where the - /// records should actually be sent. The identifier is unique to the edge and is used only for logging purposes. - fn connect_to>+'static>(self, target: Target, pusher: P, identifier: usize); - /// The scope containing the stream. - fn scope(&self) -> S; -} - -/// Abstraction of a stream of `C: Container` records timestamped with `S::Timestamp`. +/// A tee attached to a stream. /// -/// Internally `Stream` maintains a list of data recipients who should be presented with data +/// Internally `Tee` maintains a list of data recipients who should be presented with data /// produced by the source of the stream. -pub struct StreamCore { +pub struct StreamTee { /// The progress identifier of the stream's data source. name: Source, /// The `Scope` containing the stream. @@ -40,7 +27,7 @@ pub struct StreamCore { ports: TeeHelper, } -impl Clone for StreamCore { +impl Clone for StreamTee { fn clone(&self) -> Self { Self { name: self.name.clone(), @@ -56,39 +43,29 @@ impl Clone for StreamCore { } } -/// A stream that owns a single pusher. -pub struct OwnedStream { +/// Abstraction of a stream of `C: Container` records timestamped with `S::Timestamp`. +/// +/// Internally `Stream` has a single target it can push data at. To fan out to multiple +/// targets, the stream can be forked using [`StreamCore::tee`]. +pub struct StreamCore { /// The progress identifier of the stream's data source. name: Source, /// The `Scope` containing the stream. scope: S, /// The single pusher interested in the stream's output, if any. - port: PushOwned, -} - -impl OwnedStream { - /// Allocates an `OwnedStream` from a supplied `Source` name and rendezvous point within a scope. - pub fn new(name: Source, port: PushOwned, scope: S) -> Self { - Self { name, port, scope } - } - - /// Convert the stream into a `StreamCore` that can be cloned. Requires elements to be `Clone`. - /// Consumes this stream. - pub fn tee(self) -> StreamCore where C: Clone { - let (target, registrar) = Tee::new(); - self.port.set(target); - StreamCore::new(self.name, registrar, self.scope) - } - - /// Allows the assertion of a container type, for the benefit of type inference. - pub fn container(self) -> OwnedStream where Self: AsStream { self.as_stream() } + port: PushOwned, } /// A stream batching data in vectors. pub type Stream = StreamCore>; -impl StreamLike for &StreamCore { - fn connect_to>+'static>(self, target: Target, pusher: P, identifier: usize) { + +impl StreamCore { + /// Connects the stream to a destination. + /// + /// The destination is described both by a `Target`, for progress tracking information, and a `P: Push` where the + /// records should actually be sent. The identifier is unique to the edge and is used only for logging purposes. + pub fn connect_to>+'static>(self, target: Target, pusher: P, identifier: usize) { let mut logging = self.scope().logging(); logging.as_mut().map(|l| l.log(crate::logging::ChannelsEvent { id: identifier, @@ -98,35 +75,31 @@ impl StreamLike for &StreamCore { })); self.scope.add_edge(self.name, target); - self.ports.add_pusher(pusher); + self.port.set_pusher(pusher); } - - fn scope(&self) -> S { - self.scope.clone() + /// Allocates a `Stream` from a supplied `Source` name and rendezvous point within a scope. + pub fn new(source: Source, output: PushOwned, scope: S) -> Self { + Self { name: source, port: output, scope } } -} - -impl StreamLike for OwnedStream { - fn connect_to> + 'static>(self, target: Target, pusher: P, identifier: usize) { - let mut logging = self.scope().logging(); - logging.as_mut().map(|l| l.log(crate::logging::ChannelsEvent { - id: identifier, - scope_addr: self.scope.addr().to_vec(), - source: (self.name.node, self.name.port), - target: (target.node, target.port), - })); + /// The name of the stream's source operator. + pub fn name(&self) -> &Source { &self.name } + /// The scope immediately containing the stream. + pub fn scope(&self) -> S { self.scope.clone() } - self.scope.add_edge(self.name, target); - self.port.set(pusher); - } + /// Allows the assertion of a container type, for the benefit of type inference. + pub fn container(self) -> StreamCore where Self: AsStream { self.as_stream() } - fn scope(&self) -> S { - self.scope.clone() + /// Convert the stream into a `Tee` that can be cloned. Requires elements to be `Clone`. + /// Consumes this stream. + pub fn tee(self) -> StreamTee where C: Clone { + let (target, registrar) = Tee::new(); + self.port.set_pusher(target); + StreamTee::new(self.name, registrar, self.scope) } } -impl StreamCore { - /// Allocates a `Stream` from a supplied `Source` name and rendezvous point. +impl StreamTee { + /// Allocates a `Tee` from a supplied `Source` name and rendezvous point. pub fn new(source: Source, output: TeeHelper, scope: S) -> Self { Self { name: source, ports: output, scope } } @@ -135,24 +108,24 @@ impl StreamCore { /// The scope immediately containing the stream. pub fn scope(&self) -> S { self.scope.clone() } - /// Allows the assertion of a container type, for the benefit of type inference. - pub fn container(self) -> StreamCore where Self: AsStream { self.as_stream() } + /// Create a `Stream` that attaches to the tee. + pub fn owned(&self) -> StreamCore { + let (target, registrar) = PushOwned::new(); + self.ports.add_pusher(target); + StreamCore::new(self.name, registrar, self.scope()) + } } /// A type that can be translated to a [StreamCore]. pub trait AsStream { - /// Translate `self` to a stream. - fn as_stream(self) -> Self; + /// Translate `self` to a [StreamCore]. + fn as_stream(self) -> StreamCore; } impl AsStream for StreamCore { fn as_stream(self) -> Self { self } } -impl AsStream for OwnedStream { - fn as_stream(self) -> Self { self } -} - impl Debug for StreamCore where S: Scope, @@ -187,5 +160,4 @@ mod tests { }); }); } - } From 2f18034abe7d4c35a8d349a9a01e1709c0218cc2 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Tue, 10 Dec 2024 20:20:25 -0500 Subject: [PATCH 5/5] Avoid a box Signed-off-by: Moritz Hoffmann --- timely/src/dataflow/channels/pushers/tee.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/timely/src/dataflow/channels/pushers/tee.rs b/timely/src/dataflow/channels/pushers/tee.rs index e12a9178b2..7b0ffa8178 100644 --- a/timely/src/dataflow/channels/pushers/tee.rs +++ b/timely/src/dataflow/channels/pushers/tee.rs @@ -8,8 +8,9 @@ use crate::dataflow::channels::Message; use crate::communication::Push; use crate::Container; +use crate::dataflow::channels::pushers::PushOwned; -type PushList = Rc>>>>>; +type PushList = Rc>>>; /// Wraps a shared list of `Box` to forward pushes to. Owned by `Stream`. pub struct Tee { @@ -86,8 +87,8 @@ pub struct TeeHelper { impl TeeHelper { /// Adds a new `Push` implementor to the list of recipients shared with a `Stream`. - pub fn add_pusher>+'static>(&self, pusher: P) { - self.shared.borrow_mut().push(Box::new(pusher)); + pub fn add_pusher(&self, pusher: PushOwned) { + self.shared.borrow_mut().push(pusher); } }