From 09ee604cddf91407a3f05e8f0f7b01b28d016b38 Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Tue, 30 Jun 2020 15:44:17 +0100 Subject: [PATCH 1/8] group by 2.0-alpha --- README.md | 4 +- amadeus-core/src/par_pipe.rs | 12 +- amadeus-core/src/par_sink.rs | 47 ++- amadeus-core/src/par_sink/all.rs | 135 +++----- amadeus-core/src/par_sink/any.rs | 135 +++----- amadeus-core/src/par_sink/collect.rs | 399 ++++++++---------------- amadeus-core/src/par_sink/combine.rs | 4 +- amadeus-core/src/par_sink/combiner.rs | 23 +- amadeus-core/src/par_sink/count.rs | 2 +- amadeus-core/src/par_sink/fold.rs | 4 +- amadeus-core/src/par_sink/folder.rs | 67 ++-- amadeus-core/src/par_sink/for_each.rs | 89 ++---- amadeus-core/src/par_sink/group_by.rs | 365 ++++++++++++++++++---- amadeus-core/src/par_sink/histogram.rs | 4 +- amadeus-core/src/par_sink/max.rs | 4 +- amadeus-core/src/par_sink/pipe.rs | 14 +- amadeus-core/src/par_sink/sample.rs | 2 +- amadeus-core/src/par_sink/sum.rs | 4 +- amadeus-core/src/par_sink/tuple.rs | 73 ++--- amadeus-core/src/par_stream.rs | 63 ++-- amadeus-core/src/par_stream/identity.rs | 9 +- amadeus-types/src/list.rs | 11 +- examples/cloudfront_logs.rs | 30 +- 23 files changed, 716 insertions(+), 784 deletions(-) diff --git a/README.md b/README.md index 4f85fd82..71ca7cea 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@

- 📖 Docs | 🌐 Home | 💬 Chat + 📖 Docs | 🌐 Home | 💬 Chat

## Amadeus provides: @@ -50,7 +50,7 @@ We aim to create a community that is welcoming and helpful to anyone that is int Amadeus has deep, pluggable, integration with various file formats, databases and interfaces: -| Data format | [`Source`](https://docs.rs/amadeus/0.2.5/amadeus/trait.Source.html) | [`Destination`](https://docs.rs/amadeus/0.2.5/amadeus/trait.Destination.html) | +| Data format | [`Source`](https://docs.rs/amadeus/0.2/amadeus/trait.Source.html) | [`Destination`](https://docs.rs/amadeus/0.2/amadeus/trait.Destination.html) | |---|---|---| | CSV | ✔ | ✔ | | JSON | ✔ | ✔ | diff --git a/amadeus-core/src/par_pipe.rs b/amadeus-core/src/par_pipe.rs index 4a62466e..fc8254b7 100644 --- a/amadeus-core/src/par_pipe.rs +++ b/amadeus-core/src/par_pipe.rs @@ -147,15 +147,17 @@ impl_par_dist_rename! { assert_parallel_sink(Fold::new(self, identity, op)) } - fn group_by(self, identity: ID, op: F) -> GroupBy + fn group_by(self, sink: S) -> GroupBy where A: Eq + Hash + Send + 'static, - ID: FnMut() -> C + Clone + Send + 'static, - F: FnMut(C, Either) -> C + Clone + Send + 'static, - C: Send + 'static, + S: ParallelSink, + S::Pipe: Clone + Send + 'static, + S::ReduceA: 'static, + S::ReduceC: Clone, + S::Output: Send + 'static, Self: ParallelPipe + Sized, { - assert_parallel_sink(GroupBy::new(self, identity, op)) + assert_parallel_sink(GroupBy::new(self, sink)) } fn histogram(self) -> Histogram diff --git a/amadeus-core/src/par_sink.rs b/amadeus-core/src/par_sink.rs index 56b2e2ff..95bdb633 100644 --- a/amadeus-core/src/par_sink.rs +++ b/amadeus-core/src/par_sink.rs @@ -88,49 +88,38 @@ where } } -pub trait Factory { - type Item; - - fn make(&self) -> Self::Item; -} - #[must_use] -pub trait DistributedSink { +pub trait ParallelSink { type Output; - type Pipe: DistributedPipe; - type ReduceAFactory: Factory + Clone + ProcessSend; - type ReduceBFactory: Factory; - type ReduceA: ReducerSend>::Item> + Send; - type ReduceB: ReducerProcessSend::Output> + ProcessSend; - type ReduceC: Reducer::Output, Output = Self::Output>; + type Pipe: ParallelPipe; + type ReduceA: ReducerSend>::Item> + Clone + Send; + type ReduceC: Reducer::Output, Output = Self::Output>; - fn reducers( - self, - ) -> ( - Self::Pipe, - Self::ReduceAFactory, - Self::ReduceBFactory, - Self::ReduceC, - ); + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC); } #[inline(always)] -pub(crate) fn assert_distributed_sink, Source>(r: R) -> R { +pub(crate) fn assert_parallel_sink, Source>(r: R) -> R { r } #[must_use] -pub trait ParallelSink { +pub trait DistributedSink { type Output; - type Pipe: ParallelPipe; - type ReduceAFactory: Factory; - type ReduceA: ReducerSend>::Item> + Send; - type ReduceC: Reducer::Output, Output = Self::Output>; + type Pipe: DistributedPipe; + type ReduceA: ReducerSend>::Item> + + Clone + + ProcessSend + + Send; + type ReduceB: ReducerProcessSend::Output> + + Clone + + ProcessSend; + type ReduceC: Reducer::Output, Output = Self::Output>; - fn reducers(self) -> (Self::Pipe, Self::ReduceAFactory, Self::ReduceC); + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC); } #[inline(always)] -pub(crate) fn assert_parallel_sink, Source>(r: R) -> R { +pub(crate) fn assert_distributed_sink, Source>(r: R) -> R { r } diff --git a/amadeus-core/src/par_sink/all.rs b/amadeus-core/src/par_sink/all.rs index a80a6aa6..6f8357ce 100644 --- a/amadeus-core/src/par_sink/all.rs +++ b/amadeus-core/src/par_sink/all.rs @@ -1,4 +1,5 @@ use derive_new::new; +use educe::Educe; use futures::{ready, Stream}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; @@ -7,7 +8,7 @@ use std::{ }; use super::{ - DistributedPipe, DistributedSink, Factory, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend }; use crate::pool::ProcessSend; @@ -18,75 +19,70 @@ pub struct All { f: F, } -impl, Source, F> DistributedSink for All +impl, Source, F> ParallelSink for All where - F: FnMut(I::Item) -> bool + Clone + ProcessSend + 'static, + F: FnMut(I::Item) -> bool + Clone + Send + 'static, { type Output = bool; type Pipe = I; - type ReduceAFactory = AllReducerFactory; - type ReduceBFactory = BoolAndReducerFactory; type ReduceA = AllReducer; - type ReduceB = BoolAndReducer; type ReduceC = BoolAndReducer; - fn reducers( - self, - ) -> ( - Self::Pipe, - Self::ReduceAFactory, - Self::ReduceBFactory, - Self::ReduceC, - ) { - ( - self.i, - AllReducerFactory(self.f, PhantomData), - BoolAndReducerFactory, - BoolAndReducer(true), - ) + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { + (self.i, AllReducer(self.f, PhantomData), BoolAndReducer) } } -impl, Source, F> ParallelSink for All +impl, Source, F> DistributedSink for All where - F: FnMut(I::Item) -> bool + Clone + Send + 'static, + F: FnMut(I::Item) -> bool + Clone + ProcessSend + 'static, { type Output = bool; type Pipe = I; - type ReduceAFactory = AllReducerFactory; type ReduceA = AllReducer; + type ReduceB = BoolAndReducer; type ReduceC = BoolAndReducer; - fn reducers(self) -> (Self::Pipe, Self::ReduceAFactory, Self::ReduceC) { + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { ( self.i, - AllReducerFactory(self.f, PhantomData), - BoolAndReducer(true), + AllReducer(self.f, PhantomData), + BoolAndReducer, + BoolAndReducer, ) } } -#[derive(Serialize, Deserialize)] +#[derive(Educe, Serialize, Deserialize)] +#[educe(Clone(bound = "F: Clone"))] #[serde( bound(serialize = "F: Serialize"), bound(deserialize = "F: Deserialize<'de>") )] -pub struct AllReducerFactory(F, PhantomData A>); -impl Factory for AllReducerFactory +pub struct AllReducer(F, PhantomData A>); + +impl Reducer for AllReducer where - F: FnMut(A) -> bool + Clone, + F: FnMut(A) -> bool, { - type Item = AllReducer; - fn make(&self) -> Self::Item { - AllReducer(self.0.clone(), true, PhantomData) + type Item = A; + type Output = bool; + type Async = AllReducerAsync; + + fn into_async(self) -> Self::Async { + AllReducerAsync(self.0, true, PhantomData) } } -impl Clone for AllReducerFactory +impl ReducerProcessSend for AllReducer where - F: Clone, + F: FnMut(A) -> bool, { - fn clone(&self) -> Self { - Self(self.0.clone(), PhantomData) - } + type Output = bool; +} +impl ReducerSend for AllReducer +where + F: FnMut(A) -> bool, +{ + type Output = bool; } #[pin_project] @@ -95,20 +91,9 @@ where bound(serialize = "F: Serialize"), bound(deserialize = "F: Deserialize<'de>") )] -pub struct AllReducer(F, bool, PhantomData A>); +pub struct AllReducerAsync(F, bool, PhantomData A>); -impl Reducer for AllReducer -where - F: FnMut(A) -> bool, -{ - type Item = A; - type Output = bool; - type Async = Self; - fn into_async(self) -> Self::Async { - self - } -} -impl ReducerAsync for AllReducer +impl ReducerAsync for AllReducerAsync where F: FnMut(A) -> bool, { @@ -134,41 +119,29 @@ where Poll::Ready(self.1) } } -impl ReducerProcessSend for AllReducer -where - F: FnMut(A) -> bool, -{ - type Output = bool; -} -impl ReducerSend for AllReducer -where - F: FnMut(A) -> bool, -{ - type Output = bool; -} - -#[derive(Serialize, Deserialize)] -pub struct BoolAndReducerFactory; -impl Factory for BoolAndReducerFactory { - type Item = BoolAndReducer; - fn make(&self) -> Self::Item { - BoolAndReducer(true) - } -} -#[pin_project] -#[derive(Serialize, Deserialize)] -pub struct BoolAndReducer(bool); +#[derive(Clone, Serialize, Deserialize)] +pub struct BoolAndReducer; impl Reducer for BoolAndReducer { type Item = bool; type Output = bool; - type Async = Self; + type Async = BoolAndReducerAsync; + fn into_async(self) -> Self::Async { - self + BoolAndReducerAsync(true) } } -impl ReducerAsync for BoolAndReducer { +impl ReducerProcessSend for BoolAndReducer { + type Output = bool; +} +impl ReducerSend for BoolAndReducer { + type Output = bool; +} + +#[pin_project] +pub struct BoolAndReducerAsync(bool); +impl ReducerAsync for BoolAndReducerAsync { type Item = bool; type Output = bool; @@ -190,9 +163,3 @@ impl ReducerAsync for BoolAndReducer { Poll::Ready(self.0) } } -impl ReducerProcessSend for BoolAndReducer { - type Output = bool; -} -impl ReducerSend for BoolAndReducer { - type Output = bool; -} diff --git a/amadeus-core/src/par_sink/any.rs b/amadeus-core/src/par_sink/any.rs index 51810684..3af65580 100644 --- a/amadeus-core/src/par_sink/any.rs +++ b/amadeus-core/src/par_sink/any.rs @@ -1,4 +1,5 @@ use derive_new::new; +use educe::Educe; use futures::{ready, Stream}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; @@ -7,7 +8,7 @@ use std::{ }; use super::{ - DistributedPipe, DistributedSink, Factory, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend }; use crate::pool::ProcessSend; @@ -18,75 +19,70 @@ pub struct Any { f: F, } -impl, Source, F> DistributedSink for Any +impl, Source, F> ParallelSink for Any where - F: FnMut(I::Item) -> bool + Clone + ProcessSend + 'static, + F: FnMut(I::Item) -> bool + Clone + Send + 'static, { type Output = bool; type Pipe = I; - type ReduceAFactory = AnyReducerFactory; - type ReduceBFactory = BoolOrReducerFactory; type ReduceA = AnyReducer; - type ReduceB = BoolOrReducer; type ReduceC = BoolOrReducer; - fn reducers( - self, - ) -> ( - Self::Pipe, - Self::ReduceAFactory, - Self::ReduceBFactory, - Self::ReduceC, - ) { - ( - self.i, - AnyReducerFactory(self.f, PhantomData), - BoolOrReducerFactory, - BoolOrReducer(true), - ) + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { + (self.i, AnyReducer(self.f, PhantomData), BoolOrReducer) } } -impl, Source, F> ParallelSink for Any +impl, Source, F> DistributedSink for Any where - F: FnMut(I::Item) -> bool + Clone + Send + 'static, + F: FnMut(I::Item) -> bool + Clone + ProcessSend + 'static, { type Output = bool; type Pipe = I; - type ReduceAFactory = AnyReducerFactory; type ReduceA = AnyReducer; + type ReduceB = BoolOrReducer; type ReduceC = BoolOrReducer; - fn reducers(self) -> (Self::Pipe, Self::ReduceAFactory, Self::ReduceC) { + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { ( self.i, - AnyReducerFactory(self.f, PhantomData), - BoolOrReducer(true), + AnyReducer(self.f, PhantomData), + BoolOrReducer, + BoolOrReducer, ) } } -#[derive(Serialize, Deserialize)] +#[derive(Educe, Serialize, Deserialize)] +#[educe(Clone(bound = "F: Clone"))] #[serde( bound(serialize = "F: Serialize"), bound(deserialize = "F: Deserialize<'de>") )] -pub struct AnyReducerFactory(F, PhantomData A>); -impl Factory for AnyReducerFactory +pub struct AnyReducer(F, PhantomData A>); + +impl Reducer for AnyReducer where - F: FnMut(A) -> bool + Clone, + F: FnMut(A) -> bool, { - type Item = AnyReducer; - fn make(&self) -> Self::Item { - AnyReducer(self.0.clone(), true, PhantomData) + type Item = A; + type Output = bool; + type Async = AnyReducerAsync; + + fn into_async(self) -> Self::Async { + AnyReducerAsync(self.0, true, PhantomData) } } -impl Clone for AnyReducerFactory +impl ReducerProcessSend for AnyReducer where - F: Clone, + F: FnMut(A) -> bool, { - fn clone(&self) -> Self { - Self(self.0.clone(), PhantomData) - } + type Output = bool; +} +impl ReducerSend for AnyReducer +where + F: FnMut(A) -> bool, +{ + type Output = bool; } #[pin_project] @@ -95,20 +91,9 @@ where bound(serialize = "F: Serialize"), bound(deserialize = "F: Deserialize<'de>") )] -pub struct AnyReducer(F, bool, PhantomData A>); +pub struct AnyReducerAsync(F, bool, PhantomData A>); -impl Reducer for AnyReducer -where - F: FnMut(A) -> bool, -{ - type Item = A; - type Output = bool; - type Async = Self; - fn into_async(self) -> Self::Async { - self - } -} -impl ReducerAsync for AnyReducer +impl ReducerAsync for AnyReducerAsync where F: FnMut(A) -> bool, { @@ -134,41 +119,31 @@ where Poll::Ready(!self.1) } } -impl ReducerProcessSend for AnyReducer -where - F: FnMut(A) -> bool, -{ + +#[derive(Clone, Serialize, Deserialize)] +pub struct BoolOrReducer; + +impl Reducer for BoolOrReducer { + type Item = bool; type Output = bool; + type Async = BoolOrReducerAsync; + + fn into_async(self) -> Self::Async { + BoolOrReducerAsync(true) + } } -impl ReducerSend for AnyReducer -where - F: FnMut(A) -> bool, -{ +impl ReducerProcessSend for BoolOrReducer { type Output = bool; } - -#[derive(Serialize, Deserialize)] -pub struct BoolOrReducerFactory; -impl Factory for BoolOrReducerFactory { - type Item = BoolOrReducer; - fn make(&self) -> Self::Item { - BoolOrReducer(true) - } +impl ReducerSend for BoolOrReducer { + type Output = bool; } #[pin_project] #[derive(Serialize, Deserialize)] -pub struct BoolOrReducer(bool); +pub struct BoolOrReducerAsync(bool); -impl Reducer for BoolOrReducer { - type Item = bool; - type Output = bool; - type Async = Self; - fn into_async(self) -> Self::Async { - self - } -} -impl ReducerAsync for BoolOrReducer { +impl ReducerAsync for BoolOrReducerAsync { type Item = bool; type Output = bool; @@ -190,9 +165,3 @@ impl ReducerAsync for BoolOrReducer { Poll::Ready(!self.0) } } -impl ReducerProcessSend for BoolOrReducer { - type Output = bool; -} -impl ReducerSend for BoolOrReducer { - type Output = bool; -} diff --git a/amadeus-core/src/par_sink/collect.rs b/amadeus-core/src/par_sink/collect.rs index e252b38b..3b48e5d3 100644 --- a/amadeus-core/src/par_sink/collect.rs +++ b/amadeus-core/src/par_sink/collect.rs @@ -1,4 +1,5 @@ use derive_new::new; +use educe::Educe; use futures::{pin_mut, ready, Stream, StreamExt}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; @@ -7,7 +8,7 @@ use std::{ }; use super::{ - DistributedPipe, DistributedSink, Factory, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend }; use crate::pool::ProcessSend; @@ -23,11 +24,10 @@ impl, Source, T: FromParallelStream> ParallelSi { type Output = T; type Pipe = I; - type ReduceAFactory = T::ReduceAFactory; type ReduceA = T::ReduceA; type ReduceC = T::ReduceC; - fn reducers(self) -> (Self::Pipe, Self::ReduceAFactory, Self::ReduceC) { + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { let (a, b) = T::reducers(); (self.i, a, b) } @@ -37,99 +37,64 @@ impl, Source, T: FromDistributedStream> Dist { type Output = T; type Pipe = I; - type ReduceAFactory = T::ReduceAFactory; - type ReduceBFactory = T::ReduceBFactory; type ReduceA = T::ReduceA; type ReduceB = T::ReduceB; type ReduceC = T::ReduceC; - fn reducers( - self, - ) -> ( - Self::Pipe, - Self::ReduceAFactory, - Self::ReduceBFactory, - Self::ReduceC, - ) { + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { let (a, b, c) = T::reducers(); (self.i, a, b, c) } } -pub trait FromDistributedStream: Sized { - type ReduceAFactory: Factory + Clone + ProcessSend; - type ReduceBFactory: Factory; - type ReduceA: ReducerSend + Send; - type ReduceB: ReducerProcessSend::Output> + ProcessSend; - type ReduceC: Reducer::Output, Output = Self>; +pub trait FromParallelStream: Sized { + type ReduceA: ReducerSend + Clone + Send; + type ReduceC: Reducer::Output, Output = Self>; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC); - // fn from_dist_stream(dist_stream: I, pool: &Pool) -> Self where T: Serialize + DeserializeOwned + Send + 'static, I: IntoDistributedStream, <::Iter as DistributedStream>::Task: Serialize + DeserializeOwned + Send + 'static; + fn reducers() -> (Self::ReduceA, Self::ReduceC); } -pub trait FromParallelStream: Sized { - type ReduceAFactory: Factory; - type ReduceA: ReducerSend + Send; - type ReduceC: Reducer::Output, Output = Self>; +pub trait FromDistributedStream: Sized { + type ReduceA: ReducerSend + Clone + ProcessSend; + type ReduceB: ReducerProcessSend::Output> + + Clone + + ProcessSend; + type ReduceC: Reducer::Output, Output = Self>; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC); + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC); + // fn from_dist_stream(dist_stream: I, pool: &Pool) -> Self where T: Serialize + DeserializeOwned + Send + 'static, I: IntoDistributedStream, <::Iter as DistributedStream>::Task: Serialize + DeserializeOwned + Send + 'static; } -#[derive(Serialize, Deserialize)] +#[derive(Educe, Serialize, Deserialize, new)] +#[educe(Clone, Default)] #[serde(bound = "")] -pub struct DefaultReducerFactory(PhantomData T>); -impl DefaultReducerFactory { - pub fn new() -> Self { - Self(PhantomData) - } -} -impl Default for DefaultReducerFactory { - fn default() -> Self { - Self(PhantomData) - } -} -impl Factory for DefaultReducerFactory { - type Item = T; - fn make(&self) -> Self::Item { - T::default() - } -} -impl Clone for DefaultReducerFactory { - fn clone(&self) -> Self { - Self(PhantomData) - } -} +pub struct PushReducer(PhantomData (T, A)>); -#[pin_project] -#[derive(Serialize, Deserialize)] -#[serde( - bound(serialize = "T: Serialize"), - bound(deserialize = "T: Deserialize<'de>") -)] -pub struct PushReducer(Option, PhantomData A>); -impl PushReducer { - pub(crate) fn new(t: T) -> Self { - Self(Some(t), PhantomData) +impl> Reducer for PushReducer { + type Item = A; + type Output = T; + type Async = PushReducerAsync; + + fn into_async(self) -> Self::Async { + PushReducerAsync(Some(Default::default()), PhantomData) } } -impl Default for PushReducer +impl> ReducerProcessSend for PushReducer where - T: Default, + T: ProcessSend + 'static, { - fn default() -> Self { - Self(Some(T::default()), PhantomData) - } + type Output = T; } -impl> Reducer for PushReducer { - type Item = A; +impl> ReducerSend for PushReducer +where + T: Send + 'static, +{ type Output = T; - type Async = Self; - - fn into_async(self) -> Self::Async { - self - } } -impl> ReducerAsync for PushReducer { + +#[pin_project] +pub struct PushReducerAsync(Option, PhantomData A>); +impl> ReducerAsync for PushReducerAsync { type Item = A; type Output = T; @@ -148,56 +113,37 @@ impl> ReducerAsync for PushReducer { Poll::Ready(self.0.take().unwrap()) } } -impl> ReducerProcessSend for PushReducer -where - T: ProcessSend + 'static, -{ - type Output = T; -} -impl> ReducerSend for PushReducer -where - T: Send + 'static, -{ + +#[derive(Educe, Serialize, Deserialize)] +#[educe(Clone, Default)] +#[serde(bound = "")] +pub struct ExtendReducer(PhantomData (T, A)>); +impl, T: Default + Extend, B> Reducer for ExtendReducer { + type Item = A; type Output = T; -} + type Async = ExtendReducerAsync; -#[pin_project] -#[derive(Serialize, Deserialize)] -#[serde( - bound(serialize = "T: Serialize"), - bound(deserialize = "T: Deserialize<'de>") -)] -pub struct ExtendReducer(Option, PhantomData A>); -impl Default for ExtendReducer -where - T: Default, -{ - fn default() -> Self { - Self(Some(T::default()), PhantomData) + fn into_async(self) -> Self::Async { + ExtendReducerAsync(Some(T::default()), PhantomData) } } -impl, T: Extend, B> ReducerProcessSend for ExtendReducer +impl, T: Default + Extend, B> ReducerProcessSend + for ExtendReducer where T: ProcessSend + 'static, { type Output = T; } -impl, T: Extend, B> ReducerSend for ExtendReducer +impl, T: Default + Extend, B> ReducerSend for ExtendReducer where T: Send + 'static, { type Output = T; } -impl, T: Extend, B> Reducer for ExtendReducer { - type Item = A; - type Output = T; - type Async = Self; - fn into_async(self) -> Self::Async { - self - } -} -impl, T: Extend, B> ReducerAsync for ExtendReducer { +#[pin_project] +pub struct ExtendReducerAsync(Option, PhantomData A>); +impl, T: Extend, B> ReducerAsync for ExtendReducerAsync { type Item = A; type Output = T; @@ -217,8 +163,13 @@ impl, T: Extend, B> ReducerAsync for ExtendReducer< } } -#[pin_project] -pub struct IntoReducer(#[pin] R, PhantomData T>); +#[derive(Educe, Serialize, Deserialize)] +#[educe(Clone(bound = "R: Clone"))] //, Default(bound = "R: Default"))] +#[serde( + bound(serialize = "R: Serialize"), + bound(deserialize = "R: Deserialize<'de>") +)] +pub struct IntoReducer(R, PhantomData T>); impl Default for IntoReducer where R: Default, @@ -233,13 +184,17 @@ where { type Item = R::Item; type Output = T; - type Async = IntoReducer; + type Async = IntoReducerAsync; fn into_async(self) -> Self::Async { - IntoReducer(self.0.into_async(), PhantomData) + IntoReducerAsync(self.0.into_async(), PhantomData) } } -impl ReducerAsync for IntoReducer + +#[pin_project] +pub struct IntoReducerAsync(#[pin] R, PhantomData T>); + +impl ReducerAsync for IntoReducerAsync where R::Output: Into, { @@ -259,37 +214,34 @@ where } } -#[derive(Clone, Serialize, Deserialize)] -pub struct OptionReducerFactory(RF); -impl Factory for OptionReducerFactory { - type Item = OptionReducer; +#[derive(Clone, Default, Serialize, Deserialize)] +pub struct OptionReducer(R); +impl Reducer for OptionReducer { + type Item = Option; + type Output = Option; + type Async = OptionReducerAsync; - fn make(&self) -> Self::Item { - OptionReducer(Some(self.0.make())) + fn into_async(self) -> Self::Async { + OptionReducerAsync(Some(self.0.into_async())) } } - -#[pin_project] -#[derive(Serialize, Deserialize)] -pub struct OptionReducer(#[pin] Option); -impl Default for OptionReducer +impl ReducerProcessSend for OptionReducer where - R: Default, + R::Output: ProcessSend + 'static, { - fn default() -> Self { - Self(Some(R::default())) - } + type Output = Option; } -impl Reducer for OptionReducer { - type Item = Option; +impl ReducerSend for OptionReducer +where + R::Output: Send + 'static, +{ type Output = Option; - type Async = OptionReducer; - - fn into_async(self) -> Self::Async { - OptionReducer(Some(self.0.unwrap().into_async())) - } } -impl ReducerAsync for OptionReducer { + +#[pin_project] +pub struct OptionReducerAsync(#[pin] Option); + +impl ReducerAsync for OptionReducerAsync { type Item = Option; type Output = Option; @@ -315,45 +267,13 @@ impl ReducerAsync for OptionReducer { .unwrap_or(Poll::Ready(None)) } } -impl ReducerProcessSend for OptionReducer -where - R::Output: ProcessSend + 'static, -{ - type Output = Option; -} -impl ReducerSend for OptionReducer -where - R::Output: Send + 'static, -{ - type Output = Option; -} -#[derive(Serialize, Deserialize)] +#[derive(Educe, Serialize, Deserialize)] +#[educe(Clone(bound = "R: Clone"))] // , Default(bound = "R: Default"))] #[serde( - bound(serialize = "RF: Serialize"), - bound(deserialize = "RF: Deserialize<'de>") + bound(serialize = "R: Serialize"), + bound(deserialize = "R: Deserialize<'de>") )] -pub struct ResultReducerFactory(RF, PhantomData E>); -impl Factory for ResultReducerFactory -where - RF: Factory, -{ - type Item = ResultReducer; - - fn make(&self) -> Self::Item { - ResultReducer(self.0.make(), PhantomData) - } -} -impl Clone for ResultReducerFactory -where - RF: Clone, -{ - fn clone(&self) -> Self { - Self(self.0.clone(), PhantomData) - } -} - -#[derive(Serialize, Deserialize)] pub struct ResultReducer(R, PhantomData E>); impl Default for ResultReducer where @@ -372,6 +292,21 @@ impl Reducer for ResultReducer { ResultReducerAsync::Ok(self.0.into_async()) } } +impl ReducerProcessSend for ResultReducer +where + R::Output: ProcessSend + 'static, + E: ProcessSend + 'static, +{ + type Output = Result; +} +impl ReducerSend for ResultReducer +where + R::Output: Send + 'static, + E: Send + 'static, +{ + type Output = Result; +} + #[pin_project(project = ResultReducerAsyncProj)] pub enum ResultReducerAsync { Ok(#[pin] R), @@ -403,30 +338,15 @@ impl ReducerAsync for ResultReducerAsync { } } } -impl ReducerProcessSend for ResultReducer -where - R::Output: ProcessSend + 'static, - E: ProcessSend + 'static, -{ - type Output = Result; -} -impl ReducerSend for ResultReducer -where - R::Output: Send + 'static, - E: Send + 'static, -{ - type Output = Result; -} impl FromParallelStream for Vec where T: Send + 'static, { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } @@ -435,11 +355,10 @@ impl FromParallelStream for VecDeque where T: Send + 'static, { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer>; type ReduceC = IntoReducer>, Self>; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } @@ -448,11 +367,10 @@ impl FromParallelStream for BinaryHeap where T: Send + 'static, { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer>; type ReduceC = IntoReducer>, Self>; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } @@ -461,11 +379,10 @@ impl FromParallelStream for LinkedList where T: Send + 'static, { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } @@ -475,11 +392,10 @@ where T: Eq + Hash + Send + 'static, S: BuildHasher + Default + Send + 'static, { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } @@ -490,11 +406,10 @@ where V: Send + 'static, S: BuildHasher + Default + Send + 'static, { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer<(K, V), Self>; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } @@ -503,11 +418,10 @@ impl FromParallelStream for BTreeSet where T: Ord + Send + 'static, { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } @@ -517,53 +431,48 @@ where K: Ord + Send + 'static, V: Send + 'static, { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer<(K, V), Self>; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } impl FromParallelStream for String { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceC = PushReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } impl FromParallelStream for String { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceC = PushReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } impl FromParallelStream<()> for () { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceC = PushReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } impl> FromParallelStream> for Option { - type ReduceAFactory = OptionReducerFactory; type ReduceA = OptionReducer; type ReduceC = OptionReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { let (a, c) = C::reducers(); - (OptionReducerFactory(a), OptionReducer(Some(c))) + (OptionReducer(a), OptionReducer(c)) } } @@ -571,16 +480,12 @@ impl, E> FromParallelStream> for Result where E: Send + 'static, { - type ReduceAFactory = ResultReducerFactory; type ReduceA = ResultReducer; type ReduceC = ResultReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { let (a, c) = C::reducers(); - ( - ResultReducerFactory(a, PhantomData), - ResultReducer(c, PhantomData), - ) + (ResultReducer(a, PhantomData), ResultReducer(c, PhantomData)) } } @@ -588,13 +493,11 @@ impl FromDistributedStream for Vec where T: ProcessSend + 'static, { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceB = ExtendReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } @@ -603,13 +506,11 @@ impl FromDistributedStream for VecDeque where T: ProcessSend + 'static, { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer>; type ReduceB = ExtendReducer>; type ReduceC = IntoReducer>, Self>; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } @@ -618,13 +519,11 @@ impl FromDistributedStream for BinaryHeap where T: ProcessSend + 'static, { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer>; type ReduceB = ExtendReducer>; type ReduceC = IntoReducer>, Self>; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } @@ -633,13 +532,11 @@ impl FromDistributedStream for LinkedList where T: ProcessSend + 'static, { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceB = ExtendReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } @@ -649,13 +546,11 @@ where T: Eq + Hash + ProcessSend + 'static, S: BuildHasher + Default + Send + 'static, { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceB = ExtendReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } @@ -666,13 +561,11 @@ where V: ProcessSend + 'static, S: BuildHasher + Default + Send + 'static, { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer<(K, V), Self>; type ReduceB = ExtendReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } @@ -681,13 +574,11 @@ impl FromDistributedStream for BTreeSet where T: Ord + ProcessSend + 'static, { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceB = ExtendReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } @@ -697,67 +588,53 @@ where K: Ord + ProcessSend + 'static, V: ProcessSend + 'static, { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer<(K, V), Self>; type ReduceB = ExtendReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } impl FromDistributedStream for String { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceB = PushReducer; type ReduceC = PushReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } impl FromDistributedStream for String { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceB = PushReducer; type ReduceC = PushReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } impl FromDistributedStream<()> for () { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceB = PushReducer; type ReduceC = PushReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } impl> FromDistributedStream> for Option { - type ReduceAFactory = OptionReducerFactory; - type ReduceBFactory = OptionReducerFactory; type ReduceA = OptionReducer; type ReduceB = OptionReducer; type ReduceC = OptionReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { let (a, b, c) = C::reducers(); - ( - OptionReducerFactory(a), - OptionReducerFactory(b), - OptionReducer(Some(c)), - ) + (OptionReducer(a), OptionReducer(b), OptionReducer(c)) } } @@ -765,17 +642,15 @@ impl, E> FromDistributedStream> for where E: ProcessSend + 'static, { - type ReduceAFactory = ResultReducerFactory; - type ReduceBFactory = ResultReducerFactory; type ReduceA = ResultReducer; type ReduceB = ResultReducer; type ReduceC = ResultReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { let (a, b, c) = C::reducers(); ( - ResultReducerFactory(a, PhantomData), - ResultReducerFactory(b, PhantomData), + ResultReducer(a, PhantomData), + ResultReducer(b, PhantomData), ResultReducer(c, PhantomData), ) } diff --git a/amadeus-core/src/par_sink/combine.rs b/amadeus-core/src/par_sink/combine.rs index ef12cb40..b0c255f0 100644 --- a/amadeus-core/src/par_sink/combine.rs +++ b/amadeus-core/src/par_sink/combine.rs @@ -3,9 +3,7 @@ use educe::Educe; use serde::{Deserialize, Serialize}; use std::marker::PhantomData; -use super::{ - combiner_par_sink, FolderSync, FolderSyncReducer, FolderSyncReducerFactory, ParallelPipe, ParallelSink -}; +use super::{combiner_par_sink, FolderSync, FolderSyncReducer, ParallelPipe, ParallelSink}; #[derive(Educe, Serialize, Deserialize, new)] #[educe(Clone(bound = "F: Clone"))] diff --git a/amadeus-core/src/par_sink/combiner.rs b/amadeus-core/src/par_sink/combiner.rs index c81cb20d..d4b1d7d3 100644 --- a/amadeus-core/src/par_sink/combiner.rs +++ b/amadeus-core/src/par_sink/combiner.rs @@ -2,41 +2,38 @@ use super::FolderSync; mod macros { #[macro_export] - macro_rules! combiner_dist_sink { + macro_rules! combiner_par_sink { ($combiner:ty, $self:ident, $init:expr) => { type Output = ::Output; type Pipe = I; - type ReduceAFactory = FolderSyncReducerFactory; - type ReduceBFactory = FolderSyncReducerFactory<::Output, $combiner>; type ReduceA = FolderSyncReducer; - type ReduceB = FolderSyncReducer<::Output, $combiner>; - type ReduceC = FolderSyncReducer<::Output, $combiner>; + type ReduceC = FolderSyncReducer<::Output, $combiner>; - fn reducers($self) -> (I, Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers($self) -> (I, Self::ReduceA, Self::ReduceC) { let init = $init; ( $self.i, - FolderSyncReducerFactory::new(init.clone()), - FolderSyncReducerFactory::new(init.clone()), + FolderSyncReducer::new(init.clone()), FolderSyncReducer::new(init), ) } }; } #[macro_export] - macro_rules! combiner_par_sink { + macro_rules! combiner_dist_sink { ($combiner:ty, $self:ident, $init:expr) => { type Output = ::Output; type Pipe = I; - type ReduceAFactory = FolderSyncReducerFactory; type ReduceA = FolderSyncReducer; - type ReduceC = FolderSyncReducer<::Output, $combiner>; + type ReduceB = FolderSyncReducer<::Output, $combiner>; + type ReduceC = FolderSyncReducer<::Output, $combiner>; - fn reducers($self) -> (I, Self::ReduceAFactory, Self::ReduceC) { + fn reducers($self) -> (I, Self::ReduceA, Self::ReduceB, Self::ReduceC) { let init = $init; ( $self.i, - FolderSyncReducerFactory::new(init.clone()), + FolderSyncReducer::new(init.clone()), + FolderSyncReducer::new(init.clone()), FolderSyncReducer::new(init), ) } diff --git a/amadeus-core/src/par_sink/count.rs b/amadeus-core/src/par_sink/count.rs index 005206b1..128496b0 100644 --- a/amadeus-core/src/par_sink/count.rs +++ b/amadeus-core/src/par_sink/count.rs @@ -2,7 +2,7 @@ use derive_new::new; use serde::{Deserialize, Serialize}; use super::{ - folder_par_sink, FolderSync, FolderSyncReducer, FolderSyncReducerFactory, ParallelPipe, ParallelSink, SumFolder + folder_par_sink, FolderSync, FolderSyncReducer, ParallelPipe, ParallelSink, SumFolder }; #[derive(new)] diff --git a/amadeus-core/src/par_sink/fold.rs b/amadeus-core/src/par_sink/fold.rs index 7e29bead..c568d5bc 100644 --- a/amadeus-core/src/par_sink/fold.rs +++ b/amadeus-core/src/par_sink/fold.rs @@ -7,9 +7,7 @@ use replace_with::replace_with_or_abort; use serde::{Deserialize, Serialize}; use std::marker::PhantomData; -use super::{ - folder_par_sink, FolderSync, FolderSyncReducer, FolderSyncReducerFactory, ParallelPipe, ParallelSink -}; +use super::{folder_par_sink, FolderSync, FolderSyncReducer, ParallelPipe, ParallelSink}; #[derive(new)] #[must_use] diff --git a/amadeus-core/src/par_sink/folder.rs b/amadeus-core/src/par_sink/folder.rs index 20515984..bf34d3a1 100644 --- a/amadeus-core/src/par_sink/folder.rs +++ b/amadeus-core/src/par_sink/folder.rs @@ -9,7 +9,7 @@ use std::{ marker::PhantomData, pin::Pin, task::{Context, Poll} }; -use super::{Factory, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend}; +use super::{Reducer, ReducerAsync, ReducerProcessSend, ReducerSend}; use crate::pool::ProcessSend; mod macros { @@ -18,16 +18,15 @@ mod macros { ($folder_a:ty, $folder_b:ty, $self:ident, $init_a:expr, $init_b:expr) => { type Output = ::Output; type Pipe = I; - type ReduceAFactory = FolderSyncReducerFactory; type ReduceA = FolderSyncReducer; type ReduceC = FolderSyncReducer<::Output, $folder_b>; - fn reducers($self) -> (I, Self::ReduceAFactory, Self::ReduceC) { + fn reducers($self) -> (I, Self::ReduceA, Self::ReduceC) { let init_a = $init_a; let init_b = $init_b; ( $self.i, - FolderSyncReducerFactory::new(init_a), + FolderSyncReducer::new(init_a), FolderSyncReducer::new(init_b), ) } @@ -38,19 +37,17 @@ mod macros { ($folder_a:ty, $folder_b:ty, $self:ident, $init_a:expr, $init_b:expr) => { type Output = ::Output; type Pipe = I; - type ReduceAFactory = FolderSyncReducerFactory; - type ReduceBFactory = FolderSyncReducerFactory<::Output, $folder_b>; type ReduceA = FolderSyncReducer; type ReduceB = FolderSyncReducer<::Output, $folder_b>; type ReduceC = FolderSyncReducer<::Output, $folder_b>; - fn reducers($self) -> (I, Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers($self) -> (I, Self::ReduceA, Self::ReduceB, Self::ReduceC) { let init_a = $init_a; let init_b = $init_b; ( $self.i, - FolderSyncReducerFactory::new(init_a), - FolderSyncReducerFactory::new(init_b.clone()), + FolderSyncReducer::new(init_a), + FolderSyncReducer::new(init_b.clone()), FolderSyncReducer::new(init_b), ) } @@ -69,30 +66,6 @@ pub trait FolderSync { fn push(&mut self, state: &mut Self::Output, item: A); } -#[derive(Educe, Serialize, Deserialize, new)] -#[educe(Clone(bound = "C: Clone"))] -#[serde( - bound(serialize = "C: Serialize"), - bound(deserialize = "C: Deserialize<'de>") -)] -pub struct FolderSyncReducerFactory { - folder: C, - marker: PhantomData A>, -} -impl Factory for FolderSyncReducerFactory -where - C: FolderSync + Clone, -{ - type Item = FolderSyncReducer; - - fn make(&self) -> Self::Item { - FolderSyncReducer { - folder: self.folder.clone(), - marker: PhantomData, - } - } -} - #[derive(Educe, Serialize, Deserialize, new)] #[educe(Clone(bound = "C: Clone"))] #[serde( @@ -120,6 +93,20 @@ where } } } +impl ReducerProcessSend for FolderSyncReducer +where + C: FolderSync, + C::Output: ProcessSend + 'static, +{ + type Output = C::Output; +} +impl ReducerSend for FolderSyncReducer +where + C: FolderSync, + C::Output: Send + 'static, +{ + type Output = C::Output; +} #[pin_project] pub struct FolderSyncReducerAsync { @@ -150,17 +137,3 @@ where Poll::Ready(self.project().state.take().unwrap()) } } -impl ReducerProcessSend for FolderSyncReducer -where - C: FolderSync, - C::Output: ProcessSend + 'static, -{ - type Output = C::Output; -} -impl ReducerSend for FolderSyncReducer -where - C: FolderSync, - C::Output: Send + 'static, -{ - type Output = C::Output; -} diff --git a/amadeus-core/src/par_sink/for_each.rs b/amadeus-core/src/par_sink/for_each.rs index aebc2dce..381243f0 100644 --- a/amadeus-core/src/par_sink/for_each.rs +++ b/amadeus-core/src/par_sink/for_each.rs @@ -1,4 +1,5 @@ use derive_new::new; +use educe::Educe; use futures::{ready, Stream}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; @@ -7,7 +8,7 @@ use std::{ }; use super::{ - DefaultReducerFactory, DistributedPipe, DistributedSink, Factory, ParallelPipe, ParallelSink, PushReducer, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PushReducer, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend }; use crate::pool::ProcessSend; @@ -18,79 +19,46 @@ pub struct ForEach { f: F, } -impl, Source, F> DistributedSink for ForEach +impl, Source, F> ParallelSink for ForEach where - F: FnMut(I::Item) + Clone + ProcessSend + 'static, + F: FnMut(I::Item) + Clone + Send + 'static, { type Output = (); type Pipe = I; - type ReduceAFactory = ForEachReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = ForEachReducer; - type ReduceB = PushReducer<()>; type ReduceC = PushReducer<()>; - fn reducers( - self, - ) -> ( - Self::Pipe, - Self::ReduceAFactory, - Self::ReduceBFactory, - Self::ReduceC, - ) { + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { ( self.i, - ForEachReducerFactory(self.f, PhantomData), - DefaultReducerFactory::new(), - PushReducer::new(()), + ForEachReducer(self.f, PhantomData), + PushReducer::new(), ) } } -impl, Source, F> ParallelSink for ForEach +impl, Source, F> DistributedSink for ForEach where - F: FnMut(I::Item) + Clone + Send + 'static, + F: FnMut(I::Item) + Clone + ProcessSend + 'static, { type Output = (); type Pipe = I; - type ReduceAFactory = ForEachReducerFactory; type ReduceA = ForEachReducer; + type ReduceB = PushReducer<()>; type ReduceC = PushReducer<()>; - fn reducers(self) -> (Self::Pipe, Self::ReduceAFactory, Self::ReduceC) { + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { ( self.i, - ForEachReducerFactory(self.f, PhantomData), - PushReducer::new(()), + ForEachReducer(self.f, PhantomData), + PushReducer::new(), + PushReducer::new(), ) } } -#[derive(Serialize, Deserialize)] -#[serde( - bound(serialize = "F: Serialize"), - bound(deserialize = "F: Deserialize<'de>") -)] -pub struct ForEachReducerFactory(F, PhantomData A>); -impl Factory for ForEachReducerFactory -where - F: FnMut(A) + Clone, -{ - type Item = ForEachReducer; - fn make(&self) -> Self::Item { - ForEachReducer(self.0.clone(), PhantomData) - } -} -impl Clone for ForEachReducerFactory -where - F: Clone, -{ - fn clone(&self) -> Self { - Self(self.0.clone(), PhantomData) - } -} - #[pin_project] -#[derive(Serialize, Deserialize)] +#[derive(Educe, Serialize, Deserialize)] +#[educe(Clone(bound = "F: Clone"))] #[serde( bound(serialize = "F: Serialize"), bound(deserialize = "F: Deserialize<'de>") @@ -109,6 +77,19 @@ where self } } +impl ReducerProcessSend for ForEachReducer +where + F: FnMut(A) + Clone, +{ + type Output = (); +} +impl ReducerSend for ForEachReducer +where + F: FnMut(A) + Clone, +{ + type Output = (); +} + impl ReducerAsync for ForEachReducer where F: FnMut(A) + Clone, @@ -131,15 +112,3 @@ where Poll::Ready(()) } } -impl ReducerProcessSend for ForEachReducer -where - F: FnMut(A) + Clone, -{ - type Output = (); -} -impl ReducerSend for ForEachReducer -where - F: FnMut(A) + Clone, -{ - type Output = (); -} diff --git a/amadeus-core/src/par_sink/group_by.rs b/amadeus-core/src/par_sink/group_by.rs index b9ccd62b..b2fc625e 100644 --- a/amadeus-core/src/par_sink/group_by.rs +++ b/amadeus-core/src/par_sink/group_by.rs @@ -2,89 +2,346 @@ use derive_new::new; use educe::Educe; -use either::Either; -use replace_with::{replace_with_or_abort, replace_with_or_default}; +use futures::{pin_mut, ready, stream, Stream, StreamExt}; +use pin_project::pin_project; use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, hash::Hash, marker::PhantomData, mem}; +use std::{ + collections::HashMap, hash::Hash, marker::PhantomData, mem, pin::Pin, task::{Context, Poll} +}; use super::{ - folder_par_sink, FolderSync, FolderSyncReducer, FolderSyncReducerFactory, ParallelPipe, ParallelSink + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend }; +use crate::{pool::ProcessSend, util::type_coerce}; #[derive(new)] #[must_use] -pub struct GroupBy { - i: I, - identity: ID, - op: F, - marker: PhantomData B>, +pub struct GroupBy { + a: A, + b: B, } -impl_par_dist! { - impl, Source, A, B, ID, F, C> ParallelSink - for GroupBy - where - A: Eq + Hash + Send + 'static, - ID: FnMut() -> C + Clone + Send + 'static, - F: FnMut(C, Either) -> C + Clone + Send + 'static, - C: Send + 'static, - { - folder_par_sink!(GroupByFolder, GroupByFolder, self, GroupByFolder::new(self.identity.clone(), self.op.clone()), GroupByFolder::new(self.identity, self.op)); +impl, B: ParallelSink, Source, T, U> ParallelSink + for GroupBy +where + T: Eq + Hash + Send + 'static, + B::Pipe: Clone + Send + 'static, + B::ReduceA: Clone + Send + 'static, + B::ReduceC: Clone, + B::Output: Send + 'static, +{ + type Output = HashMap; + type Pipe = A; + type ReduceA = GroupByReducerA; + type ReduceC = GroupByReducerB::Output>; + + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { + let (a, b, c) = self.b.reducers(); + (self.a, GroupByReducerA::new(a, b), GroupByReducerB::new(c)) + } +} + +impl, B: DistributedSink, Source, T, U> + DistributedSink for GroupBy +where + T: Eq + Hash + ProcessSend + 'static, + B::Pipe: Clone + ProcessSend + 'static, + B::ReduceA: Clone + ProcessSend + 'static, + B::ReduceB: Clone, + B::ReduceC: Clone, + B::Output: ProcessSend + 'static, +{ + type Output = HashMap; + type Pipe = A; + type ReduceA = GroupByReducerA; + type ReduceB = GroupByReducerB::Output>; + type ReduceC = GroupByReducerB::Output>; + + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { + let (a, b, c, d) = self.b.reducers(); + ( + self.a, + GroupByReducerA::new(a, b), + GroupByReducerB::new(c), + GroupByReducerB::new(d), + ) } } #[derive(Educe, Serialize, Deserialize, new)] -#[educe(Clone(bound = "ID: Clone, F: Clone"))] +#[educe(Clone(bound = "P: Clone, R: Clone"))] #[serde( - bound(serialize = "ID: Serialize, F: Serialize"), - bound(deserialize = "ID: Deserialize<'de>, F: Deserialize<'de>") + bound(serialize = "P: Serialize, R: Serialize"), + bound(deserialize = "P: Deserialize<'de>, R: Deserialize<'de>") )] -pub struct GroupByFolder { - identity: ID, - op: F, - marker: PhantomData (A, B, C, Step)>, +pub struct GroupByReducerA(P, R, PhantomData (R, T, U)>); + +impl Reducer for GroupByReducerA +where + R: Reducer + Clone, + T: Eq + Hash, +{ + type Item = (T, U); + type Output = HashMap; + type Async = GroupByReducerAAsync; + + fn into_async(self) -> Self::Async { + GroupByReducerAAsync::new(self.0, self.1) + } +} +impl ReducerProcessSend for GroupByReducerA +where + R: Reducer + Clone, + T: Eq + Hash + ProcessSend + 'static, + R::Output: ProcessSend + 'static, +{ + type Output = HashMap; +} +impl ReducerSend for GroupByReducerA +where + R: Reducer + Clone, + T: Eq + Hash + Send + 'static, + R::Output: Send + 'static, +{ + type Output = HashMap; } -pub struct StepA; -pub struct StepB; +#[pin_project] +#[derive(new)] +pub struct GroupByReducerAAsync { + pipe: P, + factory: R, + #[new(default)] + pending: Option, Option>>)>>, + #[new(default)] + map: HashMap>>, + marker: PhantomData (R, U)>, +} -impl FolderSync<(A, B)> for GroupByFolder +impl ReducerAsync for GroupByReducerAAsync where - A: Eq + Hash, - ID: FnMut() -> C, - F: FnMut(C, Either) -> C, + R: Reducer + Clone, + T: Eq + Hash, { - type Output = HashMap; + type Item = (T, U); + type Output = HashMap; - fn zero(&mut self) -> Self::Output { - HashMap::new() + #[inline(always)] + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, + mut stream: Pin<&mut impl Stream>, + ) -> Poll<()> { + let self_ = self.project(); + loop { + if !self_.pending.is_some() { + *self_.pending = Some(ready!(stream.as_mut().poll_next(cx)).map(|(t, u)| { + let r = if !self_.map.contains_key(&t) { + Some(Box::pin(self_.factory.clone().into_async())) + } else { + None + }; + (t, Some(u), r) + })); + } + if let Some((t, u, r)) = self_.pending.as_mut().unwrap() { + let stream = stream::poll_fn(|_cx| { + if let Some(u) = u.take() { + Poll::Ready(Some(type_coerce(u))) + } else { + Poll::Pending + } + }) + .fuse(); + pin_mut!(stream); + let map = &mut *self_.map; + let r_ = r.as_mut().unwrap_or_else(|| map.get_mut(&t).unwrap()); + let _ = r_.as_mut().poll_forward(cx, stream); + if u.is_some() { + return Poll::Pending; + } + let (t, _u, r) = self_.pending.take().unwrap().unwrap(); + if let Some(r) = r { + let _ = self_.map.insert(t, r); + } + } else { + for r in self_.map.values_mut() { + let stream = stream::empty(); + pin_mut!(stream); + ready!(r.as_mut().poll_forward(cx, stream)); + } + return Poll::Ready(()); + } + } } - fn push(&mut self, state: &mut Self::Output, (key, value): (A, B)) { - let state = state.entry(key).or_insert_with(&mut self.identity); - replace_with_or_abort(state, |state| (self.op)(state, Either::Left(value))) + fn poll_output(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + let self_ = self.project(); + Poll::Ready( + mem::take(self_.map) + .into_iter() + .map(|(k, mut v)| { + ( + k, + if let Poll::Ready(x) = v.as_mut().poll_output(cx) { + x + } else { + todo!() + }, + ) + }) + .collect(), + ) } } -impl FolderSync> for GroupByFolder + +#[derive(Educe, Serialize, Deserialize, new)] +#[educe(Clone(bound = "R: Clone"))] +#[serde( + bound(serialize = "R: Serialize"), + bound(deserialize = "R: Deserialize<'de>") +)] +pub struct GroupByReducerB(R, PhantomData (T, U)>); + +impl Reducer for GroupByReducerB where - A: Eq + Hash, - ID: FnMut() -> C, - F: FnMut(C, Either) -> C, + R: Reducer + Clone, + T: Eq + Hash, { - type Output = HashMap; + type Item = HashMap; + type Output = HashMap; + type Async = GroupByReducerBAsync; - fn zero(&mut self) -> Self::Output { - HashMap::new() + fn into_async(self) -> Self::Async { + GroupByReducerBAsync::new(self.0) } - fn push(&mut self, state: &mut Self::Output, mut b: HashMap) { - replace_with_or_default(state, |mut a| { - if a.len() < b.len() { - mem::swap(&mut a, &mut b); +} +impl ReducerProcessSend for GroupByReducerB +where + R: Reducer + Clone, + T: Eq + Hash + ProcessSend + 'static, + R::Output: ProcessSend + 'static, +{ + type Output = HashMap; +} +impl ReducerSend for GroupByReducerB +where + R: Reducer + Clone, + T: Eq + Hash + Send + 'static, + R::Output: Send + 'static, +{ + type Output = HashMap; +} + +#[pin_project] +#[derive(new)] +pub struct GroupByReducerBAsync { + f: R, + #[new(default)] + pending: Option>>)>>>, + #[new(default)] + map: HashMap>>, + marker: PhantomData (T, U)>, +} + +impl ReducerAsync for GroupByReducerBAsync +where + R: Reducer + Clone, + R::Async: ReducerAsync, + T: Eq + Hash, +{ + type Item = HashMap; + type Output = HashMap; + + #[inline(always)] + fn poll_forward( + self: Pin<&mut Self>, cx: &mut Context, + mut stream: Pin<&mut impl Stream>, + ) -> Poll<()> { + let self_ = self.project(); + loop { + if self_.pending.is_none() { + *self_.pending = Some(ready!(stream.as_mut().poll_next(cx)).map(|item| { + item.into_iter() + .map(|(k, v)| { + let r = if !self_.map.contains_key(&k) { + Some(Box::pin(self_.f.clone().into_async())) + } else { + None + }; + (k, (v, r)) + }) + .collect() + })); } - for (key, value) in b { - let state = a.entry(key).or_insert_with(&mut self.identity); - replace_with_or_abort(state, |state| (self.op)(state, Either::Right(value))) + let pending = self_.pending.as_mut().unwrap(); + if let Some(pending) = pending { + while let Some((k, (v, mut r))) = remove_arbitrary(pending) { + let mut v = Some(v); + let stream = stream::poll_fn(|_cx| { + if let Some(v) = v.take() { + Poll::Ready(Some(type_coerce::(v))) + } else { + Poll::Pending + } + }); + pin_mut!(stream); + let map = &mut *self_.map; + let r_ = r.as_mut().unwrap_or_else(|| map.get_mut(&k).unwrap()); + let _ = r_.as_mut().poll_forward(cx, stream); + if let Some(v) = v { + let _ = pending.insert(k, (v, r)); + return Poll::Pending; + } + if let Some(r) = r { + let _ = self_.map.insert(k, r); + } + } + *self_.pending = None; + } else { + for r in self_.map.values_mut() { + let stream = stream::empty(); + pin_mut!(stream); + ready!(r.as_mut().poll_forward(cx, stream)); + } + return Poll::Ready(()); } - a - }) + } + } + fn poll_output(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + let self_ = self.project(); + Poll::Ready( + mem::take(self_.map) + .into_iter() + .map(|(k, mut v)| { + ( + k, + if let Poll::Ready(x) = ReducerAsync::poll_output(v.as_mut(), cx) { + x + } else { + todo!() + }, + ) + }) + .collect(), + ) } } + +#[allow(clippy::unnecessary_filter_map)] +fn remove_arbitrary(map: &mut HashMap) -> Option<(K, V)> +where + K: Eq + Hash, +{ + let mut first = None; + *map = mem::take(map) + .into_iter() + .filter_map(|el| { + if first.is_none() { + first = Some(el); + None + } else { + Some(el) + } + }) + .collect(); + first +} diff --git a/amadeus-core/src/par_sink/histogram.rs b/amadeus-core/src/par_sink/histogram.rs index a2ae45bf..bbd3c416 100644 --- a/amadeus-core/src/par_sink/histogram.rs +++ b/amadeus-core/src/par_sink/histogram.rs @@ -7,9 +7,7 @@ use replace_with::replace_with_or_default; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, hash::Hash, marker::PhantomData}; -use super::{ - folder_par_sink, FolderSync, FolderSyncReducer, FolderSyncReducerFactory, ParallelPipe, ParallelSink -}; +use super::{folder_par_sink, FolderSync, FolderSyncReducer, ParallelPipe, ParallelSink}; #[derive(new)] #[must_use] diff --git a/amadeus-core/src/par_sink/max.rs b/amadeus-core/src/par_sink/max.rs index 3ffe300f..e5578202 100644 --- a/amadeus-core/src/par_sink/max.rs +++ b/amadeus-core/src/par_sink/max.rs @@ -3,9 +3,7 @@ use educe::Educe; use serde::{Deserialize, Serialize}; use std::{cmp::Ordering, marker::PhantomData}; -use super::{ - combiner_par_sink, CombinerSync, FolderSyncReducer, FolderSyncReducerFactory, ParallelPipe, ParallelSink -}; +use super::{combiner_par_sink, CombinerSync, FolderSyncReducer, ParallelPipe, ParallelSink}; #[derive(new)] #[must_use] diff --git a/amadeus-core/src/par_sink/pipe.rs b/amadeus-core/src/par_sink/pipe.rs index 6bb797d5..f5a0a301 100644 --- a/amadeus-core/src/par_sink/pipe.rs +++ b/amadeus-core/src/par_sink/pipe.rs @@ -36,11 +36,10 @@ impl, B: ParallelSink, Source> ParallelSink; - type ReduceAFactory = B::ReduceAFactory; type ReduceA = B::ReduceA; type ReduceC = B::ReduceC; - fn reducers(self) -> (Self::Pipe, Self::ReduceAFactory, Self::ReduceC) { + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { let (a, b, c) = self.b.reducers(); (Pipe::new(self.a, a), b, c) } @@ -50,20 +49,11 @@ impl, B: DistributedSink, Source> Distribute { type Output = B::Output; type Pipe = Pipe; - type ReduceAFactory = B::ReduceAFactory; - type ReduceBFactory = B::ReduceBFactory; type ReduceA = B::ReduceA; type ReduceB = B::ReduceB; type ReduceC = B::ReduceC; - fn reducers( - self, - ) -> ( - Self::Pipe, - Self::ReduceAFactory, - Self::ReduceBFactory, - Self::ReduceC, - ) { + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { let (a, b, c, d) = self.b.reducers(); (Pipe::new(self.a, a), b, c, d) } diff --git a/amadeus-core/src/par_sink/sample.rs b/amadeus-core/src/par_sink/sample.rs index 71fce56d..1798f552 100644 --- a/amadeus-core/src/par_sink/sample.rs +++ b/amadeus-core/src/par_sink/sample.rs @@ -7,7 +7,7 @@ use std::hash::Hash; use streaming_algorithms::{HyperLogLogMagnitude, SampleUnstable as SASampleUnstable, Top}; use super::{ - folder_par_sink, FolderSync, FolderSyncReducer, FolderSyncReducerFactory, ParallelPipe, ParallelSink, SumFolder, SumZeroFolder + folder_par_sink, FolderSync, FolderSyncReducer, ParallelPipe, ParallelSink, SumFolder, SumZeroFolder }; #[derive(new)] diff --git a/amadeus-core/src/par_sink/sum.rs b/amadeus-core/src/par_sink/sum.rs index b4a88ec3..aaaa7aab 100644 --- a/amadeus-core/src/par_sink/sum.rs +++ b/amadeus-core/src/par_sink/sum.rs @@ -4,9 +4,7 @@ use replace_with::replace_with_or_abort; use serde::{Deserialize, Serialize}; use std::{iter, marker::PhantomData, mem}; -use super::{ - folder_par_sink, FolderSync, FolderSyncReducer, FolderSyncReducerFactory, ParallelPipe, ParallelSink -}; +use super::{folder_par_sink, FolderSync, FolderSyncReducer, ParallelPipe, ParallelSink}; #[derive(new)] #[must_use] diff --git a/amadeus-core/src/par_sink/tuple.rs b/amadeus-core/src/par_sink/tuple.rs index 9dd538e1..d716ed42 100644 --- a/amadeus-core/src/par_sink/tuple.rs +++ b/amadeus-core/src/par_sink/tuple.rs @@ -1,5 +1,6 @@ -#![allow(non_snake_case, clippy::type_complexity, irrefutable_let_patterns, clippy::new_without_default, unused_mut, unreachable_code)] +#![allow(non_snake_case, clippy::type_complexity, irrefutable_let_patterns, clippy::new_without_default, unused_mut, unreachable_code, clippy::too_many_arguments)] +use derive_new::new; use futures::{pin_mut, ready, stream, Stream, StreamExt}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; @@ -9,7 +10,7 @@ use std::{ use sum::*; use super::{ - DistributedPipe, DistributedSink, Factory, ParallelPipe, ParallelSink, PipeTask, PipeTaskAsync, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask, PipeTaskAsync, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend }; use crate::{ pool::ProcessSend, sink::{Sink, SinkMap} @@ -39,7 +40,7 @@ where } macro_rules! impl_tuple { - ($reduceafactory:ident $reducea:ident $reduceaasync:ident $reducebfactory:ident $reduceb:ident $reducebasync:ident $async:ident $enum:ident $($copy:ident)? : $($i:ident $r:ident $o:ident $c:ident $iterator:ident $reducera:ident $reducerb:ident $num:tt $t:ident $($copyb:ident)? , $comma:tt)*) => ( + ($reducea:ident $reduceaasync:ident $reduceb:ident $reducebasync:ident $async:ident $enum:ident $($copy:ident)? : $($i:ident $r:ident $o:ident $c:ident $iterator:ident $reducera:ident $reducerb:ident $num:tt $t:ident $($copyb:ident)? , $comma:tt)*) => ( impl< Source, $($r: ParallelSink,)* @@ -49,15 +50,14 @@ macro_rules! impl_tuple { { type Output = ($($o,)*); type Pipe = ($($r::Pipe,)*); - type ReduceAFactory = $reduceafactory<$($r::ReduceAFactory,)*>; type ReduceA = $reducea<$($r::ReduceA,)*>; type ReduceC = $reduceb<$($r::ReduceC,)*>; - fn reducers(self) -> (Self::Pipe, Self::ReduceAFactory, Self::ReduceC) { + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { $(let ($iterator, $reducera, $t) = self.$num.reducers();)* ( ($($iterator,)*), - $reduceafactory($($reducera,)*), + $reducea{$($t: $reducera,)*}, $reduceb{$($t,)*}, ) } @@ -71,18 +71,16 @@ macro_rules! impl_tuple { { type Output = ($($o,)*); type Pipe = ($($r::Pipe,)*); - type ReduceAFactory = $reduceafactory<$($r::ReduceAFactory,)*>; - type ReduceBFactory = $reducebfactory<$($r::ReduceBFactory,)*>; type ReduceA = $reducea<$($r::ReduceA,)*>; type ReduceB = $reduceb<$($r::ReduceB,)*>; type ReduceC = $reduceb<$($r::ReduceC,)*>; - fn reducers(self) -> (Self::Pipe, Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { $(let ($iterator, $reducera, $reducerb, $t) = self.$num.reducers();)* ( ($($iterator,)*), - $reduceafactory($($reducera,)*), - $reducebfactory($($reducerb,)*), + $reducea{$($t: $reducera,)*}, + $reduceb{$($t: $reducerb,)*}, $reduceb{$($t,)*}, ) } @@ -193,19 +191,7 @@ macro_rules! impl_tuple { } } - #[derive(Clone, Serialize, Deserialize)] - pub struct $reduceafactory<$($r,)*>($(pub(crate) $r,)*); - impl<$($r:Factory,)*> Factory for $reduceafactory<$($r,)*> { - type Item = $reducea<$($r::Item,)*>; - - fn make(&self) -> Self::Item { - $reducea{ - $($t: self.$num.make(),)* - } - } - } - - #[derive(Serialize, Deserialize)] + #[derive(Clone, Serialize, Deserialize, new)] pub struct $reducea<$($t,)*> { $($t: $t,)* } @@ -279,29 +265,10 @@ macro_rules! impl_tuple { type Output = ($($t::Output,)*); } - pub struct $reducebfactory<$($r,)*>($(pub(crate) $r,)*); - impl<$($r:Factory,)*> Factory for $reducebfactory<$($r,)*> { - type Item = $reduceb<$($r::Item,)*>; - - fn make(&self) -> Self::Item { - $reduceb{ - $($t: self.$num.make(),)* - } - } - } - - #[derive(Serialize, Deserialize)] + #[derive(Clone, Serialize, Deserialize, new)] pub struct $reduceb<$($t,)*> { $($t: $t,)* } - impl<$($t,)*> $reduceb<$($t,)*> { - #[allow(clippy::too_many_arguments)] - pub fn new($($t: $t,)*) -> Self { - Self { - $($t,)* - } - } - } impl<$($t: Reducer,)*> Reducer for $reduceb<$($t,)*> { type Item = ($($t::Item,)*); type Output = ($($t::Output,)*); @@ -391,15 +358,15 @@ macro_rules! impl_tuple { } ); } -impl_tuple!(ReduceA0Factory ReduceA0 ReduceA0Async ReduceC0Factory ReduceC0 ReduceC0Async AsyncTuple0 Sum0:); -impl_tuple!(ReduceA1Factory ReduceA1 ReduceA1Async ReduceC1Factory ReduceC1 ReduceC1Async AsyncTuple1 Sum1: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A,,); -impl_tuple!(ReduceA2Factory ReduceA2 ReduceA2Async ReduceC2Factory ReduceC2 ReduceC2Async AsyncTuple2 Sum2 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,,); -impl_tuple!(ReduceA3Factory ReduceA3 ReduceA3Async ReduceC3Factory ReduceC3 ReduceC3Async AsyncTuple3 Sum3 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,,); -impl_tuple!(ReduceA4Factory ReduceA4 ReduceA4Async ReduceC4Factory ReduceC4 ReduceC4Async AsyncTuple4 Sum4 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,,); -impl_tuple!(ReduceA5Factory ReduceA5 ReduceA5Async ReduceC5Factory ReduceC5 ReduceC5Async AsyncTuple5 Sum5 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,,); -impl_tuple!(ReduceA6Factory ReduceA6 ReduceA6Async ReduceC6Factory ReduceC6 ReduceC6Async AsyncTuple6 Sum6 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,, I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 5 F Copy,,); -impl_tuple!(ReduceA7Factory ReduceA7 ReduceA7Async ReduceC7Factory ReduceC7 ReduceC7Async AsyncTuple7 Sum7 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,, I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 5 F Copy,, I6 R6 O6 C6 iterator_6 reducer_a_6 reducer_b_6 6 G Copy,,); -impl_tuple!(ReduceA8Factory ReduceA8 ReduceA8Async ReduceC8Factory ReduceC8 ReduceC8Async AsyncTuple8 Sum8 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,, I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 5 F Copy,, I6 R6 O6 C6 iterator_6 reducer_a_6 reducer_b_6 6 G Copy,, I7 R7 O7 C7 iterator_7 reducer_a_7 reducer_b_7 7 H Copy,,); +impl_tuple!(ReduceA0 ReduceA0Async ReduceC0 ReduceC0Async AsyncTuple0 Sum0:); +impl_tuple!(ReduceA1 ReduceA1Async ReduceC1 ReduceC1Async AsyncTuple1 Sum1: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A,,); +impl_tuple!(ReduceA2 ReduceA2Async ReduceC2 ReduceC2Async AsyncTuple2 Sum2 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,,); +impl_tuple!(ReduceA3 ReduceA3Async ReduceC3 ReduceC3Async AsyncTuple3 Sum3 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,,); +impl_tuple!(ReduceA4 ReduceA4Async ReduceC4 ReduceC4Async AsyncTuple4 Sum4 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,,); +impl_tuple!(ReduceA5 ReduceA5Async ReduceC5 ReduceC5Async AsyncTuple5 Sum5 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,,); +impl_tuple!(ReduceA6 ReduceA6Async ReduceC6 ReduceC6Async AsyncTuple6 Sum6 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,, I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 5 F Copy,,); +impl_tuple!(ReduceA7 ReduceA7Async ReduceC7 ReduceC7Async AsyncTuple7 Sum7 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,, I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 5 F Copy,, I6 R6 O6 C6 iterator_6 reducer_a_6 reducer_b_6 6 G Copy,,); +impl_tuple!(ReduceA8 ReduceA8Async ReduceC8 ReduceC8Async AsyncTuple8 Sum8 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,, I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 5 F Copy,, I6 R6 O6 C6 iterator_6 reducer_a_6 reducer_b_6 6 G Copy,, I7 R7 O7 C7 iterator_7 reducer_a_7 reducer_b_7 7 H Copy,,); #[pin_project(project = PeekableProj)] #[derive(Debug)] diff --git a/amadeus-core/src/par_stream.rs b/amadeus-core/src/par_stream.rs index eebe383a..f93e17ef 100644 --- a/amadeus-core/src/par_stream.rs +++ b/amadeus-core/src/par_stream.rs @@ -132,15 +132,13 @@ pub trait DistributedStream { assert_distributed_stream(Chain::new(self, chain.into_dist_stream())) } - async fn reduce( - mut self, pool: &P, reduce_a_factory: R1F, reduce_b_factory: R2F, reduce_c: R3, + async fn reduce( + mut self, pool: &P, reduce_a_factory: R1, reduce_b_factory: R2, reduce_c: R3, ) -> B where P: ProcessPool, - R1F: Factory + Clone + ProcessSend + 'static, - R2F: Factory, - R1: ReducerSend + Send + 'static, - R2: ReducerProcessSend::Output> + ProcessSend + 'static, + R1: ReducerSend + Clone + ProcessSend + 'static, + R2: ReducerProcessSend::Output> + Clone + ProcessSend + 'static, R3: Reducer::Output, Output = B>, Self::Task: 'static, Self: Sized, @@ -191,7 +189,7 @@ pub trait DistributedStream { .into_iter() .filter(|tasks| !tasks.is_empty()) .map(|tasks| { - let reduce_b = reduce_b_factory.make(); + let reduce_b = reduce_b_factory.clone(); let reduce_a_factory = reduce_a_factory.clone(); pool.spawn(FnOnce!(move |pool: &P::ThreadPool| { #[pin_project] @@ -257,7 +255,7 @@ pub trait DistributedStream { .into_iter() .filter(|tasks| !tasks.is_empty()) .map(|tasks| { - let reduce_a = reduce_a_factory.make(); + let reduce_a = reduce_a_factory.clone(); pool.spawn(move || { let sink = Connect(reduce_a.into_async()); async move { @@ -316,7 +314,6 @@ pub trait DistributedStream { P: ProcessPool, DistSink: DistributedSink, >::Task: 'static, - DistSink::ReduceAFactory: 'static, DistSink::ReduceA: 'static, DistSink::ReduceB: 'static, Self::Task: 'static, @@ -403,10 +400,8 @@ pub trait DistributedStream { DistSinkA: DistributedSink, DistSinkB: for<'a> DistributedSink<&'a Self::Item, Output = B> + 'static, >::Task: 'static, - DistSinkA::ReduceAFactory: 'static, DistSinkA::ReduceA: 'static, DistSinkA::ReduceB: 'static, - >::ReduceAFactory: 'static, >::ReduceA: 'static, >::ReduceB: 'static, <>::Pipe as DistributedPipe< @@ -572,8 +567,8 @@ pub trait DistributedStream { Connect(self, iterator_a, iterator_b, PhantomData) .reduce( pool, - ReduceA2Factory(reducer_a_a, reducer_b_a), - ReduceC2Factory(reducer_a_b, reducer_b_b), + ReduceA2::new(reducer_a_a, reducer_b_a), + ReduceC2::new(reducer_a_b, reducer_b_b), ReduceC2::new(reducer_a_c, reducer_b_c), ) .await @@ -608,21 +603,23 @@ pub trait DistributedStream { .await } - async fn group_by(self, pool: &P, identity: ID, op: F) -> HashMap + async fn group_by(self, pool: &P, sink: S) -> HashMap where P: ProcessPool, A: Eq + Hash + ProcessSend + 'static, B: 'static, - ID: FnMut() -> C + Clone + ProcessSend + 'static, - F: FnMut(C, Either) -> C + Clone + ProcessSend + 'static, - C: ProcessSend + 'static, - Self::Item: 'static, + S: DistributedSink, + S::Pipe: Clone + ProcessSend + 'static, + S::ReduceA: 'static, + S::ReduceB: 'static, + S::ReduceC: Clone, + S::Output: ProcessSend + 'static, Self::Task: 'static, Self: DistributedStream + Sized, { self.pipe( pool, - DistributedPipe::::group_by(Identity, identity, op), + DistributedPipe::::group_by(Identity, sink), ) .await } @@ -828,7 +825,6 @@ pub trait DistributedStream { where P: ProcessPool, B: FromDistributedStream, - B::ReduceAFactory: ProcessSend + 'static, B::ReduceA: ProcessSend + 'static, B::ReduceB: ProcessSend + 'static, Self::Task: 'static, @@ -902,11 +898,10 @@ pub trait ParallelStream { // assert_parallel_stream(Chain::new(self, chain.into_par_stream())) // } - async fn reduce(mut self, pool: &P, reduce_a_factory: R1F, reduce_c: R3) -> B + async fn reduce(mut self, pool: &P, reduce_a_factory: R1, reduce_c: R3) -> B where P: ThreadPool, - R1F: Factory, - R1: ReducerSend + Send + 'static, + R1: ReducerSend + Clone + Send + 'static, R3: Reducer::Output, Output = B>, Self::Task: 'static, Self: Sized, @@ -957,7 +952,7 @@ pub trait ParallelStream { .into_iter() .filter(|tasks| !tasks.is_empty()) .map(|tasks| { - let reduce_a = reduce_a_factory.make(); + let reduce_a = reduce_a_factory.clone(); pool.spawn(FnOnce!(move || { #[pin_project] struct Connect(#[pin] B); @@ -1254,7 +1249,7 @@ pub trait ParallelStream { Connect(self, iterator_a, iterator_b, PhantomData) .reduce( pool, - ReduceA2Factory(reducer_a_a, reducer_b_a), + ReduceA2::new(reducer_a_a, reducer_b_a), ReduceC2::new(reducer_a_b, reducer_b_b), ) .await @@ -1289,23 +1284,21 @@ pub trait ParallelStream { .await } - async fn group_by(self, pool: &P, identity: ID, op: F) -> HashMap + async fn group_by(self, pool: &P, sink: S) -> HashMap where P: ThreadPool, A: Eq + Hash + Send + 'static, B: 'static, - ID: FnMut() -> C + Clone + Send + 'static, - F: FnMut(C, Either) -> C + Clone + Send + 'static, - C: Send + 'static, - Self::Item: 'static, + S: ParallelSink, + S::Pipe: Clone + Send + 'static, + S::ReduceA: 'static, + S::ReduceC: Clone, + S::Output: Send + 'static, Self::Task: 'static, Self: ParallelStream + Sized, { - self.pipe( - pool, - ParallelPipe::::group_by(Identity, identity, op), - ) - .await + self.pipe(pool, ParallelPipe::::group_by(Identity, sink)) + .await } async fn histogram

(self, pool: &P) -> Vec<(Self::Item, usize)> diff --git a/amadeus-core/src/par_stream/identity.rs b/amadeus-core/src/par_stream/identity.rs index a50f8f50..bdd5ac3b 100644 --- a/amadeus-core/src/par_stream/identity.rs +++ b/amadeus-core/src/par_stream/identity.rs @@ -11,6 +11,7 @@ use crate::sink::Sink; // TODO: add type parameter to Identity when type the type system includes HRTB in the ParallelPipe impl https://github.com/dtolnay/ghost/ +#[derive(Clone, Copy, Debug)] pub struct Identity; impl_par_dist! { @@ -93,12 +94,8 @@ mod workaround { Fold::new(self, identity, op) } - pub fn group_by(self, identity: ID, op: F) -> GroupBy - where - ID: FnMut() -> C + Clone + Send + 'static, - C: Send + 'static, - { - GroupBy::new(self, identity, op) + pub fn group_by(self, sink: S) -> GroupBy { + GroupBy::new(self, sink) } pub fn histogram(self) -> Histogram { diff --git a/amadeus-types/src/list.rs b/amadeus-types/src/list.rs index 79d39e70..e9c73671 100644 --- a/amadeus-types/src/list.rs +++ b/amadeus-types/src/list.rs @@ -7,9 +7,7 @@ use std::{ use super::{AmadeusOrd, Data}; use amadeus_core::{ - par_sink::{ - DefaultReducerFactory, ExtendReducer, FromDistributedStream, FromParallelStream, PushReducer - }, pool::ProcessSend, util::type_coerce + par_sink::{ExtendReducer, FromDistributedStream, FromParallelStream, PushReducer}, pool::ProcessSend, util::type_coerce }; pub struct List { @@ -370,11 +368,10 @@ impl FromParallelStream for List where T: Send + 'static, { - type ReduceAFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceC) { Default::default() } } @@ -383,13 +380,11 @@ impl FromDistributedStream for List where T: ProcessSend + 'static, { - type ReduceAFactory = DefaultReducerFactory; - type ReduceBFactory = DefaultReducerFactory; type ReduceA = PushReducer; type ReduceB = ExtendReducer; type ReduceC = ExtendReducer; - fn reducers() -> (Self::ReduceAFactory, Self::ReduceBFactory, Self::ReduceC) { + fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC) { Default::default() } } diff --git a/examples/cloudfront_logs.rs b/examples/cloudfront_logs.rs index 40ec1a91..6b889823 100644 --- a/examples/cloudfront_logs.rs +++ b/examples/cloudfront_logs.rs @@ -28,28 +28,30 @@ async fn main() { .await .unwrap(); - let (sample, (histogram, count)) = rows + let histogram = rows // (sample, (histogram, count)) = rows .par_stream() .map(Result::unwrap) - .fork( + .pipe( pool, - Identity.sample_unstable(10), - ( - Identity - .map(|row: &CloudfrontRow| (row.time.truncate_minutes(60), ())) - .group_by( - || 0, - |count, fold| count + fold.map_left(|()| 1).into_inner(), - ), - Identity.count(), - ), + // Identity.sample_unstable(10), + // ( + Identity + .map(|row: CloudfrontRow| (row.time.truncate_minutes(60), ())) + .group_by( + Identity.count() + // || 0, + // |count, fold| count + fold.map_left(|()| 1).into_inner(), + ), + // Identity.map(|row: CloudfrontRow| (row.time.truncate_minutes(60), ())).group_by(Identity.count()), + // Identity.count(), + // ), ) .await; let mut histogram = histogram.into_iter().collect::>(); histogram.sort(); - println!("{} log lines analysed.", count); - println!("sample: {:#?}", sample); + // println!("{} log lines analysed.", count); + // println!("sample: {:#?}", sample); println!( "histogram:\n {}", histogram From acc1fda2303ab74c6717c085fec905e13e24b90a Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Tue, 30 Jun 2020 22:11:03 +0100 Subject: [PATCH 2/8] avoid using some unstable features --- amadeus-core/src/into_par_stream/slice.rs | 4 +- amadeus-core/src/lib.rs | 2 - src/lib.rs | 1 - src/pool/thread.rs | 79 ++++++++++++----------- 4 files changed, 46 insertions(+), 40 deletions(-) diff --git a/amadeus-core/src/into_par_stream/slice.rs b/amadeus-core/src/into_par_stream/slice.rs index 26ae9544..f49b03df 100644 --- a/amadeus-core/src/into_par_stream/slice.rs +++ b/amadeus-core/src/into_par_stream/slice.rs @@ -52,7 +52,9 @@ impl_par_dist_rename! { } } -pub struct Never(!); +enum NeverInner {} + +pub struct Never(NeverInner); impl StreamTask for Never { type Item = Self; diff --git a/amadeus-core/src/lib.rs b/amadeus-core/src/lib.rs index 58130ac8..fc18c396 100644 --- a/amadeus-core/src/lib.rs +++ b/amadeus-core/src/lib.rs @@ -1,7 +1,5 @@ #![doc(html_root_url = "https://docs.rs/amadeus-core/0.2.5")] -#![feature(never_type)] #![feature(specialization)] -#![feature(read_initializer)] #![allow(incomplete_features)] #![recursion_limit = "25600"] diff --git a/src/lib.rs b/src/lib.rs index c2a1f7e7..170d65d1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,6 @@ #![doc( html_logo_url = "https://raw.githubusercontent.com/constellation-rs/amadeus/master/logo.svg?sanitize=true" )] -#![feature(specialization)] #![warn( // missing_copy_implementations, // missing_debug_implementations, diff --git a/src/pool/thread.rs b/src/pool/thread.rs index 535c4cc9..1d639977 100644 --- a/src/pool/thread.rs +++ b/src/pool/thread.rs @@ -1,8 +1,10 @@ -use futures::{future, FutureExt, TryFutureExt}; +use futures::TryFutureExt; use std::{ future::Future, io, panic::{RefUnwindSafe, UnwindSafe}, sync::Arc }; -use tokio::task::spawn; +use tokio::{ + runtime::Handle, task::{JoinError, LocalSet} +}; use super::util::{assert_sync_and_send, Panicked}; @@ -35,8 +37,8 @@ impl ThreadPool { T: Send + 'static, { let _self = self; - spawn(DuckSend(future::lazy(|_| work()).flatten())) - .map_err(tokio::task::JoinError::into_panic) + spawn_pinned(|| work()) + .map_err(JoinError::into_panic) .map_err(Panicked::from) } } @@ -58,42 +60,47 @@ fn _assert() { let _ = assert_sync_and_send::; } -// TODO remove when spawn_pinned exists https://github.com/tokio-rs/tokio/issues/2545 - -use pin_project::pin_project; -use std::{ - pin::Pin, task::{Context, Poll} -}; -#[pin_project] -struct DuckSend(#[pin] F); -impl Future for DuckSend +fn spawn_pinned(task: F) -> impl Future> + Send where - F: Future, + F: FnOnce() -> Fut + Send + 'static, + Fut: Future + 'static, + T: Send + 'static, { - type Output = F::Output; - - fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - // TODO! - // assert!(::is_send(), "{}", std::any::type_name::()); - self.project().0.poll(cx) + thread_local! { + static LOCAL: LocalSet = LocalSet::new(); } + let handle = Handle::current(); + let handle1 = handle.clone(); + handle.spawn_blocking(move || LOCAL.with(|local| handle1.block_on(local.run_until(task())))) } -#[allow(unsafe_code)] -unsafe impl Send for DuckSend {} -trait IsSend { - fn is_send() -> bool; -} -impl IsSend for T { - default fn is_send() -> bool { - false - } -} -impl IsSend for T -where - T: Send, -{ - fn is_send() -> bool { - true +#[cfg(test)] +mod tests { + use super::*; + + use futures::future::join_all; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, Arc + }; + + #[tokio::test] + async fn spawn_pinned_() { + const TASKS: usize = 1000; + const ITERS: usize = 1000; + let count = Arc::new(AtomicUsize::new((1..TASKS).sum())); + for _ in 0..ITERS { + join_all((0..TASKS).map(|i| { + let count = count.clone(); + spawn_pinned(move || async move { + let _ = count.fetch_sub(i, Ordering::Relaxed); + }) + })) + .await + .into_iter() + .collect::>() + .unwrap(); + assert_eq!(count.load(Ordering::Relaxed), 0); + count.store((1..TASKS).sum(), Ordering::Relaxed); + } } } From ab890c93dd0e00627b5e9a13495acb71cb64dea6 Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Tue, 30 Jun 2020 23:41:54 +0100 Subject: [PATCH 3/8] spawn_pinned --- Cargo.toml | 1 + azure-pipelines.yml | 4 +-- src/pool/thread.rs | 79 ++++++++++++++++++++++++++++++++++++--------- tests/csv.rs | 1 + tests/csv_dist.rs | 1 + tests/json.rs | 1 + tests/json_dist.rs | 1 + 7 files changed, 70 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 323fded4..68c42842 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ amadeus-commoncrawl = { version = "=0.2.5", path = "amadeus-commoncrawl", option amadeus-parquet = { version = "=0.2.5", path = "amadeus-parquet", optional = true } amadeus-postgres = { version = "=0.2.5", path = "amadeus-postgres", optional = true } amadeus-serde = { version = "=0.2.5", path = "amadeus-serde", optional = true } +async-std = { version = "1.6", features = ["unstable"] } constellation-rs = { version = "0.1", default-features = false, optional = true } derive-new = "0.5" doc-comment = "0.3" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e067a702..8cde4b71 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,7 +20,7 @@ jobs: endpoint: alecmocatta default: rust_toolchain: nightly - rust_lint_toolchain: nightly-2020-06-26 + rust_lint_toolchain: nightly-2020-06-30 rust_flags: '' rust_features_clippy: ';aws;commoncrawl;parquet;postgres;csv;json;constellation aws commoncrawl parquet postgres csv json' rust_features: 'constellation aws commoncrawl parquet postgres csv json' @@ -44,7 +44,7 @@ jobs: endpoint: alecmocatta default: rust_toolchain: nightly - rust_lint_toolchain: nightly-2020-06-26 + rust_lint_toolchain: nightly-2020-06-30 rust_flags: '' rust_features_clippy: ';aws;commoncrawl;parquet;postgres;csv;json;aws commoncrawl parquet postgres csv json' rust_features: 'aws commoncrawl parquet postgres csv json' diff --git a/src/pool/thread.rs b/src/pool/thread.rs index 1d639977..cac13a72 100644 --- a/src/pool/thread.rs +++ b/src/pool/thread.rs @@ -1,6 +1,7 @@ -use futures::TryFutureExt; +use async_std::sync::{channel, Sender}; +use futures::{FutureExt, TryFutureExt}; use std::{ - future::Future, io, panic::{RefUnwindSafe, UnwindSafe}, sync::Arc + any::Any, future::Future, io, panic::{RefUnwindSafe, UnwindSafe}, pin::Pin, sync::Arc }; use tokio::{ runtime::Handle, task::{JoinError, LocalSet} @@ -14,6 +15,7 @@ const DEFAULT_TASKS_PER_CORE: usize = 100; struct ThreadPoolInner { logical_cores: usize, tasks_per_core: usize, + pool: Pool, } #[derive(Debug)] @@ -22,9 +24,11 @@ impl ThreadPool { pub fn new(tasks_per_core: Option) -> io::Result { let logical_cores = num_cpus::get(); let tasks_per_core = tasks_per_core.unwrap_or(DEFAULT_TASKS_PER_CORE); + let pool = Pool::new(logical_cores); Ok(ThreadPool(Arc::new(ThreadPoolInner { logical_cores, tasks_per_core, + pool, }))) } pub fn threads(&self) -> usize { @@ -36,8 +40,9 @@ impl ThreadPool { Fut: Future + 'static, T: Send + 'static, { - let _self = self; - spawn_pinned(|| work()) + self.0 + .pool + .spawn_pinned(|| work()) .map_err(JoinError::into_panic) .map_err(Panicked::from) } @@ -60,18 +65,58 @@ fn _assert() { let _ = assert_sync_and_send::; } -fn spawn_pinned(task: F) -> impl Future> + Send -where - F: FnOnce() -> Fut + Send + 'static, - Fut: Future + 'static, - T: Send + 'static, -{ - thread_local! { - static LOCAL: LocalSet = LocalSet::new(); +#[derive(Debug)] +struct Pool { + sender: Sender<(Request, Sender)>, +} + +type Request = Box Box> + Send>; +type Response = Box; + +impl Pool { + fn new(threads: usize) -> Self { + let handle = Handle::current(); + let handle1 = handle.clone(); + let (sender, receiver) = channel::<(Request, Sender)>(1); + for _ in 0..threads { + let receiver = receiver.clone(); + let handle = handle.clone(); + let _ = handle1.spawn_blocking(move || { + let local = LocalSet::new(); + handle.block_on(local.run_until(async { + while let Ok((task, sender)) = receiver.recv().await { + let _ = local.spawn_local(async move { + let res = Pin::from(task()).await; + sender.send(res).await; + }); + } + })) + }); + } + Self { sender } + } + fn spawn_pinned(&self, task: F) -> impl Future> + Send + where + F: FnOnce() -> Fut + Send + 'static, + Fut: Future + 'static, + T: Send + 'static, + { + let sender = self.sender.clone(); + async move { + let (sender_, receiver) = channel::(1); + sender + .send(( + Box::new(|| Box::new(task().map(|t| Box::new(t) as Box))), + sender_, + )) + .await; + receiver + .recv() + .await + .map(|x| *Box::::downcast(x).unwrap()) + .map_err(|_e| todo!()) + } } - let handle = Handle::current(); - let handle1 = handle.clone(); - handle.spawn_blocking(move || LOCAL.with(|local| handle1.block_on(local.run_until(task())))) } #[cfg(test)] @@ -87,11 +132,13 @@ mod tests { async fn spawn_pinned_() { const TASKS: usize = 1000; const ITERS: usize = 1000; + const THREADS: usize = 4; + let pool = Pool::new(THREADS); let count = Arc::new(AtomicUsize::new((1..TASKS).sum())); for _ in 0..ITERS { join_all((0..TASKS).map(|i| { let count = count.clone(); - spawn_pinned(move || async move { + pool.spawn_pinned(move || async move { let _ = count.fetch_sub(i, Ordering::Relaxed); }) })) diff --git a/tests/csv.rs b/tests/csv.rs index 987d98dd..36681e2c 100644 --- a/tests/csv.rs +++ b/tests/csv.rs @@ -1,3 +1,4 @@ +#![type_length_limit = "1353029"] #![allow(clippy::suspicious_map)] use std::{path::PathBuf, time::SystemTime}; diff --git a/tests/csv_dist.rs b/tests/csv_dist.rs index 73a08769..f2cd49a9 100644 --- a/tests/csv_dist.rs +++ b/tests/csv_dist.rs @@ -1,3 +1,4 @@ +#![type_length_limit = "1726739"] #[cfg(feature = "constellation")] use constellation::*; use std::{ diff --git a/tests/json.rs b/tests/json.rs index 93a26716..24d16f9a 100644 --- a/tests/json.rs +++ b/tests/json.rs @@ -1,3 +1,4 @@ +#![type_length_limit = "1393589"] #![allow(clippy::suspicious_map)] use std::{path::PathBuf, time::SystemTime}; diff --git a/tests/json_dist.rs b/tests/json_dist.rs index 42b4ff91..fa4c2c14 100644 --- a/tests/json_dist.rs +++ b/tests/json_dist.rs @@ -1,3 +1,4 @@ +#![type_length_limit = "1774739"] #[cfg(feature = "constellation")] use constellation::*; use std::{ From da35bfea67b5cbc7d724d745b27552276d865714 Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Sun, 5 Jul 2020 18:19:23 +0100 Subject: [PATCH 4/8] refactor ParallelStream/Pipe/Sink to use Stream/Pipe/Sink traits directly --- amadeus-core/Cargo.toml | 2 +- amadeus-core/src/into_par_stream.rs | 2 +- amadeus-core/src/into_par_stream/iterator.rs | 18 +- amadeus-core/src/into_par_stream/slice.rs | 11 +- amadeus-core/src/lib.rs | 4 +- amadeus-core/src/par_pipe.rs | 65 +- amadeus-core/src/par_sink.rs | 91 ++- amadeus-core/src/par_sink/all.rs | 57 +- amadeus-core/src/par_sink/any.rs | 57 +- amadeus-core/src/par_sink/collect.rs | 176 +++--- amadeus-core/src/par_sink/combiner.rs | 10 +- amadeus-core/src/par_sink/folder.rs | 42 +- amadeus-core/src/par_sink/for_each.rs | 32 +- amadeus-core/src/par_sink/fork.rs | 301 ++++++++++ amadeus-core/src/par_sink/group_by.rs | 202 ++++--- amadeus-core/src/par_sink/pipe.rs | 66 +-- amadeus-core/src/par_sink/tuple.rs | 256 ++++---- amadeus-core/src/par_stream.rs | 593 ++----------------- amadeus-core/src/par_stream/chain.rs | 14 +- amadeus-core/src/par_stream/cloned.rs | 23 +- amadeus-core/src/par_stream/filter.rs | 94 +-- amadeus-core/src/par_stream/flat_map.rs | 75 +-- amadeus-core/src/par_stream/identity.rs | 19 +- amadeus-core/src/par_stream/inspect.rs | 47 +- amadeus-core/src/par_stream/map.rs | 31 +- amadeus-core/src/par_stream/sum_type.rs | 30 +- amadeus-core/src/par_stream/update.rs | 47 +- amadeus-core/src/pipe.rs | 262 ++++++++ amadeus-core/src/pipe/filter.rs | 56 ++ amadeus-core/src/pipe/flat_map.rs | 71 +++ amadeus-core/src/sink.rs | 219 ------- amadeus-core/src/util.rs | 73 +-- amadeus-parquet/src/internal/record/impls.rs | 30 +- amadeus-types/src/list.rs | 6 +- examples/cloudfront_logs.rs | 1 - src/lib.rs | 1 - src/pool/thread.rs | 30 +- src/source.rs | 21 +- tests/cloudfront.rs | 1 - tests/cloudfront_dist.rs | 2 - tests/csv.rs | 1 - tests/csv_dist.rs | 1 - tests/json.rs | 1 - tests/json_dist.rs | 1 - tests/parquet.rs | 1 - tests/parquet_dist.rs | 1 - 46 files changed, 1457 insertions(+), 1687 deletions(-) create mode 100644 amadeus-core/src/par_sink/fork.rs create mode 100644 amadeus-core/src/pipe.rs create mode 100644 amadeus-core/src/pipe/filter.rs create mode 100644 amadeus-core/src/pipe/flat_map.rs delete mode 100644 amadeus-core/src/sink.rs diff --git a/amadeus-core/Cargo.toml b/amadeus-core/Cargo.toml index 8a12a7f7..36de531d 100644 --- a/amadeus-core/Cargo.toml +++ b/amadeus-core/Cargo.toml @@ -32,6 +32,6 @@ replace_with = "0.1" serde = { version = "1.0", features = ["derive"] } serde_closure = "0.2" streaming_algorithms = "0.2" -sum = { version = "0.1", features = ["serde"] } +sum = { version = "0.1", features = ["futures", "serde"] } walkdir = "2.2" widestring = "0.4" diff --git a/amadeus-core/src/into_par_stream.rs b/amadeus-core/src/into_par_stream.rs index 8d138811..61d6269c 100644 --- a/amadeus-core/src/into_par_stream.rs +++ b/amadeus-core/src/into_par_stream.rs @@ -1,4 +1,4 @@ -use crate::par_stream::{DistributedStream, ParallelStream, StreamTask, StreamTaskAsync}; +use crate::par_stream::{DistributedStream, ParallelStream, StreamTask}; mod collections; mod iterator; diff --git a/amadeus-core/src/into_par_stream/iterator.rs b/amadeus-core/src/into_par_stream/iterator.rs index 4919cc48..8aa264bd 100644 --- a/amadeus-core/src/into_par_stream/iterator.rs +++ b/amadeus-core/src/into_par_stream/iterator.rs @@ -1,14 +1,14 @@ -use futures::{pin_mut, stream}; +use futures::Stream; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - iter, ops::{Range, RangeFrom, RangeInclusive}, pin::Pin, task::{Context, Poll} + ops::{Range, RangeFrom, RangeInclusive}, pin::Pin, task::{Context, Poll} }; use super::{ - DistributedStream, IntoDistributedStream, IntoParallelStream, ParallelStream, StreamTask, StreamTaskAsync + DistributedStream, IntoDistributedStream, IntoParallelStream, ParallelStream, StreamTask }; -use crate::{pool::ProcessSend, sink::Sink}; +use crate::pool::ProcessSend; pub trait IteratorExt: Iterator + Sized { fn par(self) -> IterParStream { @@ -56,15 +56,11 @@ impl StreamTask for IterStreamTask { self } } -impl StreamTaskAsync for IterStreamTask { +impl Stream for IterStreamTask { type Item = T; - fn poll_run( - mut self: Pin<&mut Self>, cx: &mut Context, sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - let stream = stream::iter(iter::from_fn(|| self.0.take())); - pin_mut!(stream); - sink.poll_forward(cx, stream) + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { + Poll::Ready(self.project().0.take()) } } diff --git a/amadeus-core/src/into_par_stream/slice.rs b/amadeus-core/src/into_par_stream/slice.rs index f49b03df..88e460e3 100644 --- a/amadeus-core/src/into_par_stream/slice.rs +++ b/amadeus-core/src/into_par_stream/slice.rs @@ -1,12 +1,13 @@ +use futures::Stream; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::{ iter, pin::Pin, slice, task::{Context, Poll} }; use super::{ - DistributedStream, IntoDistributedStream, IntoParallelStream, IterDistStream, IterParStream, ParallelStream, StreamTask, StreamTaskAsync + DistributedStream, IntoDistributedStream, IntoParallelStream, IterDistStream, IterParStream, ParallelStream, StreamTask }; -use crate::{pool::ProcessSend, sink::Sink}; +use crate::pool::ProcessSend; impl_par_dist_rename! { impl IntoParallelStream for [T] @@ -64,12 +65,10 @@ impl StreamTask for Never { self } } -impl StreamTaskAsync for Never { +impl Stream for Never { type Item = Self; - fn poll_run( - self: Pin<&mut Self>, _cx: &mut Context, _sink: Pin<&mut impl Sink>, - ) -> Poll<()> { + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { unreachable!() } } diff --git a/amadeus-core/src/lib.rs b/amadeus-core/src/lib.rs index fc18c396..3ca597c9 100644 --- a/amadeus-core/src/lib.rs +++ b/amadeus-core/src/lib.rs @@ -1,6 +1,4 @@ #![doc(html_root_url = "https://docs.rs/amadeus-core/0.2.5")] -#![feature(specialization)] -#![allow(incomplete_features)] #![recursion_limit = "25600"] macro_rules! impl_par_dist { @@ -64,8 +62,8 @@ pub mod misc_serde; pub mod par_pipe; pub mod par_sink; pub mod par_stream; +pub mod pipe; pub mod pool; -pub mod sink; mod source; pub mod util; diff --git a/amadeus-core/src/par_pipe.rs b/amadeus-core/src/par_pipe.rs index fc8254b7..3630121d 100644 --- a/amadeus-core/src/par_pipe.rs +++ b/amadeus-core/src/par_pipe.rs @@ -1,57 +1,18 @@ use either::Either; use futures::Stream; -use std::{ - cmp::Ordering, future::Future, hash::Hash, iter, ops::{DerefMut, FnMut}, pin::Pin, task::{Context, Poll} -}; +use std::{cmp::Ordering, hash::Hash, iter, ops::FnMut}; -use crate::{pool::ProcessSend, sink::Sink}; +use crate::{pipe::Pipe, pool::ProcessSend}; use super::{par_sink::*, par_stream::*}; #[must_use] pub trait PipeTask { type Item; - type Async: PipeTaskAsync; + type Async: Pipe; fn into_async(self) -> Self::Async; } -#[must_use] -pub trait PipeTaskAsync { - type Item; - - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()>; -} - -impl PipeTaskAsync for Pin

-where - P: DerefMut + Unpin, - P::Target: PipeTaskAsync, -{ - type Item = >::Item; - - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - self.get_mut().as_mut().poll_run(cx, stream, sink) - } -} -impl PipeTaskAsync for &mut T -where - T: PipeTaskAsync + Unpin, -{ - type Item = T::Item; - - fn poll_run( - mut self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - Pin::new(&mut **self).poll_run(cx, stream, sink) - } -} impl_par_dist_rename! { #[must_use] @@ -94,10 +55,9 @@ impl_par_dist_rename! { assert_parallel_pipe(FlatMap::new(self, f)) } - fn filter(self, f: F) -> Filter + fn filter(self, f: F) -> Filter where - F: FnMut(&Self::Item) -> Fut + Clone + Send + 'static, - Fut: Future, + F: FnMut(&Self::Item) -> bool + Clone + Send + 'static, Self: Sized, { assert_parallel_pipe(Filter::new(self, f)) @@ -121,12 +81,21 @@ impl_par_dist_rename! { // assert_parallel_pipe(Chain::new(self, chain.into_par_stream())) // } - fn pipe(self, sink: S) -> Pipe + fn pipe(self, sink: S) -> super::par_sink::Pipe where S: ParallelSink, Self: Sized, { - assert_parallel_sink(Pipe::new(self, sink)) + assert_parallel_sink(super::par_sink::Pipe::new(self, sink)) + } + + fn fork(self, sink: A, sink_ref: B) -> super::par_sink::Fork + where + A: ParallelSink, + B: for<'a> ParallelSink<&'a Self::Item>, + Self: Sized, + { + assert_parallel_sink(super::par_sink::Fork::new(self, sink, sink_ref)) } fn for_each(self, f: F) -> ForEach @@ -151,7 +120,7 @@ impl_par_dist_rename! { where A: Eq + Hash + Send + 'static, S: ParallelSink, - S::Pipe: Clone + Send + 'static, + >::Task: Clone + Send + 'static, S::ReduceA: 'static, S::ReduceC: Clone, S::Output: Send + 'static, diff --git a/amadeus-core/src/par_sink.rs b/amadeus-core/src/par_sink.rs index 95bdb633..21d97a15 100644 --- a/amadeus-core/src/par_sink.rs +++ b/amadeus-core/src/par_sink.rs @@ -7,6 +7,7 @@ mod count; mod fold; mod folder; mod for_each; +mod fork; mod group_by; mod histogram; mod max; @@ -15,76 +16,59 @@ mod sample; mod sum; mod tuple; -use futures::Stream; -use std::{ - ops::DerefMut, pin::Pin, task::{Context, Poll} -}; +use std::{future::Future, ops::DerefMut, pin::Pin}; -use crate::pool::ProcessSend; +use crate::{pipe::Sink, pool::ProcessSend}; use super::par_pipe::*; pub use self::{ - all::*, any::*, collect::*, combine::*, combiner::*, count::*, fold::*, folder::*, for_each::*, group_by::*, histogram::*, max::*, pipe::*, sample::*, sum::*, tuple::* + all::*, any::*, collect::*, combine::*, combiner::*, count::*, fold::*, folder::*, for_each::*, fork::*, group_by::*, histogram::*, max::*, pipe::*, sample::*, sum::*, tuple::* }; #[must_use] -pub trait Reducer { - type Item; +pub trait Reducer { type Output; - type Async: ReducerAsync; + type Async: ReducerAsync; fn into_async(self) -> Self::Async; } -#[must_use] -pub trait ReducerAsync { - type Item; - type Output; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll<()>; - fn poll_output(self: Pin<&mut Self>, cx: &mut Context) -> Poll; -} -pub trait ReducerSend: Reducer::Output> { +pub trait ReducerSend: + Reducer>::Output> +{ type Output: Send + 'static; } -pub trait ReducerProcessSend: ReducerSend::Output> { +pub trait ReducerProcessSend: + ReducerSend>::Output> +{ type Output: ProcessSend + 'static; } +#[must_use] +pub trait ReducerAsync: Sink { + type Output; + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>>; +} -impl

ReducerAsync for Pin

+impl ReducerAsync for Pin

StreamTaskAsync for Pin

-where - P: DerefMut + Unpin, - P::Target: StreamTaskAsync, -{ - type Item = ::Item; - - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - self.get_mut().as_mut().poll_run(cx, sink) - } -} -impl StreamTaskAsync for &mut T -where - T: StreamTaskAsync + Unpin, -{ - type Item = T::Item; - - fn poll_run( - mut self: Pin<&mut Self>, cx: &mut Context, sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - Pin::new(&mut **self).poll_run(cx, sink) - } -} #[async_trait(?Send)] #[must_use] @@ -115,10 +76,9 @@ pub trait DistributedStream { assert_distributed_stream(FlatMap::new(self, f)) } - fn filter(self, f: F) -> Filter + fn filter(self, f: F) -> Filter where - F: FnMut(&Self::Item) -> Fut + Clone + ProcessSend + 'static, - Fut: Future, + F: FnMut(&Self::Item) -> bool + Clone + ProcessSend + 'static, Self: Sized, { assert_distributed_stream(Filter::new(self, f)) @@ -137,9 +97,15 @@ pub trait DistributedStream { ) -> B where P: ProcessPool, - R1: ReducerSend + Clone + ProcessSend + 'static, - R2: ReducerProcessSend::Output> + Clone + ProcessSend + 'static, - R3: Reducer::Output, Output = B>, + R1: ReducerSend + Clone + ProcessSend + 'static, + R2: ReducerProcessSend<>::Output> + + Clone + + ProcessSend + + 'static, + R3: Reducer< + >::Output>>::Output, + Output = B, + >, Self::Task: 'static, Self: Sized, { @@ -192,23 +158,6 @@ pub trait DistributedStream { let reduce_b = reduce_b_factory.clone(); let reduce_a_factory = reduce_a_factory.clone(); pool.spawn(FnOnce!(move |pool: &P::ThreadPool| { - #[pin_project] - struct Connect(#[pin] B); - impl Sink for Connect - where - B: ReducerAsync, - { - type Item = B::Item; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - let self_ = self.project(); - self_.0.poll_forward(cx, stream) - } - } - let mut process_tasks = tasks.into_iter(); let mut tasks = (0..pool.threads()).map(|_| vec![]).collect::>(); @@ -257,22 +206,15 @@ pub trait DistributedStream { .map(|tasks| { let reduce_a = reduce_a_factory.clone(); pool.spawn(move || { - let sink = Connect(reduce_a.into_async()); async move { + let sink = reduce_a.into_async(); pin_mut!(sink); // TODO: short circuit for task in tasks { let task = task.into_async(); - pin_mut!(task); - futures::future::poll_fn(|cx| { - task.as_mut().poll_run(cx, sink.as_mut()) - }) - .await + task.sink(sink.as_mut()).await; } - futures::future::poll_fn(|cx| { - sink.as_mut().project().0.as_mut().poll_output(cx) - }) - .await + sink.output().await } }) }) @@ -285,13 +227,9 @@ pub trait DistributedStream { }); let reduce_b = reduce_b.into_async(); async move { - pin_mut!(stream); pin_mut!(reduce_b); - futures::future::poll_fn(|cx| { - ready!(reduce_b.as_mut().poll_forward(cx, stream.as_mut())); - reduce_b.as_mut().poll_output(cx) - }) - .await + stream.sink(reduce_b.as_mut()).await; + reduce_b.output().await } })) }) @@ -299,14 +237,10 @@ pub trait DistributedStream { let stream = handles.map(|item| { item.unwrap_or_else(|err| panic!("Amadeus: task '' panicked at '{}'", err)) }); - pin_mut!(stream); let reduce_c = reduce_c.into_async(); pin_mut!(reduce_c); - futures::future::poll_fn(|cx| { - ready!(reduce_c.as_mut().poll_forward(cx, stream.as_mut())); - reduce_c.as_mut().poll_output(cx) - }) - .await + stream.sink(reduce_c.as_mut()).await; + reduce_c.output().await } async fn pipe(self, pool: &P, sink: DistSink) -> A @@ -319,74 +253,8 @@ pub trait DistributedStream { Self::Task: 'static, Self: Sized, { - struct Connect(A, B); - impl> DistributedStream for Connect { - type Item = B::Item; - type Task = ConnectTask; - fn size_hint(&self) -> (usize, Option) { - self.0.size_hint() - } - fn next_task(&mut self) -> Option { - self.0 - .next_task() - .map(|task| ConnectTask(task, self.1.task())) - } - } - #[derive(Serialize, Deserialize)] - #[serde( - bound(serialize = "A: Serialize, B: Serialize"), - bound(deserialize = "A: Deserialize<'de>, B: Deserialize<'de>") - )] - struct ConnectTask(A, B); - impl StreamTask for ConnectTask - where - A: StreamTask, - B: PipeTask, - { - type Item = B::Item; - type Async = ConnectStreamTaskAsync; - fn into_async(self) -> Self::Async { - ConnectStreamTaskAsync(self.0.into_async(), self.1.into_async()) - } - } - #[pin_project] - struct ConnectStreamTaskAsync(#[pin] A, #[pin] B); - impl StreamTaskAsync for ConnectStreamTaskAsync - where - A: StreamTaskAsync, - B: PipeTaskAsync, - { - type Item = B::Item; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - #[pin_project] - struct Proxy<'a, I, B, Item>(#[pin] I, Pin<&'a mut B>, PhantomData Item>); - impl<'a, I, B, Item> Sink for Proxy<'a, I, B, Item> - where - I: Sink, - B: PipeTaskAsync, - { - type Item = Item; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - let self_ = self.project(); - self_.1.as_mut().poll_run(cx, stream, self_.0) - } - } - let self_ = self.project(); - let sink = Proxy(sink, self_.1, PhantomData); - pin_mut!(sink); - self_.0.poll_run(cx, sink) - } - } - let (iterator, reducer_a, reducer_b, reducer_c) = sink.reducers(); - Connect(self, iterator) + Pipe::new(self, iterator) .reduce(pool, reducer_a, reducer_b, reducer_c) .await } @@ -411,160 +279,9 @@ pub trait DistributedStream { Self::Task: 'static, Self: Sized, { - struct Connect(A, B, C, PhantomData RefAItem>); - impl DistributedStream for Connect - where - A: DistributedStream, - B: DistributedPipe, - C: DistributedPipe, - RefAItem: 'static, - { - type Item = Sum2; - type Task = ConnectTask; - fn size_hint(&self) -> (usize, Option) { - self.0.size_hint() - } - fn next_task(&mut self) -> Option { - self.0 - .next_task() - .map(|task| ConnectTask(task, self.1.task(), self.2.task(), PhantomData)) - } - } - #[derive(Serialize, Deserialize)] - #[serde( - bound(serialize = "A: Serialize, B: Serialize, C: Serialize"), - bound(deserialize = "A: Deserialize<'de>, B: Deserialize<'de>, C: Deserialize<'de>") - )] - struct ConnectTask(A, B, C, PhantomData RefAItem>); - impl StreamTask for ConnectTask - where - A: StreamTask, - B: PipeTask, - C: PipeTask, - { - type Item = Sum2; - type Async = ConnectStreamTaskAsync; - fn into_async(self) -> Self::Async { - ConnectStreamTaskAsync( - self.0.into_async(), - self.1.into_async(), - self.2.into_async(), - false, - None, - PhantomData, - ) - } - } - #[pin_project] - struct ConnectStreamTaskAsync( - #[pin] A, - #[pin] B, - #[pin] C, - bool, - Option>, - PhantomData RefAItem>, - ); - impl StreamTaskAsync for ConnectStreamTaskAsync - where - A: StreamTaskAsync, - B: PipeTaskAsync, - C: PipeTaskAsync, - { - type Item = Sum2; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - #[pin_project] - pub struct SinkFn<'a, S, A, B, T, RefAItem>( - #[pin] S, - Pin<&'a mut A>, - Pin<&'a mut B>, - &'a mut bool, - &'a mut Option>, - PhantomData (T, RefAItem)>, - ); - impl<'a, S, A, B, T, RefAItem> Sink for SinkFn<'a, S, A, B, T, RefAItem> - where - S: Sink>, - A: PipeTaskAsync, - B: PipeTaskAsync, - { - type Item = T; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - let mut self_ = self.project(); - let mut ready = (false, false); - loop { - if self_.4.is_none() { - **self_.4 = match stream.as_mut().poll_next(cx) { - Poll::Ready(x) => Some(x), - Poll::Pending => None, - }; - } - let given = &mut **self_.3; - let pending = &mut **self_.4; - let mut progress = false; - if !ready.0 { - let stream = stream::poll_fn(|_cx| match pending { - Some(x) if !*given => { - *given = true; - progress = true; - Poll::Ready(type_coerce(x.as_ref())) - } - _ => Poll::Pending, - }) - .fuse(); - pin_mut!(stream); - let sink_ = SinkMap::new(self_.0.as_mut(), |item| { - Sum2::B(type_coerce(item)) - }); - pin_mut!(sink_); - ready.0 = self_.2.as_mut().poll_run(cx, stream, sink_).is_ready(); - } - if !ready.1 { - let stream = stream::poll_fn(|_cx| { - if !ready.0 && !*given { - return Poll::Pending; - } - match pending.take() { - Some(x) => { - *given = false; - progress = true; - Poll::Ready(x) - } - None => Poll::Pending, - } - }) - .fuse(); - pin_mut!(stream); - let sink_ = SinkMap::new(self_.0.as_mut(), Sum2::A); - pin_mut!(sink_); - ready.1 = self_.1.as_mut().poll_run(cx, stream, sink_).is_ready(); - } - if ready.0 && ready.1 { - break Poll::Ready(()); - } - if !progress { - break Poll::Pending; - } - } - } - } - - let self_ = self.project(); - let sink = SinkFn(sink, self_.1, self_.2, self_.3, self_.4, PhantomData); - pin_mut!(sink); - self_.0.poll_run(cx, sink) - } - } - let (iterator_a, reducer_a_a, reducer_a_b, reducer_a_c) = sink_a.reducers(); let (iterator_b, reducer_b_a, reducer_b_b, reducer_b_c) = sink_b.reducers(); - Connect(self, iterator_a, iterator_b, PhantomData) + Fork::new(self, iterator_a, iterator_b) .reduce( pool, ReduceA2::new(reducer_a_a, reducer_b_a), @@ -609,7 +326,7 @@ pub trait DistributedStream { A: Eq + Hash + ProcessSend + 'static, B: 'static, S: DistributedSink, - S::Pipe: Clone + ProcessSend + 'static, + >::Task: Clone + ProcessSend + 'static, S::ReduceA: 'static, S::ReduceB: 'static, S::ReduceC: Clone, @@ -881,10 +598,9 @@ pub trait ParallelStream { assert_parallel_stream(FlatMap::new(self, f)) } - fn filter(self, f: F) -> Filter + fn filter(self, f: F) -> Filter where - F: FnMut(&Self::Item) -> Fut + Clone + Send + 'static, - Fut: Future, + F: FnMut(&Self::Item) -> bool + Clone + Send + 'static, Self: Sized, { assert_parallel_stream(Filter::new(self, f)) @@ -901,8 +617,8 @@ pub trait ParallelStream { async fn reduce(mut self, pool: &P, reduce_a_factory: R1, reduce_c: R3) -> B where P: ThreadPool, - R1: ReducerSend + Clone + Send + 'static, - R3: Reducer::Output, Output = B>, + R1: ReducerSend + Clone + Send + 'static, + R3: Reducer<>::Output, Output = B>, Self::Task: 'static, Self: Sized, { @@ -954,37 +670,15 @@ pub trait ParallelStream { .map(|tasks| { let reduce_a = reduce_a_factory.clone(); pool.spawn(FnOnce!(move || { - #[pin_project] - struct Connect(#[pin] B); - impl Sink for Connect - where - B: ReducerAsync, - { - type Item = B::Item; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - let self_ = self.project(); - self_.0.poll_forward(cx, stream) - } - } - - let sink = Connect(reduce_a.into_async()); async move { + let sink = reduce_a.into_async(); pin_mut!(sink); // TODO: short circuit for task in tasks { let task = task.into_async(); - pin_mut!(task); - futures::future::poll_fn(|cx| task.as_mut().poll_run(cx, sink.as_mut())) - .await + task.sink(sink.as_mut()).await; } - futures::future::poll_fn(|cx| { - sink.as_mut().project().0.as_mut().poll_output(cx) - }) - .await + sink.output().await } })) }) @@ -992,14 +686,10 @@ pub trait ParallelStream { let stream = handles.map(|item| { item.unwrap_or_else(|err| panic!("Amadeus: task '' panicked at '{}'", err)) }); - pin_mut!(stream); let reduce_c = reduce_c.into_async(); pin_mut!(reduce_c); - futures::future::poll_fn(|cx| { - ready!(reduce_c.as_mut().poll_forward(cx, stream.as_mut())); - reduce_c.as_mut().poll_output(cx) - }) - .await + stream.sink(reduce_c.as_mut()).await; + reduce_c.output().await } async fn pipe(self, pool: &P, sink: ParSink) -> A @@ -1011,69 +701,8 @@ pub trait ParallelStream { Self::Task: 'static, Self: Sized, { - struct Connect(A, B); - impl> ParallelStream for Connect { - type Item = B::Item; - type Task = ConnectTask; - fn size_hint(&self) -> (usize, Option) { - self.0.size_hint() - } - fn next_task(&mut self) -> Option { - self.0 - .next_task() - .map(|task| ConnectTask(task, self.1.task())) - } - } - struct ConnectTask(A, B); - impl StreamTask for ConnectTask - where - A: StreamTask, - B: PipeTask, - { - type Item = B::Item; - type Async = ConnectStreamTaskAsync; - fn into_async(self) -> Self::Async { - ConnectStreamTaskAsync(self.0.into_async(), self.1.into_async()) - } - } - #[pin_project] - struct ConnectStreamTaskAsync(#[pin] A, #[pin] B); - impl StreamTaskAsync for ConnectStreamTaskAsync - where - A: StreamTaskAsync, - B: PipeTaskAsync, - { - type Item = B::Item; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - #[pin_project] - struct Proxy<'a, I, B, Item>(#[pin] I, Pin<&'a mut B>, PhantomData Item>); - impl<'a, I, B, Item> Sink for Proxy<'a, I, B, Item> - where - I: Sink, - B: PipeTaskAsync, - { - type Item = Item; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - let self_ = self.project(); - self_.1.as_mut().poll_run(cx, stream, self_.0) - } - } - let self_ = self.project(); - let sink = Proxy(sink, self_.1, PhantomData); - pin_mut!(sink); - self_.0.poll_run(cx, sink) - } - } - let (iterator, reducer_a, reducer_b) = sink.reducers(); - Connect(self, iterator) + Pipe::new(self, iterator) .reduce(pool, reducer_a, reducer_b) .await } @@ -1096,157 +725,9 @@ pub trait ParallelStream { Self::Task: 'static, Self: Sized, { - struct Connect(A, B, C, PhantomData RefAItem>); - impl ParallelStream for Connect - where - A: ParallelStream, - B: ParallelPipe, - C: ParallelPipe, - B::Task: 'static, - C::Task: 'static, - RefAItem: 'static, - { - type Item = Sum2; - type Task = ConnectTask; - fn size_hint(&self) -> (usize, Option) { - self.0.size_hint() - } - fn next_task(&mut self) -> Option { - self.0 - .next_task() - .map(|task| ConnectTask(task, self.1.task(), self.2.task(), PhantomData)) - } - } - struct ConnectTask(A, B, C, PhantomData RefAItem>); - impl StreamTask for ConnectTask - where - A: StreamTask, - B: PipeTask, - C: PipeTask, - { - type Item = Sum2; - type Async = ConnectStreamTaskAsync; - fn into_async(self) -> Self::Async { - ConnectStreamTaskAsync( - self.0.into_async(), - self.1.into_async(), - self.2.into_async(), - false, - None, - PhantomData, - ) - } - } - #[pin_project] - struct ConnectStreamTaskAsync( - #[pin] A, - #[pin] B, - #[pin] C, - bool, - Option>, - PhantomData RefAItem>, - ); - impl StreamTaskAsync for ConnectStreamTaskAsync - where - A: StreamTaskAsync, - B: PipeTaskAsync, - C: PipeTaskAsync, - { - type Item = Sum2; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - #[pin_project] - pub struct SinkFn<'a, S, A, B, T, RefAItem>( - #[pin] S, - Pin<&'a mut A>, - Pin<&'a mut B>, - &'a mut bool, - &'a mut Option>, - PhantomData (T, RefAItem)>, - ); - impl<'a, S, A, B, T, RefAItem> Sink for SinkFn<'a, S, A, B, T, RefAItem> - where - S: Sink>, - A: PipeTaskAsync, - B: PipeTaskAsync, - { - type Item = T; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - let mut self_ = self.project(); - let mut ready = (false, false); - loop { - if self_.4.is_none() { - **self_.4 = match stream.as_mut().poll_next(cx) { - Poll::Ready(x) => Some(x), - Poll::Pending => None, - }; - } - let given = &mut **self_.3; - let pending = &mut **self_.4; - let mut progress = false; - if !ready.0 { - let stream = stream::poll_fn(|_cx| match pending { - Some(x) if !*given => { - *given = true; - progress = true; - Poll::Ready(type_coerce(x.as_ref())) - } - _ => Poll::Pending, - }) - .fuse(); - pin_mut!(stream); - let sink_ = SinkMap::new(self_.0.as_mut(), |item| { - Sum2::B(type_coerce(item)) - }); - pin_mut!(sink_); - ready.0 = self_.2.as_mut().poll_run(cx, stream, sink_).is_ready(); - } - if !ready.1 { - let stream = stream::poll_fn(|_cx| { - if !ready.0 && !*given { - return Poll::Pending; - } - match pending.take() { - Some(x) => { - *given = false; - progress = true; - Poll::Ready(x) - } - None => Poll::Pending, - } - }) - .fuse(); - pin_mut!(stream); - let sink_ = SinkMap::new(self_.0.as_mut(), Sum2::A); - pin_mut!(sink_); - ready.1 = self_.1.as_mut().poll_run(cx, stream, sink_).is_ready(); - } - if ready.0 && ready.1 { - break Poll::Ready(()); - } - if !progress { - break Poll::Pending; - } - } - } - } - - let self_ = self.project(); - let sink = SinkFn(sink, self_.1, self_.2, self_.3, self_.4, PhantomData); - pin_mut!(sink); - self_.0.poll_run(cx, sink) - } - } - let (iterator_a, reducer_a_a, reducer_a_b) = sink_a.reducers(); let (iterator_b, reducer_b_a, reducer_b_b) = sink_b.reducers(); - Connect(self, iterator_a, iterator_b, PhantomData) + Fork::new(self, iterator_a, iterator_b) .reduce( pool, ReduceA2::new(reducer_a_a, reducer_b_a), @@ -1290,7 +771,7 @@ pub trait ParallelStream { A: Eq + Hash + Send + 'static, B: 'static, S: ParallelSink, - S::Pipe: Clone + Send + 'static, + >::Task: Clone + Send + 'static, S::ReduceA: 'static, S::ReduceC: Clone, S::Output: Send + 'static, diff --git a/amadeus-core/src/par_stream/chain.rs b/amadeus-core/src/par_stream/chain.rs index 74985394..13a52cf4 100644 --- a/amadeus-core/src/par_stream/chain.rs +++ b/amadeus-core/src/par_stream/chain.rs @@ -1,12 +1,12 @@ use derive_new::new; +use futures::Stream; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ pin::Pin, task::{Context, Poll} }; -use super::{ParallelStream, StreamTask, StreamTaskAsync}; -use crate::sink::Sink; +use super::{ParallelStream, StreamTask}; #[derive(new)] #[must_use] @@ -57,15 +57,13 @@ impl> StreamTask for ChainTask> StreamTaskAsync for ChainTask { +impl> Stream for ChainTask { type Item = A::Item; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, sink: Pin<&mut impl Sink>, - ) -> Poll<()> { + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { match self.project() { - ChainTaskProj::A(a) => a.poll_run(cx, sink), - ChainTaskProj::B(b) => b.poll_run(cx, sink), + ChainTaskProj::A(a) => a.poll_next(cx), + ChainTaskProj::B(b) => b.poll_next(cx), } } } diff --git a/amadeus-core/src/par_stream/cloned.rs b/amadeus-core/src/par_stream/cloned.rs index 0ccec5b1..e5f65758 100644 --- a/amadeus-core/src/par_stream/cloned.rs +++ b/amadeus-core/src/par_stream/cloned.rs @@ -1,13 +1,13 @@ use derive_new::new; -use futures::{pin_mut, Stream}; +use futures::Stream; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ marker::PhantomData, pin::Pin, task::{Context, Poll} }; -use super::{ParallelPipe, PipeTask, PipeTaskAsync}; -use crate::sink::{Sink, SinkMap}; +use super::{ParallelPipe, PipeTask}; +use crate::pipe::Pipe; #[derive(new)] #[must_use] @@ -64,6 +64,7 @@ where { type Item = T; type Async = ClonedTask; + fn into_async(self) -> Self::Async { ClonedTask { task: self.task.into_async(), @@ -80,19 +81,19 @@ where // self.task.run(&mut |item| i(item.clone())) // } // } -impl<'a, C, Source: 'a, T: 'a> PipeTaskAsync<&'a Source> for ClonedTask +impl<'a, C, Source: 'a, T: 'a> Pipe<&'a Source> for ClonedTask where - C: PipeTaskAsync<&'a Source, Item = &'a T>, + C: Pipe<&'a Source, Item = &'a T>, T: Clone, { type Item = T; - fn poll_run( + fn poll_next( self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - let sink = SinkMap::new(sink, |item: &T| item.clone()); - pin_mut!(sink); - self.project().task.poll_run(cx, stream, sink) + ) -> Poll> { + self.project() + .task + .poll_next(cx, stream) + .map(|item| item.cloned()) } } diff --git a/amadeus-core/src/par_stream/filter.rs b/amadeus-core/src/par_stream/filter.rs index a42ad9e4..d3b2fd5a 100644 --- a/amadeus-core/src/par_stream/filter.rs +++ b/amadeus-core/src/par_stream/filter.rs @@ -1,13 +1,8 @@ use derive_new::new; -use futures::{pin_mut, Stream}; -use pin_project::pin_project; use serde::{Deserialize, Serialize}; -use std::{ - future::Future, pin::Pin, task::{Context, Poll} -}; -use super::{ParallelPipe, ParallelStream, PipeTask, PipeTaskAsync, StreamTask, StreamTaskAsync}; -use crate::sink::{Sink, SinkFilter, SinkFilterState}; +use super::{ParallelPipe, ParallelStream, PipeTask, StreamTask}; +use crate::pipe::{Pipe, StreamExt}; #[derive(new)] #[must_use] @@ -17,10 +12,9 @@ pub struct Filter { } impl_par_dist! { - impl ParallelStream for Filter + impl ParallelStream for Filter where - F: FnMut(&I::Item) -> Fut + Clone + Send + 'static, - Fut: Future, + F: FnMut(&I::Item) -> bool + Clone + Send + 'static, { type Item = I::Item; type Task = FilterTask; @@ -36,10 +30,9 @@ impl_par_dist! { } } - impl, F, Fut, Source> ParallelPipe for Filter + impl, F, Source> ParallelPipe for Filter where - F: FnMut(&>::Item) -> Fut + Clone + Send + 'static, - Fut: Future, + F: FnMut(&I::Item) -> bool + Clone + Send + 'static, { type Item = I::Item; type Task = FilterTask; @@ -58,80 +51,25 @@ pub struct FilterTask { f: F, } -impl StreamTask for FilterTask +impl StreamTask for FilterTask where - F: FnMut(&C::Item) -> Fut + Clone, - Fut: Future, + F: FnMut(&C::Item) -> bool, { type Item = C::Item; - type Async = FilterStreamTaskAsync; - fn into_async(self) -> Self::Async { - FilterStreamTaskAsync { - task: self.task.into_async(), - f: self.f, - state: SinkFilterState::None, - } - } -} -impl, F, Fut, Source> PipeTask for FilterTask -where - F: FnMut(&C::Item) -> Fut + Clone, - Fut: Future, -{ - type Item = C::Item; - type Async = FilterStreamTaskAsync; - fn into_async(self) -> Self::Async { - FilterStreamTaskAsync { - task: self.task.into_async(), - f: self.f, - state: SinkFilterState::None, - } - } -} + type Async = crate::pipe::Filter; -#[pin_project] -pub struct FilterStreamTaskAsync { - #[pin] - task: C, - f: F, - #[pin] - state: SinkFilterState, -} - -impl StreamTaskAsync for FilterStreamTaskAsync -where - F: FnMut(&C::Item) -> Fut + Clone, - Fut: Future, -{ - type Item = C::Item; - - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - let mut self_ = self.project(); - let (task, f) = (self_.task, &mut self_.f); - let sink = SinkFilter::new(self_.state, sink, |item: &_| f(item)); - pin_mut!(sink); - task.poll_run(cx, sink) + fn into_async(self) -> Self::Async { + self.task.into_async().filter(self.f) } } - -impl, F, Fut, Source> PipeTaskAsync - for FilterStreamTaskAsync +impl, F, Source> PipeTask for FilterTask where - F: FnMut(&>::Item) -> Fut + Clone, - Fut: Future, + F: FnMut(&C::Item) -> bool, { type Item = C::Item; + type Async = crate::pipe::Filter; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - let mut self_ = self.project(); - let (task, f) = (self_.task, &mut self_.f); - let sink = SinkFilter::new(self_.state, sink, |item: &_| f(item)); - pin_mut!(sink); - task.poll_run(cx, stream, sink) + fn into_async(self) -> Self::Async { + self.task.into_async().filter(self.f) } } diff --git a/amadeus-core/src/par_stream/flat_map.rs b/amadeus-core/src/par_stream/flat_map.rs index 025c572a..9b52ff8e 100644 --- a/amadeus-core/src/par_stream/flat_map.rs +++ b/amadeus-core/src/par_stream/flat_map.rs @@ -1,13 +1,9 @@ use derive_new::new; -use futures::{pin_mut, Stream}; -use pin_project::pin_project; +use futures::Stream; use serde::{Deserialize, Serialize}; -use std::{ - pin::Pin, task::{Context, Poll} -}; -use super::{ParallelPipe, ParallelStream, PipeTask, PipeTaskAsync, StreamTask, StreamTaskAsync}; -use crate::sink::{Sink, SinkFlatMap}; +use super::{ParallelPipe, ParallelStream, PipeTask, StreamTask}; +use crate::pipe::{Pipe, StreamExt}; #[derive(new)] #[must_use] @@ -37,7 +33,7 @@ impl_par_dist! { impl, F, R: Stream, Source> ParallelPipe for FlatMap where - F: FnMut(>::Item) -> R + Clone + Send + 'static, + F: FnMut(I::Item) -> R + Clone + Send + 'static, { type Item = R::Item; type Task = FlatMapTask; @@ -57,70 +53,19 @@ pub struct FlatMapTask { } impl R + Clone, R: Stream> StreamTask for FlatMapTask { type Item = R::Item; - type Async = FlatMapStreamTaskAsync; + type Async = crate::pipe::FlatMap; + fn into_async(self) -> Self::Async { - FlatMapStreamTaskAsync { - task: self.task.into_async(), - f: self.f, - fut: None, - } + self.task.into_async().flat_map(self.f) } } impl, F: FnMut(C::Item) -> R + Clone, R: Stream, Source> PipeTask for FlatMapTask { type Item = R::Item; - type Async = FlatMapStreamTaskAsync; - fn into_async(self) -> Self::Async { - FlatMapStreamTaskAsync { - task: self.task.into_async(), - f: self.f, - fut: None, - } - } -} - -#[pin_project] -#[derive(Serialize, Deserialize)] -pub struct FlatMapStreamTaskAsync { - #[pin] - task: C, - f: F, - #[pin] - fut: Option, -} - -impl R + Clone, R: Stream> StreamTaskAsync - for FlatMapStreamTaskAsync -{ - type Item = R::Item; + type Async = crate::pipe::FlatMap; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - let mut self_ = self.project(); - let (task, f) = (self_.task, &mut self_.f); - let sink = SinkFlatMap::new(self_.fut, sink, |item| f(item)); - pin_mut!(sink); - task.poll_run(cx, sink) - } -} - -impl, F, R: Stream, Source> PipeTaskAsync - for FlatMapStreamTaskAsync -where - F: FnMut(>::Item) -> R + Clone, -{ - type Item = R::Item; - - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - let mut self_ = self.project(); - let (task, f) = (self_.task, &mut self_.f); - let sink = SinkFlatMap::new(self_.fut, sink, |item| f(item)); - pin_mut!(sink); - task.poll_run(cx, stream, sink) + fn into_async(self) -> Self::Async { + self.task.into_async().flat_map(self.f) } } diff --git a/amadeus-core/src/par_stream/identity.rs b/amadeus-core/src/par_stream/identity.rs index bdd5ac3b..8ccc351d 100644 --- a/amadeus-core/src/par_stream/identity.rs +++ b/amadeus-core/src/par_stream/identity.rs @@ -5,9 +5,8 @@ use std::{ }; use super::{ - All, Any, Collect, Combine, Count, Filter, FlatMap, Fold, ForEach, GroupBy, Histogram, Inspect, Map, Max, MaxBy, MaxByKey, Min, MinBy, MinByKey, MostDistinct, MostFrequent, ParallelPipe, Pipe, PipeTask, PipeTaskAsync, SampleUnstable, Sum, Update + All, Any, Collect, Combine, Count, Filter, FlatMap, Fold, ForEach, Fork, GroupBy, Histogram, Inspect, Map, Max, MaxBy, MaxByKey, Min, MinBy, MinByKey, MostDistinct, MostFrequent, ParallelPipe, Pipe, PipeTask, SampleUnstable, Sum, Update }; -use crate::sink::Sink; // TODO: add type parameter to Identity when type the type system includes HRTB in the ParallelPipe impl https://github.com/dtolnay/ghost/ @@ -35,6 +34,10 @@ mod workaround { Pipe::new(self, sink) } + pub fn fork(self, sink: A, sink_ref: B) -> Fork { + Fork::new(self, sink, sink_ref) + } + pub fn inspect(self, f: F) -> Inspect where F: Clone + Send + 'static, @@ -192,22 +195,22 @@ mod workaround { } } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct IdentityTask; impl PipeTask for IdentityTask { type Item = Item; type Async = IdentityTask; + fn into_async(self) -> Self::Async { IdentityTask } } -impl PipeTaskAsync for IdentityTask { +impl crate::pipe::Pipe for IdentityTask { type Item = Item; - fn poll_run( + fn poll_next( self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - sink.poll_forward(cx, stream) + ) -> Poll> { + stream.poll_next(cx) } } diff --git a/amadeus-core/src/par_stream/inspect.rs b/amadeus-core/src/par_stream/inspect.rs index 8005ff4e..76b53aa6 100644 --- a/amadeus-core/src/par_stream/inspect.rs +++ b/amadeus-core/src/par_stream/inspect.rs @@ -1,13 +1,13 @@ use derive_new::new; -use futures::{pin_mut, Stream}; +use futures::Stream; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ pin::Pin, task::{Context, Poll} }; -use super::{ParallelPipe, ParallelStream, PipeTask, PipeTaskAsync, StreamTask, StreamTaskAsync}; -use crate::sink::{Sink, SinkMap}; +use super::{ParallelPipe, ParallelStream, PipeTask, StreamTask}; +use crate::pipe::Pipe; #[derive(new)] #[must_use] @@ -37,7 +37,7 @@ impl_par_dist! { impl, F, Source> ParallelPipe for Inspect where - F: FnMut(&>::Item) + Clone + Send + 'static, + F: FnMut(&I::Item) + Clone + Send + 'static, { type Item = I::Item; type Task = InspectTask; @@ -85,43 +85,40 @@ where } } -impl StreamTaskAsync for InspectTask +impl Stream for InspectTask where F: FnMut(&C::Item) + Clone, { type Item = C::Item; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, sink: Pin<&mut impl Sink>, - ) -> Poll<()> { + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { let mut self_ = self.project(); let (task, f) = (self_.task, &mut self_.f); - let sink = SinkMap::new(sink, |item| { - f(&item); - item - }); - pin_mut!(sink); - task.poll_run(cx, sink) + task.poll_next(cx).map(|item| { + item.map(|item| { + f(&item); + item + }) + }) } } -impl, F, Source> PipeTaskAsync for InspectTask +impl, F, Source> Pipe for InspectTask where - F: FnMut(&>::Item) + Clone, + F: FnMut(&C::Item) + Clone, { type Item = C::Item; - fn poll_run( + fn poll_next( self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { + ) -> Poll> { let mut self_ = self.project(); let (task, f) = (self_.task, &mut self_.f); - let sink = SinkMap::new(sink, |item| { - f(&item); - item - }); - pin_mut!(sink); - task.poll_run(cx, stream, sink) + task.poll_next(cx, stream).map(|item| { + item.map(|item| { + f(&item); + item + }) + }) } } diff --git a/amadeus-core/src/par_stream/map.rs b/amadeus-core/src/par_stream/map.rs index f5a46c5f..14741260 100644 --- a/amadeus-core/src/par_stream/map.rs +++ b/amadeus-core/src/par_stream/map.rs @@ -1,13 +1,13 @@ use derive_new::new; -use futures::{pin_mut, Stream}; +use futures::Stream; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ pin::Pin, task::{Context, Poll} }; -use super::{ParallelPipe, ParallelStream, PipeTask, PipeTaskAsync, StreamTask, StreamTaskAsync}; -use crate::sink::{Sink, SinkMap}; +use super::{ParallelPipe, ParallelStream, PipeTask, StreamTask}; +use crate::pipe::Pipe; #[derive(new)] #[must_use] @@ -37,7 +37,7 @@ impl_par_dist! { impl, F, R, Source> ParallelPipe for Map where - F: FnMut(>::Item) -> R + Clone + Send + 'static, + F: FnMut(I::Item) -> R + Clone + Send + 'static, { type Item = R; type Task = MapTask; @@ -85,37 +85,30 @@ where } } -impl StreamTaskAsync for MapTask +impl Stream for MapTask where F: FnMut(C::Item) -> R + Clone, { type Item = R; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, sink: Pin<&mut impl Sink>, - ) -> Poll<()> { + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { let mut self_ = self.project(); let (task, f) = (self_.task, &mut self_.f); - let sink = SinkMap::new(sink, |item| f(item)); - pin_mut!(sink); - task.poll_run(cx, sink) + task.poll_next(cx).map(|t| t.map(f)) } } -impl, F, R, Source> PipeTaskAsync for MapTask +impl, F, R, Source> Pipe for MapTask where - F: FnMut(>::Item) -> R + Clone, + F: FnMut(C::Item) -> R + Clone, { type Item = R; - fn poll_run( + fn poll_next( self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { + ) -> Poll> { let mut self_ = self.project(); let (task, f) = (self_.task, &mut self_.f); - let sink = SinkMap::new(sink, |item| f(item)); - pin_mut!(sink); - task.poll_run(cx, stream, sink) + task.poll_next(cx, stream).map(|t| t.map(f)) } } diff --git a/amadeus-core/src/par_stream/sum_type.rs b/amadeus-core/src/par_stream/sum_type.rs index df9f577f..8037d82d 100644 --- a/amadeus-core/src/par_stream/sum_type.rs +++ b/amadeus-core/src/par_stream/sum_type.rs @@ -4,8 +4,8 @@ use std::{ }; use sum::Sum2; -use super::{ParallelPipe, ParallelStream, PipeTask, PipeTaskAsync, StreamTask, StreamTaskAsync}; -use crate::sink::Sink; +use super::{ParallelPipe, ParallelStream, PipeTask, StreamTask}; +use crate::pipe::Pipe; impl_par_dist! { impl> ParallelStream for Sum2 { @@ -64,31 +64,15 @@ impl, B: PipeTask, Source> PipeTask< } } -impl> StreamTaskAsync for Sum2 { +impl, B: Pipe, Source> Pipe for Sum2 { type Item = A::Item; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - match self.as_pin_mut() { - Sum2::A(task) => task.poll_run(cx, sink), - Sum2::B(task) => task.poll_run(cx, sink), - } - } -} - -impl, B: PipeTaskAsync, Source> - PipeTaskAsync for Sum2 -{ - type Item = A::Item; - - fn poll_run( + fn poll_next( self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { + ) -> Poll> { match self.as_pin_mut() { - Sum2::A(task) => task.poll_run(cx, stream, sink), - Sum2::B(task) => task.poll_run(cx, stream, sink), + Sum2::A(task) => task.poll_next(cx, stream), + Sum2::B(task) => task.poll_next(cx, stream), } } } diff --git a/amadeus-core/src/par_stream/update.rs b/amadeus-core/src/par_stream/update.rs index d6d8e943..9a4c1b85 100644 --- a/amadeus-core/src/par_stream/update.rs +++ b/amadeus-core/src/par_stream/update.rs @@ -1,13 +1,13 @@ use derive_new::new; -use futures::{pin_mut, Stream}; +use futures::Stream; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ pin::Pin, task::{Context, Poll} }; -use super::{ParallelPipe, ParallelStream, PipeTask, PipeTaskAsync, StreamTask, StreamTaskAsync}; -use crate::sink::{Sink, SinkMap}; +use super::{ParallelPipe, ParallelStream, PipeTask, StreamTask}; +use crate::pipe::Pipe; #[derive(new)] #[must_use] @@ -37,7 +37,7 @@ impl_par_dist! { impl, F, Source> ParallelPipe for Update where - F: FnMut(&mut >::Item) + Clone + Send + 'static, + F: FnMut(&mut I::Item) + Clone + Send + 'static, { type Item = I::Item; type Task = UpdateTask; @@ -85,43 +85,40 @@ where } } -impl StreamTaskAsync for UpdateTask +impl Stream for UpdateTask where F: FnMut(&mut C::Item) + Clone, { type Item = C::Item; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, sink: Pin<&mut impl Sink>, - ) -> Poll<()> { + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { let mut self_ = self.project(); let (task, f) = (self_.task, &mut self_.f); - let sink = SinkMap::new(sink, |mut item| { - f(&mut item); - item - }); - pin_mut!(sink); - task.poll_run(cx, sink) + task.poll_next(cx).map(|item| { + item.map(|mut item| { + f(&mut item); + item + }) + }) } } -impl, F, Source> PipeTaskAsync for UpdateTask +impl, F, Source> Pipe for UpdateTask where - F: FnMut(&mut >::Item) + Clone, + F: FnMut(&mut C::Item) + Clone, { type Item = C::Item; - fn poll_run( + fn poll_next( self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { + ) -> Poll> { let mut self_ = self.project(); let (task, f) = (self_.task, &mut self_.f); - let sink = SinkMap::new(sink, |mut item| { - f(&mut item); - item - }); - pin_mut!(sink); - task.poll_run(cx, stream, sink) + task.poll_next(cx, stream).map(|item| { + item.map(|mut item| { + f(&mut item); + item + }) + }) } } diff --git a/amadeus-core/src/pipe.rs b/amadeus-core/src/pipe.rs new file mode 100644 index 00000000..44af5da4 --- /dev/null +++ b/amadeus-core/src/pipe.rs @@ -0,0 +1,262 @@ +mod filter; +mod flat_map; + +use futures::{pin_mut, Future, Stream}; +use pin_project::pin_project; +use std::{ + ops::DerefMut, pin::Pin, task::{Context, Poll} +}; + +pub use self::{filter::*, flat_map::*}; + +// Sink takes Source as an input parameter rather than associated type to accept +// for<'a> &'a T, but this might not be necessary in future? +// https://github.com/rust-lang/rust/issues/49601 + +pub trait StreamExt: Stream { + fn pipe

(self, pipe: P) -> StreamPipe + where + P: Pipe, + Self: Sized, + { + assert_stream(StreamPipe { stream: self, pipe }) + } + + fn sink(self, sink: S) -> StreamSink + where + S: Sink, + Self: Sized, + { + assert_future(StreamSink { stream: self, sink }) + } + + fn flat_map(self, f: F) -> FlatMap + where + F: FnMut(Self::Item) -> R, + R: Stream, + Self: Sized, + { + assert_stream(FlatMap::new(self, f)) + } + + fn filter(self, f: F) -> Filter + where + F: FnMut(&Self::Item) -> bool, + Self: Sized, + { + assert_stream(Filter::new(self, f)) + } +} +impl StreamExt for S where S: Stream {} + +pub trait Pipe { + type Item; + + fn poll_next( + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll>; + + fn pipe

(self, pipe: P) -> PipePipe + where + P: Pipe, + Self: Sized, + { + assert_pipe(PipePipe { a: self, b: pipe }) + } + + fn sink(self, sink: S) -> PipeSink + where + S: Sink, + Self: Sized, + { + assert_sink(PipeSink { pipe: self, sink }) + } + + fn flat_map(self, f: F) -> FlatMap + where + F: FnMut(Self::Item) -> R, + R: Stream, + Self: Sized, + { + assert_pipe(FlatMap::new(self, f)) + } + + fn filter(self, f: F) -> Filter + where + F: FnMut(&Self::Item) -> bool, + Self: Sized, + { + assert_pipe(Filter::new(self, f)) + } +} + +pub trait Sink { + fn poll_pipe( + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll<()>; +} + +fn assert_stream(s: S) -> S +where + S: Stream, +{ + s +} +fn assert_pipe(p: P) -> P +where + P: Pipe, +{ + p +} +fn assert_sink(s: S) -> S +where + S: Sink, +{ + s +} +fn assert_future(f: F) -> F +where + F: Future, +{ + f +} + +#[pin_project] +pub struct StreamPipe { + #[pin] + stream: S, + #[pin] + pipe: P, +} + +impl Stream for StreamPipe +where + S: Stream, + P: Pipe, +{ + type Item = P::Item; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let self_ = self.project(); + self_.pipe.poll_next(cx, self_.stream) + } +} + +#[pin_project] +pub struct PipePipe { + #[pin] + a: A, + #[pin] + b: B, +} + +impl Pipe for PipePipe +where + A: Pipe, + B: Pipe, +{ + type Item = B::Item; + + fn poll_next( + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll> { + let self_ = self.project(); + let stream = stream.pipe(self_.a); + pin_mut!(stream); + self_.b.poll_next(cx, stream) + } +} + +#[pin_project] +pub struct PipeSink { + #[pin] + pipe: P, + #[pin] + sink: S, +} + +impl Sink for PipeSink +where + P: Pipe, + S: Sink, +{ + fn poll_pipe( + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll<()> { + let self_ = self.project(); + let stream = stream.pipe(self_.pipe); + pin_mut!(stream); + self_.sink.poll_pipe(cx, stream) + } +} + +#[pin_project] +pub struct StreamSink { + #[pin] + stream: A, + #[pin] + sink: B, +} + +impl Future for StreamSink +where + A: Stream, + B: Sink, +{ + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll<()> { + let self_ = self.project(); + self_.sink.poll_pipe(cx, self_.stream) + } +} + +impl Pipe for Pin

+where + P: DerefMut + Unpin, + P::Target: Pipe, +{ + type Item = >::Item; + + fn poll_next( + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll> { + self.get_mut().as_mut().poll_next(cx, stream) + } +} + +impl Pipe for &mut T +where + T: Pipe + Unpin, +{ + type Item = T::Item; + + fn poll_next( + mut self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll> { + Pin::new(&mut **self).poll_next(cx, stream) + } +} + +impl Sink for Pin

+where + P: DerefMut + Unpin, + P::Target: Sink, +{ + fn poll_pipe( + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll<()> { + self.get_mut().as_mut().poll_pipe(cx, stream) + } +} + +impl Sink for &mut T +where + T: Sink + Unpin, +{ + fn poll_pipe( + mut self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + ) -> Poll<()> { + Pin::new(&mut **self).poll_pipe(cx, stream) + } +} diff --git a/amadeus-core/src/pipe/filter.rs b/amadeus-core/src/pipe/filter.rs new file mode 100644 index 00000000..bba894cc --- /dev/null +++ b/amadeus-core/src/pipe/filter.rs @@ -0,0 +1,56 @@ +use derive_new::new; +use futures::{ready, Stream}; +use pin_project::pin_project; +use std::{ + pin::Pin, task::{Context, Poll} +}; + +use super::Pipe; + +#[pin_project] +#[derive(new)] +pub struct Filter { + #[pin] + task: C, + f: F, +} + +impl Stream for Filter +where + F: FnMut(&C::Item) -> bool, +{ + type Item = C::Item; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let mut self_ = self.project(); + let (mut task, f) = (self_.task, &mut self_.f); + Poll::Ready(loop { + match ready!(task.as_mut().poll_next(cx)) { + Some(t) if f(&t) => break Some(t), + Some(_) => (), + None => break None, + } + }) + } +} + +impl, F, Source> Pipe for Filter +where + F: FnMut(&C::Item) -> bool, +{ + type Item = C::Item; + + fn poll_next( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll> { + let mut self_ = self.project(); + let (mut task, f) = (self_.task, &mut self_.f); + Poll::Ready(loop { + match ready!(task.as_mut().poll_next(cx, stream.as_mut())) { + Some(t) if f(&t) => break Some(t), + Some(_) => (), + None => break None, + } + }) + } +} diff --git a/amadeus-core/src/pipe/flat_map.rs b/amadeus-core/src/pipe/flat_map.rs new file mode 100644 index 00000000..d13dad4d --- /dev/null +++ b/amadeus-core/src/pipe/flat_map.rs @@ -0,0 +1,71 @@ +use derive_new::new; +use futures::{ready, Stream}; +use pin_project::pin_project; +use std::{ + pin::Pin, task::{Context, Poll} +}; + +use super::Pipe; + +#[pin_project] +#[derive(new)] +pub struct FlatMap { + #[pin] + pipe: C, + f: F, + #[pin] + #[new(default)] + next: Option, +} + +impl Stream for FlatMap +where + F: FnMut(C::Item) -> R, + R: Stream, +{ + type Item = R::Item; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let mut self_ = self.project(); + Poll::Ready(loop { + if let Some(s) = self_.next.as_mut().as_pin_mut() { + if let Some(item) = ready!(s.poll_next(cx)) { + break Some(item); + } else { + self_.next.set(None); + } + } else if let Some(s) = ready!(self_.pipe.as_mut().poll_next(cx)) { + self_.next.set(Some((self_.f)(s))); + } else { + break None; + } + }) + } +} + +impl, F, R, Source> Pipe for FlatMap +where + F: FnMut(C::Item) -> R, + R: Stream, +{ + type Item = R::Item; + + fn poll_next( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll> { + let mut self_ = self.project(); + Poll::Ready(loop { + if let Some(s) = self_.next.as_mut().as_pin_mut() { + if let Some(item) = ready!(s.poll_next(cx)) { + break Some(item); + } else { + self_.next.set(None); + } + } else if let Some(s) = ready!(self_.pipe.as_mut().poll_next(cx, stream.as_mut())) { + self_.next.set(Some((self_.f)(s))); + } else { + break None; + } + }) + } +} diff --git a/amadeus-core/src/sink.rs b/amadeus-core/src/sink.rs deleted file mode 100644 index 92f7e68f..00000000 --- a/amadeus-core/src/sink.rs +++ /dev/null @@ -1,219 +0,0 @@ -#![allow(clippy::type_complexity)] - -use derive_new::new; -use futures::{pin_mut, ready, stream, Future, Stream, StreamExt}; -use pin_project::pin_project; -use std::{ - marker::PhantomData, ops::DerefMut, pin::Pin, task::{Context, Poll} -}; - -// Sink could take Item as an input parameter to accept for<'a> &'a T, but this might be fixed anyway? https://github.com/rust-lang/rust/issues/49601 - -pub trait Sink { - type Item; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll<()>; -} - -impl

Sink for Pin

-where - P: DerefMut + Unpin, - P::Target: Sink, -{ - type Item = ::Item; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - self.get_mut().as_mut().poll_forward(cx, stream) - } -} - -impl Sink for &mut T -where - T: Sink + Unpin, -{ - type Item = T::Item; - - fn poll_forward( - mut self: Pin<&mut Self>, cx: &mut Context, - stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - Pin::new(&mut **self).poll_forward(cx, stream) - } -} - -#[pin_project] -#[derive(new)] -pub struct SinkMap { - #[pin] - i: I, - f: F, - marker: PhantomData (R, Item)>, -} -impl Sink for SinkMap -where - F: FnMut(Item) -> R, - I: Sink, -{ - type Item = Item; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - let self_ = self.project(); - let (f, i) = (self_.f, self_.i); - let stream = stream.map(f); - pin_mut!(stream); - i.poll_forward(cx, stream) - } -} - -#[pin_project(project = SinkFilterStateProj, project_replace = SinkFilterStateProjOwn)] -pub enum SinkFilterState { - Some(T, #[pin] Fut), - None, -} -impl SinkFilterState { - fn fut(self: Pin<&mut Self>) -> Option> { - match self.project() { - SinkFilterStateProj::Some(_, fut) => Some(fut), - SinkFilterStateProj::None => None, - } - } - fn take(self: Pin<&mut Self>) -> Option { - match self.project_replace(SinkFilterState::None) { - SinkFilterStateProjOwn::Some(t, _) => Some(t), - SinkFilterStateProjOwn::None => None, - } - } -} -#[pin_project] -#[derive(new)] -pub struct SinkFilter<'a, F, I, T, Fut> { - fut: Pin<&'a mut SinkFilterState>, - #[pin] - i: I, - f: F, -} -impl<'a, F, I, Fut, Item> Sink for SinkFilter<'a, F, I, Item, Fut> -where - F: FnMut(&Item) -> Fut, - Fut: Future, - I: Sink, -{ - type Item = Item; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - let self_ = self.project(); - let f = self_.f; - let fut = self_.fut; - let stream = stream::poll_fn(move |cx| loop { - if fut.as_mut().fut().is_none() { - let item = ready!(stream.as_mut().poll_next(cx)); - if item.is_none() { - break Poll::Ready(None); - } - let item = item.unwrap(); - let fut_ = f(&item); - fut.set(SinkFilterState::Some(item, fut_)); - } - let filter = ready!(fut.as_mut().fut().unwrap().poll(cx)); - let item = fut.as_mut().take().unwrap(); - if filter { - break Poll::Ready(Some(item)); - } - }); - pin_mut!(stream); - (self_.i).poll_forward(cx, stream) - } -} - -#[pin_project] -#[derive(new)] -pub struct SinkThen<'a, F, I, Fut, R, Item> { - fut: Pin<&'a mut Option>, - #[pin] - i: I, - f: F, - marker: PhantomData, -} -impl<'a, F, I, R, Fut, Item> Sink for SinkThen<'a, F, I, Fut, R, Item> -where - F: FnMut(Item) -> Fut, - Fut: Future, - I: Sink, -{ - type Item = Item; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - let self_ = self.project(); - let f = self_.f; - let fut = self_.fut; - let stream = stream::poll_fn(|cx| { - if fut.is_none() { - let item = ready!(stream.as_mut().poll_next(cx)); - if item.is_none() { - return Poll::Ready(None); - } - fut.set(Some(f(item.unwrap()))); - } - let item = ready!(fut.as_mut().as_pin_mut().unwrap().poll(cx)); - fut.set(None); - Poll::Ready(Some(item)) - }); - pin_mut!(stream); - (self_.i).poll_forward(cx, stream) - } -} - -#[pin_project] -#[derive(new)] -pub struct SinkFlatMap<'a, F, I, Fut, R, Item> { - fut: Pin<&'a mut Option>, - #[pin] - i: I, - f: F, - marker: PhantomData (Fut, R, Item)>, -} -impl<'a, F, I, R, Fut, Item> Sink for SinkFlatMap<'a, F, I, Fut, R, Item> -where - F: FnMut(Item) -> Fut, - Fut: Stream, - I: Sink, -{ - type Item = Item; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - let self_ = self.project(); - let f = self_.f; - let fut = self_.fut; - let stream = stream::poll_fn(|cx| loop { - if fut.is_none() { - let item = ready!(stream.as_mut().poll_next(cx)); - if item.is_none() { - return Poll::Ready(None); - } - fut.set(Some(f(item.unwrap()))); - } - let item = ready!(fut.as_mut().as_pin_mut().unwrap().poll_next(cx)); - if let Some(item) = item { - break Poll::Ready(Some(item)); - } - fut.set(None); - }); - pin_mut!(stream); - (self_.i).poll_forward(cx, stream) - } -} diff --git a/amadeus-core/src/util.rs b/amadeus-core/src/util.rs index debb8b28..fa3cb7d5 100644 --- a/amadeus-core/src/util.rs +++ b/amadeus-core/src/util.rs @@ -3,12 +3,10 @@ use futures::{ready, Stream}; use pin_project::pin_project; use serde::{de::Deserializer, ser::Serializer, Deserialize, Serialize}; use std::{ - any::type_name, error, fmt, hash::{Hash, Hasher}, io, marker::PhantomData, pin::Pin, sync::Arc, task::{Context, Poll} + error, fmt, hash::{Hash, Hasher}, io, marker::PhantomData, pin::Pin, sync::Arc, task::{Context, Poll} }; -use crate::{ - par_stream::{DistributedStream, ParallelStream, StreamTask, StreamTaskAsync}, sink::Sink -}; +use crate::par_stream::{DistributedStream, ParallelStream, StreamTask}; pub struct ResultExpand(pub Result); impl IntoIterator for ResultExpand @@ -155,12 +153,10 @@ where self } } -impl StreamTaskAsync for ImplTask { +impl Stream for ImplTask { type Item = T; - fn poll_run( - self: Pin<&mut Self>, _cx: &mut Context, _sink: Pin<&mut impl Sink>, - ) -> Poll<()> { + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { unreachable!() } } @@ -222,51 +218,18 @@ where } } -pub fn type_coerce(a: A) -> B { - try_type_coerce(a) - .unwrap_or_else(|| panic!("can't coerce {} to {}", type_name::(), type_name::())) -} -pub fn try_type_coerce(a: A) -> Option { - trait Eq { - fn eq(self) -> Option; - } - - struct Foo(A, PhantomData B>); - - impl Eq for Foo { - default fn eq(self) -> Option { - None - } - } - impl Eq for Foo { - fn eq(self) -> Option { - Some(self.0) - } - } - - Foo::(a, PhantomData).eq() -} - -#[repr(transparent)] -pub struct Debug(pub T); -trait DebugDuck { - fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error>; -} -impl DebugDuck for T { - default fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { - write!(f, "{}", std::any::type_name::()) - } -} -impl DebugDuck for T -where - T: std::fmt::Debug, -{ - fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { - ::fmt(self, f) - } -} -impl std::fmt::Debug for Debug { - fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { - ::fmt(&self.0, f) - } +/// As unsafe as regular `core::mem::transmute`, but asserts size at runtime. +/// +/// # Safety +/// +/// Not. +pub unsafe fn transmute(a: A) -> B { + use std::mem; + assert_eq!( + (mem::size_of::(), mem::align_of::()), + (mem::size_of::(), mem::align_of::()) + ); + let ret = mem::transmute_copy(&a); + mem::forget(a); + ret } diff --git a/amadeus-parquet/src/internal/record/impls.rs b/amadeus-parquet/src/internal/record/impls.rs index 03801892..c2d74b27 100644 --- a/amadeus-parquet/src/internal/record/impls.rs +++ b/amadeus-parquet/src/internal/record/impls.rs @@ -1,10 +1,9 @@ use linked_hash_map::LinkedHashMap; use std::{ - collections::HashMap, convert::{TryFrom, TryInto}, fmt, hash::{BuildHasher, Hash}, marker::PhantomData, string::FromUtf8Error, sync::Arc + any::type_name, collections::HashMap, convert::{TryFrom, TryInto}, fmt, hash::{BuildHasher, Hash}, marker::PhantomData, string::FromUtf8Error, sync::Arc }; use sum::{Sum2, Sum3}; -use amadeus_core::util::type_coerce; use amadeus_types::{ Bson, Data, Date, DateTime, DateTimeWithoutTimezone, DateWithoutTimezone, Decimal, Enum, Group, IpAddr, Json, List, Time, TimeWithoutTimezone, Timezone, Url, Value, Webpage }; @@ -2096,3 +2095,30 @@ impl From for ParquetError { ParquetError::General(err.to_string()) } } + +//////////////////////////////////////////////////////////////////////////////// + +fn type_coerce(a: A) -> B { + try_type_coerce(a) + .unwrap_or_else(|| panic!("can't coerce {} to {}", type_name::(), type_name::())) +} +fn try_type_coerce(a: A) -> Option { + trait Eq { + fn eq(self) -> Option; + } + + struct Foo(A, PhantomData B>); + + impl Eq for Foo { + default fn eq(self) -> Option { + None + } + } + impl Eq for Foo { + fn eq(self) -> Option { + Some(self.0) + } + } + + Foo::(a, PhantomData).eq() +} diff --git a/amadeus-types/src/list.rs b/amadeus-types/src/list.rs index e9c73671..3a9dd8e9 100644 --- a/amadeus-types/src/list.rs +++ b/amadeus-types/src/list.rs @@ -7,7 +7,7 @@ use std::{ use super::{AmadeusOrd, Data}; use amadeus_core::{ - par_sink::{ExtendReducer, FromDistributedStream, FromParallelStream, PushReducer}, pool::ProcessSend, util::type_coerce + par_sink::{ExtendReducer, FromDistributedStream, FromParallelStream, PushReducer}, pool::ProcessSend }; pub struct List { @@ -219,13 +219,13 @@ impl Deref for List { #[inline(always)] fn deref(&self) -> &Self::Target { - &*type_coerce::<_, &Vec>(&self.vec) + &*self.vec } } impl DerefMut for List { #[inline(always)] fn deref_mut(&mut self) -> &mut Self::Target { - &mut *type_coerce::<_, &mut Vec>(&mut self.vec) + &mut *self.vec } } diff --git a/examples/cloudfront_logs.rs b/examples/cloudfront_logs.rs index 6b889823..83de457d 100644 --- a/examples/cloudfront_logs.rs +++ b/examples/cloudfront_logs.rs @@ -11,7 +11,6 @@ //! cargo run --example cloudfront_logs --release //! ``` -#![type_length_limit = "13470730"] use amadeus::prelude::*; #[allow(unreachable_code)] diff --git a/src/lib.rs b/src/lib.rs index 170d65d1..442db61d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,7 +20,6 @@ clippy::pedantic, )] #![allow( - incomplete_features, clippy::module_name_repetitions, clippy::similar_names, clippy::if_not_else, diff --git a/src/pool/thread.rs b/src/pool/thread.rs index cac13a72..412b249d 100644 --- a/src/pool/thread.rs +++ b/src/pool/thread.rs @@ -1,7 +1,7 @@ -use async_std::sync::{channel, Sender}; +use async_std::sync::{channel, RecvError, Sender}; use futures::{FutureExt, TryFutureExt}; use std::{ - any::Any, future::Future, io, panic::{RefUnwindSafe, UnwindSafe}, pin::Pin, sync::Arc + any::Any, future::Future, io, panic::{AssertUnwindSafe, RefUnwindSafe, UnwindSafe}, pin::Pin, sync::Arc }; use tokio::{ runtime::Handle, task::{JoinError, LocalSet} @@ -71,7 +71,7 @@ struct Pool { } type Request = Box Box> + Send>; -type Response = Box; +type Response = Result, Box>; impl Pool { fn new(threads: usize) -> Self { @@ -103,18 +103,20 @@ impl Pool { { let sender = self.sender.clone(); async move { + let task: Request = Box::new(|| { + Box::new( + AssertUnwindSafe(task().map(|t| Box::new(t) as Box)) + .catch_unwind(), + ) + }); let (sender_, receiver) = channel::(1); - sender - .send(( - Box::new(|| Box::new(task().map(|t| Box::new(t) as Box))), - sender_, - )) - .await; - receiver - .recv() - .await - .map(|x| *Box::::downcast(x).unwrap()) - .map_err(|_e| todo!()) + sender.send((task, sender_)).await; + let res = receiver.recv().await; + #[allow(deprecated)] + res.map_err(|RecvError| unreachable!()).and_then(|x| { + x.map(|x| *Box::::downcast(x).unwrap()) + .map_err(JoinError::panic) + }) } } } diff --git a/src/source.rs b/src/source.rs index 5bf11ccd..c42e50ec 100644 --- a/src/source.rs +++ b/src/source.rs @@ -2,14 +2,14 @@ use ::serde::{Deserialize, Serialize}; use derive_new::new; -use futures::pin_mut; +use futures::Stream; use pin_project::pin_project; use std::{ error::Error, fmt::Debug, marker::PhantomData, pin::Pin, task::{Context, Poll} }; use crate::{ - par_sink::{DistributedSink, ParallelSink}, par_stream::{DistributedStream, ParallelStream, StreamTask, StreamTaskAsync} + par_sink::{DistributedSink, ParallelSink}, par_stream::{DistributedStream, ParallelStream, StreamTask} }; #[cfg(feature = "aws")] @@ -226,21 +226,18 @@ where } } } -impl StreamTaskAsync for IntoTask +impl Stream for IntoTask where - I: StreamTaskAsync>, + I: Stream>, T: Into, { type Item = Result; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, - sink: Pin<&mut impl amadeus_core::sink::Sink>, - ) -> Poll<()> { - let sink = - amadeus_core::sink::SinkMap::new(sink, |item: Result<_, _>| item.map(Into::into)); - pin_mut!(sink); - self.project().task.poll_run(cx, sink) + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + self.project() + .task + .poll_next(cx) + .map(|t| t.map(|t| t.map(Into::into))) } } impl Iterator for IntoStream diff --git a/tests/cloudfront.rs b/tests/cloudfront.rs index 0392a996..804d39bd 100644 --- a/tests/cloudfront.rs +++ b/tests/cloudfront.rs @@ -1,4 +1,3 @@ -#![type_length_limit = "13470730"] #![allow(clippy::suspicious_map)] use amadeus::prelude::*; diff --git a/tests/cloudfront_dist.rs b/tests/cloudfront_dist.rs index 81eeed3d..cfa9fc5b 100644 --- a/tests/cloudfront_dist.rs +++ b/tests/cloudfront_dist.rs @@ -1,5 +1,3 @@ -#![type_length_limit = "13470730"] - use amadeus::dist::prelude::*; #[cfg(feature = "constellation")] use constellation::*; diff --git a/tests/csv.rs b/tests/csv.rs index 36681e2c..987d98dd 100644 --- a/tests/csv.rs +++ b/tests/csv.rs @@ -1,4 +1,3 @@ -#![type_length_limit = "1353029"] #![allow(clippy::suspicious_map)] use std::{path::PathBuf, time::SystemTime}; diff --git a/tests/csv_dist.rs b/tests/csv_dist.rs index f2cd49a9..73a08769 100644 --- a/tests/csv_dist.rs +++ b/tests/csv_dist.rs @@ -1,4 +1,3 @@ -#![type_length_limit = "1726739"] #[cfg(feature = "constellation")] use constellation::*; use std::{ diff --git a/tests/json.rs b/tests/json.rs index 24d16f9a..93a26716 100644 --- a/tests/json.rs +++ b/tests/json.rs @@ -1,4 +1,3 @@ -#![type_length_limit = "1393589"] #![allow(clippy::suspicious_map)] use std::{path::PathBuf, time::SystemTime}; diff --git a/tests/json_dist.rs b/tests/json_dist.rs index fa4c2c14..42b4ff91 100644 --- a/tests/json_dist.rs +++ b/tests/json_dist.rs @@ -1,4 +1,3 @@ -#![type_length_limit = "1774739"] #[cfg(feature = "constellation")] use constellation::*; use std::{ diff --git a/tests/parquet.rs b/tests/parquet.rs index 50b39dc0..491a56df 100644 --- a/tests/parquet.rs +++ b/tests/parquet.rs @@ -1,4 +1,3 @@ -#![type_length_limit = "13225240"] #![allow( clippy::cognitive_complexity, clippy::type_complexity, diff --git a/tests/parquet_dist.rs b/tests/parquet_dist.rs index 0f0d66cb..9e2e3ff3 100644 --- a/tests/parquet_dist.rs +++ b/tests/parquet_dist.rs @@ -1,4 +1,3 @@ -#![type_length_limit = "13225240"] #![allow(clippy::cognitive_complexity, clippy::type_complexity)] #[cfg(feature = "constellation")] From bdd5949ecfc5f8a075e540b8e08536244ca10ad7 Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Sun, 5 Jul 2020 18:20:12 +0100 Subject: [PATCH 5/8] bump CI nightly --- azure-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8cde4b71..60d90644 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,7 +20,7 @@ jobs: endpoint: alecmocatta default: rust_toolchain: nightly - rust_lint_toolchain: nightly-2020-06-30 + rust_lint_toolchain: nightly-2020-07-05 rust_flags: '' rust_features_clippy: ';aws;commoncrawl;parquet;postgres;csv;json;constellation aws commoncrawl parquet postgres csv json' rust_features: 'constellation aws commoncrawl parquet postgres csv json' @@ -35,16 +35,16 @@ jobs: linux0: imageName: 'ubuntu-latest' rust_target_run: 'x86_64-unknown-linux-gnu' - # linux1: - # imageName: 'ubuntu-latest' - # rust_target_run: 'x86_64-unknown-linux-musl' # TODO: https://github.com/rust-lang/rust/issues/73451 + linux1: + imageName: 'ubuntu-latest' + rust_target_run: 'x86_64-unknown-linux-musl' - template: rust-n.yml@templates parameters: endpoint: alecmocatta default: rust_toolchain: nightly - rust_lint_toolchain: nightly-2020-06-30 + rust_lint_toolchain: nightly-2020-07-05 rust_flags: '' rust_features_clippy: ';aws;commoncrawl;parquet;postgres;csv;json;aws commoncrawl parquet postgres csv json' rust_features: 'aws commoncrawl parquet postgres csv json' From d8119f204def9e678aa74f018e9d86a7cfe7a041 Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Sun, 5 Jul 2020 22:09:56 +0100 Subject: [PATCH 6/8] remove some dependences on unstable features --- Cargo.toml | 2 +- amadeus-aws/Cargo.toml | 1 + amadeus-aws/src/cloudfront.rs | 5 +- amadeus-derive/src/lib.rs | 17 ---- amadeus-parquet/src/internal/file/reader.rs | 2 +- amadeus-parquet/src/internal/mod.rs | 42 --------- amadeus-parquet/src/internal/record/reader.rs | 2 +- amadeus-parquet/src/lib.rs | 16 +++- amadeus-serde/Cargo.toml | 1 + amadeus-serde/src/impls.rs | 5 +- amadeus-serde/src/lib.rs | 1 - amadeus-types/Cargo.toml | 1 + amadeus-types/src/group.rs | 8 +- amadeus-types/src/lib.rs | 3 +- amadeus-types/src/list.rs | 26 ++++- amadeus-types/src/util.rs | 94 +++++++++++++++++++ amadeus-types/src/value.rs | 64 +++---------- azure-pipelines.yml | 6 +- tests/parquet.rs | 1 + tests/parquet_dist.rs | 1 + 20 files changed, 164 insertions(+), 134 deletions(-) create mode 100644 amadeus-types/src/util.rs diff --git a/Cargo.toml b/Cargo.toml index 68c42842..9c7fe567 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ pin-project = "0.4" serde = { version = "1.0", features = ["derive"] } serde_closure = "0.2" serde_traitobject = { version = "0.2", optional = true } -tokio = { version = "0.2", features = ["rt-threaded", "rt-util"] } +tokio = { version = "0.2", features = ["rt-threaded", "rt-util", "blocking"] } [dev-dependencies] either = { version = "1.5", features = ["serde"] } diff --git a/amadeus-aws/Cargo.toml b/amadeus-aws/Cargo.toml index 5533cdc7..e3f62d05 100644 --- a/amadeus-aws/Cargo.toml +++ b/amadeus-aws/Cargo.toml @@ -35,6 +35,7 @@ serde_closure = "0.2" serde = { version = "1.0", features = ["derive"] } tokio = "0.2" url = { version = "2.1", features = ["serde"] } +vec-utils = "0.2" # dependency of rusoto_core/hyper-tls/native-tls; ensure it's vendored to simplify cross-compilation [target.'cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios")))'.dependencies] diff --git a/amadeus-aws/src/cloudfront.rs b/amadeus-aws/src/cloudfront.rs index 32ffbe11..75a635d8 100644 --- a/amadeus-aws/src/cloudfront.rs +++ b/amadeus-aws/src/cloudfront.rs @@ -8,6 +8,7 @@ use serde_closure::*; use std::{ convert::identity, io::{self}, time::Duration }; +use vec_utils::VecExt; use amadeus_core::{ into_par_stream::IntoDistributedStream, par_stream::DistributedStream, util::{DistParStream, ResultExpandIter}, Source @@ -39,9 +40,7 @@ impl Cloudfront { let objects = list(&client, &bucket, &prefix) .await? - .into_iter() - .map(|object: Object| object.key.unwrap()) - .collect(); + .map(|object: Object| object.key.unwrap()); Ok(Self { region, diff --git a/amadeus-derive/src/lib.rs b/amadeus-derive/src/lib.rs index 8580b383..4e0dcae3 100644 --- a/amadeus-derive/src/lib.rs +++ b/amadeus-derive/src/lib.rs @@ -1,20 +1,3 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - #![doc(html_root_url = "https://docs.rs/amadeus-derive/0.2.5")] #![recursion_limit = "400"] #![allow(clippy::useless_let_if_seq)] diff --git a/amadeus-parquet/src/internal/file/reader.rs b/amadeus-parquet/src/internal/file/reader.rs index 31238cf8..b136a30f 100644 --- a/amadeus-parquet/src/internal/file/reader.rs +++ b/amadeus-parquet/src/internal/file/reader.rs @@ -1223,7 +1223,7 @@ mod tests { let mut def_levels = Some(vec![0; batch_size]); let mut rep_levels = None::>; - for col_reader in readers.into_iter() { + for col_reader in readers { match col_reader { r @ ColumnReader::Int64ColumnReader(..) => { let mut data_collected = Vec::with_capacity(num_rows); diff --git a/amadeus-parquet/src/internal/mod.rs b/amadeus-parquet/src/internal/mod.rs index bc7f35f9..237d23e7 100644 --- a/amadeus-parquet/src/internal/mod.rs +++ b/amadeus-parquet/src/internal/mod.rs @@ -1,45 +1,3 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! An Apache Parquet implementation in Rust. -//! -//! **[Crates.io](https://crates.io/crates/amadeus-parquet) │ [Repo](https://github.com/constellation-rs/amadeus)** - -// #![doc(html_root_url = "https://docs.rs/amadeus-parquet/0.1.0")] -// #![feature( -// specialization, -// type_alias_impl_trait, -// bufreader_seek_relative, -// read_initializer -// )] -#![warn( -// missing_copy_implementations, -// missing_debug_implementations, -// missing_docs, -// trivial_casts, -// trivial_numeric_casts, - unused_import_braces, - unused_qualifications, - unused_results, -// clippy::pedantic -)] -// from https://github.com/rust-unofficial/patterns/blob/master/anti_patterns/deny-warnings.md -#![allow(dead_code, clippy::all)] - #[macro_use] pub mod errors; pub mod basic; diff --git a/amadeus-parquet/src/internal/record/reader.rs b/amadeus-parquet/src/internal/record/reader.rs index 9615c915..38cbf35d 100644 --- a/amadeus-parquet/src/internal/record/reader.rs +++ b/amadeus-parquet/src/internal/record/reader.rs @@ -439,7 +439,7 @@ impl Reader for ValueReader { } ValueReader::ByteArray(ref mut reader) => reader .read(def_level, rep_level) - .map(|vec| Value::List(vec.into_iter().map(Value::from).collect())), + .map(|vec| Value::List(vec.map(Value::from))), ValueReader::Bson(ref mut reader) => reader.read(def_level, rep_level).map(Value::Bson), ValueReader::String(ref mut reader) => { reader.read(def_level, rep_level).map(Value::String) diff --git a/amadeus-parquet/src/lib.rs b/amadeus-parquet/src/lib.rs index cc34331b..8686e5e2 100644 --- a/amadeus-parquet/src/lib.rs +++ b/amadeus-parquet/src/lib.rs @@ -4,7 +4,21 @@ #![feature(specialization)] #![feature(type_alias_impl_trait)] #![cfg_attr(test, feature(test))] -#![allow(incomplete_features)] +#![warn( + // missing_copy_implementations, + // missing_debug_implementations, + // missing_docs, + // trivial_casts, + // trivial_numeric_casts, + unused_import_braces, + unused_qualifications, + unused_results, + // unreachable_pub, + // clippy::pedantic +)] +// from https://github.com/rust-unofficial/patterns/blob/master/anti_patterns/deny-warnings.md +#![allow(dead_code, incomplete_features, clippy::all)] + #[cfg(test)] extern crate test; diff --git a/amadeus-serde/Cargo.toml b/amadeus-serde/Cargo.toml index b8df63e4..eb248b78 100644 --- a/amadeus-serde/Cargo.toml +++ b/amadeus-serde/Cargo.toml @@ -31,3 +31,4 @@ serde_bytes = "0.11" serde_closure = "0.2" serde_json = "1.0" sum = { version = "0.1", features = ["serde"] } +vec-utils = "0.2" diff --git a/amadeus-serde/src/impls.rs b/amadeus-serde/src/impls.rs index e9efb150..d6334b3f 100644 --- a/amadeus-serde/src/impls.rs +++ b/amadeus-serde/src/impls.rs @@ -5,6 +5,7 @@ use serde::{ use std::{ collections::HashMap, fmt, hash::{BuildHasher, Hash}, str, sync::Arc }; +use vec_utils::VecExt; use amadeus_types::{ Bson, Data, Date, DateTime, DateTimeWithoutTimezone, DateWithoutTimezone, Decimal, Enum, Group, IpAddr, Json, List, SchemaIncomplete, Time, TimeWithoutTimezone, Timezone, Url, Value, ValueRequired, Webpage @@ -171,7 +172,7 @@ where S: Serializer, { let mut serializer = serializer.serialize_seq(Some(self.len()))?; - for item in self.clone().into_iter() { + for item in self.clone() { serializer.serialize_element(&SerdeSerialize(&item))?; } serializer.end() @@ -488,7 +489,7 @@ macro_rules! array { D: Deserializer<'de>, { let self_: [SerdeDeserialize; $i] = serde::Deserialize::deserialize(deserializer)?; - let self_: Box = std::array::IntoIter::new(self_).map(|a|a.0).collect::>().into_boxed_slice().try_into().unwrap(); + let self_: Box = >>::from(self_).map(|a|a.0).into_boxed_slice().try_into().unwrap(); Ok(*self_) } } diff --git a/amadeus-serde/src/lib.rs b/amadeus-serde/src/lib.rs index 2c2bf3ba..ad9ad23b 100644 --- a/amadeus-serde/src/lib.rs +++ b/amadeus-serde/src/lib.rs @@ -1,5 +1,4 @@ #![doc(html_root_url = "https://docs.rs/amadeus-serde/0.2.5")] -#![feature(array_value_iter)] #![feature(specialization)] #![feature(type_alias_impl_trait)] #![allow(incomplete_features)] diff --git a/amadeus-types/Cargo.toml b/amadeus-types/Cargo.toml index bf5332c6..00760e9b 100644 --- a/amadeus-types/Cargo.toml +++ b/amadeus-types/Cargo.toml @@ -28,3 +28,4 @@ once_cell = "1.0" ordered-float = "1.0" serde = { version = "1.0", features = ["derive"] } url = { version = "2.1", features = ["serde"] } +vec-utils = "0.2" diff --git a/amadeus-types/src/group.rs b/amadeus-types/src/group.rs index 25484a6c..158c9dca 100644 --- a/amadeus-types/src/group.rs +++ b/amadeus-types/src/group.rs @@ -7,7 +7,7 @@ use std::{ cmp::Ordering, fmt::{self, Debug}, ops::Index, slice::SliceIndex, str, sync::Arc }; -use super::{AmadeusOrd, Downcast, DowncastError, DowncastFrom, Value}; +use super::{util::IteratorExt, AmadeusOrd, Downcast, DowncastError, DowncastFrom, Value}; /// Corresponds to Parquet groups of named fields. /// @@ -74,7 +74,7 @@ impl<'de> Deserialize<'de> for Group { // impl From for internal::record::types::Group { // fn from(group: Group) -> Self { // let field_names = group.field_names(); -// Self::new(group.into_fields().into_iter().map(Into::into).collect(), field_names) +// Self::new(group.into_fields().map(Into::into), field_names) // } // } impl Index for Group @@ -106,14 +106,14 @@ impl AmadeusOrd for Group { .iter() .map(|(name, _index)| name) .zip(&self.fields) - .cmp_by( + .cmp_by_( b.iter().map(|(name, _index)| name).zip(&other.fields), |a, b| a.amadeus_cmp(&b), ), (None, None) => self .fields .iter() - .cmp_by(&other.fields, AmadeusOrd::amadeus_cmp), + .cmp_by_(&other.fields, AmadeusOrd::amadeus_cmp), (Some(_), None) => Ordering::Less, (None, Some(_)) => Ordering::Greater, } diff --git a/amadeus-types/src/lib.rs b/amadeus-types/src/lib.rs index 47a0c27a..c78b9362 100644 --- a/amadeus-types/src/lib.rs +++ b/amadeus-types/src/lib.rs @@ -1,6 +1,4 @@ #![doc(html_root_url = "https://docs.rs/amadeus-types/0.2.5")] -#![feature(iter_order_by)] -#![feature(min_specialization)] //! Implementations of Rust types that correspond to Parquet logical types. //! [`Record`](super::Record) is implemented for each of them. @@ -58,6 +56,7 @@ mod http; mod list; mod ord; mod time; +mod util; mod value; mod value_required; diff --git a/amadeus-types/src/list.rs b/amadeus-types/src/list.rs index 3a9dd8e9..66e71730 100644 --- a/amadeus-types/src/list.rs +++ b/amadeus-types/src/list.rs @@ -5,7 +5,7 @@ use std::{ cmp::Ordering, fmt::{self, Debug}, hash::{Hash, Hasher}, iter::FromIterator, mem::ManuallyDrop, ops::{Deref, DerefMut}, panic::{RefUnwindSafe, UnwindSafe} }; -use super::{AmadeusOrd, Data}; +use super::{util::IteratorExt, AmadeusOrd, Data}; use amadeus_core::{ par_sink::{ExtendReducer, FromDistributedStream, FromParallelStream, PushReducer}, pool::ProcessSend }; @@ -49,6 +49,22 @@ impl List { // pub fn iter(&self) -> <&Self as IntoIterator>::IntoIter { // self.into_iter() // } + #[inline(always)] + pub fn map(self, f: F) -> List + where + F: FnMut(T) -> U, + { + // TODO + self.into_iter().map(f).collect() + } + #[inline(always)] + pub fn try_map(self, f: F) -> Result, E> + where + F: FnMut(T) -> Result, + { + // TODO + self.into_iter().map(f).collect() + } } impl Default for List { #[inline(always)] @@ -126,7 +142,7 @@ where fn eq(&self, other: &List) -> bool { self.clone() .into_iter() - .eq_by(other.clone().into_iter(), |a, b| a.eq(&b)) + .eq_by_(other.clone().into_iter(), |a, b| a.eq(&b)) } } impl Eq for List where T: Eq {} @@ -138,7 +154,7 @@ where fn partial_cmp(&self, other: &List) -> Option { self.clone() .into_iter() - .partial_cmp_by(other.clone().into_iter(), |a, b| a.partial_cmp(&b)) + .partial_cmp_by_(other.clone().into_iter(), |a, b| a.partial_cmp(&b)) } } impl Ord for List @@ -149,7 +165,7 @@ where fn cmp(&self, other: &Self) -> Ordering { self.clone() .into_iter() - .cmp_by(other.clone().into_iter(), |a, b| a.cmp(&b)) + .cmp_by_(other.clone().into_iter(), |a, b| a.cmp(&b)) } } impl AmadeusOrd for List @@ -160,7 +176,7 @@ where fn amadeus_cmp(&self, other: &Self) -> Ordering { self.clone() .into_iter() - .cmp_by(other.clone().into_iter(), |a, b| a.amadeus_cmp(&b)) + .cmp_by_(other.clone().into_iter(), |a, b| a.amadeus_cmp(&b)) } } impl Hash for List diff --git a/amadeus-types/src/util.rs b/amadeus-types/src/util.rs new file mode 100644 index 00000000..3ae2d59c --- /dev/null +++ b/amadeus-types/src/util.rs @@ -0,0 +1,94 @@ +// from https://github.com/rust-lang/rust/blob/0cd7ff7ddfb75a38dca81ad3e76b1e984129e939/src/libcore/iter/traits/iterator.rs + +use std::cmp::Ordering; + +pub(crate) trait IteratorExt: Iterator { + fn cmp_by_(mut self, other: I, mut cmp: F) -> Ordering + where + Self: Sized, + I: IntoIterator, + F: FnMut(Self::Item, I::Item) -> Ordering, + { + let mut other = other.into_iter(); + + loop { + let x = match self.next() { + None => { + if other.next().is_none() { + return Ordering::Equal; + } else { + return Ordering::Less; + } + } + Some(val) => val, + }; + + let y = match other.next() { + None => return Ordering::Greater, + Some(val) => val, + }; + + match cmp(x, y) { + Ordering::Equal => (), + non_eq => return non_eq, + } + } + } + fn partial_cmp_by_(mut self, other: I, mut partial_cmp: F) -> Option + where + Self: Sized, + I: IntoIterator, + F: FnMut(Self::Item, I::Item) -> Option, + { + let mut other = other.into_iter(); + + loop { + let x = match self.next() { + None => { + if other.next().is_none() { + return Some(Ordering::Equal); + } else { + return Some(Ordering::Less); + } + } + Some(val) => val, + }; + + let y = match other.next() { + None => return Some(Ordering::Greater), + Some(val) => val, + }; + + match partial_cmp(x, y) { + Some(Ordering::Equal) => (), + non_eq => return non_eq, + } + } + } + fn eq_by_(mut self, other: I, mut eq: F) -> bool + where + Self: Sized, + I: IntoIterator, + F: FnMut(Self::Item, I::Item) -> bool, + { + let mut other = other.into_iter(); + + loop { + let x = match self.next() { + None => return other.next().is_none(), + Some(val) => val, + }; + + let y = match other.next() { + None => return false, + Some(val) => val, + }; + + if !eq(x, y) { + return false; + } + } + } +} + +impl IteratorExt for I where I: Iterator {} diff --git a/amadeus-types/src/value.rs b/amadeus-types/src/value.rs index 3ed694f2..1de6d01b 100644 --- a/amadeus-types/src/value.rs +++ b/amadeus-types/src/value.rs @@ -8,6 +8,7 @@ use serde::{de::Deserializer, ser::Serializer, Deserialize, Serialize}; use std::{ cmp::Ordering, collections::HashMap, convert::TryInto, fmt, fmt::Debug, hash::{BuildHasher, Hash, Hasher}, iter::FromIterator, sync::Arc }; +use vec_utils::VecExt; use crate::list::ListVec; @@ -1569,14 +1570,8 @@ impl From> for Value where T: Into, { - default fn from(value: List) -> Self { - Self::List(value.into_iter().map(Into::into).collect::>()) - } -} -#[doc(hidden)] -impl From> for Value { - fn from(value: List) -> Self { - Self::List(value) + fn from(value: List) -> Self { + Self::List(value.map(Into::into)) } } impl From> for Value @@ -1585,7 +1580,7 @@ where V: Into, S: BuildHasher, { - default fn from(value: HashMap) -> Self { + fn from(value: HashMap) -> Self { Self::Map( value .into_iter() @@ -1594,12 +1589,6 @@ where ) } } -#[doc(hidden)] -impl From> for Value { - fn from(value: HashMap) -> Self { - Self::Map(value) - } -} impl From for Value { fn from(value: Group) -> Self { Self::Group(value) @@ -1609,7 +1598,7 @@ impl From> for Value where T: Into, { - default fn from(value: Option) -> Self { + fn from(value: Option) -> Self { Self::Option( value .map(Into::into) @@ -1617,12 +1606,6 @@ where ) } } -#[doc(hidden)] -impl From> for Value { - fn from(value: Option) -> Self { - Self::Option(value.map(|x| >::from(x).unwrap())) - } -} impl From> for Value where T: Into, @@ -1800,18 +1783,10 @@ impl DowncastFrom for List where T: DowncastFrom, { - default fn downcast_from(self_: Value) -> Result { - self_.into_list().and_then(|list| { - list.into_iter() - .map(Downcast::downcast) - .collect::, _>>() - }) - } -} -#[doc(hidden)] -impl DowncastFrom for List { fn downcast_from(self_: Value) -> Result { - self_.into_list() + self_ + .into_list() + .and_then(|list| list.try_map(Downcast::downcast)) } } impl DowncastFrom for HashMap @@ -1820,7 +1795,7 @@ where V: DowncastFrom, S: BuildHasher + Default, { - default fn downcast_from(self_: Value) -> Result { + fn downcast_from(self_: Value) -> Result { self_.into_map().and_then(|map| { map.into_iter() .map(|(k, v)| Ok((k.downcast()?, v.downcast()?))) @@ -1828,13 +1803,6 @@ where }) } } -#[doc(hidden)] -#[allow(clippy::implicit_hasher)] -impl DowncastFrom for HashMap { - fn downcast_from(self_: Value) -> Result { - self_.into_map() - } -} impl DowncastFrom for Group { fn downcast_from(self_: Value) -> Result { self_.into_group() @@ -1844,19 +1812,13 @@ impl DowncastFrom for Option where T: DowncastFrom, { - default fn downcast_from(self_: Value) -> Result { + fn downcast_from(self_: Value) -> Result { match self_.into_option()? { Some(t) => t.downcast().map(Some), None => Ok(None), } } } -#[doc(hidden)] -impl DowncastFrom for Option { - fn downcast_from(self_: Value) -> Result { - self_.into_option() - } -} macro_rules! array_downcast { ($($i:tt)*) => {$( impl DowncastFrom for [T; $i] @@ -2334,9 +2296,9 @@ impl ListVec for ValueVec { #[inline(always)] fn into_vec(self) -> Vec { match self { - Self::U8(vec) => vec.into_iter().map(Into::into).collect(), - Self::U16(vec) => vec.into_iter().map(Into::into).collect(), - Self::List(vec) => vec.into_iter().map(Into::into).collect(), + Self::U8(vec) => vec.map(Into::into), + Self::U16(vec) => vec.map(Into::into), + Self::List(vec) => vec.map(Into::into), Self::Value(vec) => vec, } } diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 60d90644..41490317 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,9 +35,9 @@ jobs: linux0: imageName: 'ubuntu-latest' rust_target_run: 'x86_64-unknown-linux-gnu' - linux1: - imageName: 'ubuntu-latest' - rust_target_run: 'x86_64-unknown-linux-musl' + # linux1: + # imageName: 'ubuntu-latest' + # rust_target_run: 'x86_64-unknown-linux-musl' - template: rust-n.yml@templates parameters: diff --git a/tests/parquet.rs b/tests/parquet.rs index 491a56df..fe2528bf 100644 --- a/tests/parquet.rs +++ b/tests/parquet.rs @@ -1,3 +1,4 @@ +#![type_length_limit = "1207654"] #![allow( clippy::cognitive_complexity, clippy::type_complexity, diff --git a/tests/parquet_dist.rs b/tests/parquet_dist.rs index 9e2e3ff3..8629e01d 100644 --- a/tests/parquet_dist.rs +++ b/tests/parquet_dist.rs @@ -1,3 +1,4 @@ +#![type_length_limit = "1207654"] #![allow(clippy::cognitive_complexity, clippy::type_complexity)] #[cfg(feature = "constellation")] From 0391e4e0993e289a72bcc8b4a67b55365e108306 Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Mon, 6 Jul 2020 10:32:51 +0100 Subject: [PATCH 7/8] more clippy clean --- README.md | 2 +- amadeus-aws/src/cloudfront.rs | 10 ++-- amadeus-aws/src/file.rs | 4 +- amadeus-aws/src/lib.rs | 51 ++++++++++++++---- amadeus-commoncrawl/src/commoncrawl.rs | 12 ++--- amadeus-commoncrawl/src/lib.rs | 27 +++++++++- amadeus-commoncrawl/src/parser.rs | 28 +++++----- amadeus-core/src/file.rs | 2 +- amadeus-core/src/into_par_stream/slice.rs | 6 +-- amadeus-core/src/lib.rs | 42 +++++++++++++-- amadeus-core/src/par_pipe.rs | 4 +- amadeus-core/src/par_sink/combiner.rs | 8 +-- amadeus-core/src/par_sink/folder.rs | 8 +-- amadeus-core/src/par_sink/fork.rs | 11 ++-- amadeus-core/src/par_sink/tuple.rs | 4 +- amadeus-core/src/par_stream.rs | 2 + amadeus-core/src/par_stream/cloned.rs | 2 +- amadeus-core/src/util.rs | 1 + amadeus-derive/src/lib.rs | 43 +++++++++++---- amadeus-parquet/src/lib.rs | 24 +++++++-- amadeus-postgres/src/impls.rs | 2 +- amadeus-postgres/src/lib.rs | 65 ++++++++++++++++------- amadeus-serde/src/csv.rs | 6 +-- amadeus-serde/src/impls.rs | 4 ++ amadeus-serde/src/json.rs | 6 +-- amadeus-serde/src/lib.rs | 31 ++++++++++- amadeus-types/src/data.rs | 2 +- amadeus-types/src/lib.rs | 35 ++++++++++-- amadeus-types/src/list.rs | 6 ++- amadeus-types/src/time.rs | 10 ++-- amadeus-types/src/value.rs | 60 ++++++++++----------- amadeus-types/src/value_required.rs | 58 ++++++++++---------- azure-pipelines.yml | 4 +- examples/cloudfront_logs.rs | 15 +----- tests/into_par_stream.rs | 4 +- tests/into_par_stream_dist.rs | 4 +- 36 files changed, 411 insertions(+), 192 deletions(-) diff --git a/README.md b/README.md index 71ca7cea..0572df95 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ async fn main() -> Result<(), Box> { let row = row.ok()?.into_group().ok()?; row.get("uri")?.clone().into_url().ok() }) - .filter(|row| futures::future::ready(row.is_some())) + .filter(|row| row.is_some()) .map(Option::unwrap) .most_frequent(&pool, 100, 0.99, 0.002) .await; diff --git a/amadeus-aws/src/cloudfront.rs b/amadeus-aws/src/cloudfront.rs index 75a635d8..1dc18041 100644 --- a/amadeus-aws/src/cloudfront.rs +++ b/amadeus-aws/src/cloudfront.rs @@ -1,10 +1,12 @@ +#![allow(unused_qualifications)] + use async_compression::futures::bufread::GzipDecoder; use chrono::{NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; use futures::{future, io::BufReader, AsyncBufReadExt, FutureExt, StreamExt, TryStreamExt}; use http::{Method, StatusCode}; use rusoto_s3::{GetObjectRequest, Object, S3Client, S3}; use serde::{Deserialize, Serialize}; -use serde_closure::*; +use serde_closure::FnMut; use std::{ convert::identity, io::{self}, time::Duration }; @@ -276,7 +278,7 @@ mod http_serde { use http::{Method, StatusCode}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; - pub struct Serde(T); + pub(crate) struct Serde(T); impl Serialize for Serde<&Method> { fn serialize(&self, serializer: S) -> Result @@ -324,14 +326,14 @@ mod http_serde { } } - pub fn serialize(t: &T, serializer: S) -> Result + pub(crate) fn serialize(t: &T, serializer: S) -> Result where for<'a> Serde<&'a T>: Serialize, S: Serializer, { Serde(t).serialize(serializer) } - pub fn deserialize<'de, T, D>(deserializer: D) -> Result + pub(crate) fn deserialize<'de, T, D>(deserializer: D) -> Result where Serde: Deserialize<'de>, D: Deserializer<'de>, diff --git a/amadeus-aws/src/file.rs b/amadeus-aws/src/file.rs index af3d77e2..c742c249 100644 --- a/amadeus-aws/src/file.rs +++ b/amadeus-aws/src/file.rs @@ -84,7 +84,7 @@ impl Directory for S3Directory { || (current_path.depth() > 0 && current_path.last().unwrap() != path[current_path.depth() - 1]) { - current_path.pop().unwrap(); + let _ = current_path.pop().unwrap(); } while path.len() > current_path.depth() { current_path.push(path[current_path.depth()]); @@ -241,7 +241,7 @@ impl Page for S3Page { let mut buf_ = vec![0; len].into_boxed_slice(); let mut buf = &mut *buf_; let len: u64 = len.try_into().unwrap(); - let mut pos = 0u64; + let mut pos = 0_u64; let mut errors: usize = 0; let end = offset + len - 1; while !buf.is_empty() { diff --git a/amadeus-aws/src/lib.rs b/amadeus-aws/src/lib.rs index e5ab6df7..a143ee7e 100644 --- a/amadeus-aws/src/lib.rs +++ b/amadeus-aws/src/lib.rs @@ -1,5 +1,34 @@ +//! Harmonious distributed data processing & analysis in Rust. +//! +//!

+//! 📦  Crates.io  │  📑  GitHub  │  💬  Chat +//!

+//! +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.2/amadeus/source/index.html). + #![doc(html_root_url = "https://docs.rs/amadeus-aws/0.2.5")] #![feature(type_alias_impl_trait)] +#![warn( + // missing_copy_implementations, + // missing_debug_implementations, + // missing_docs, + trivial_numeric_casts, + unused_import_braces, + unused_qualifications, + unused_results, + unreachable_pub, + clippy::pedantic, +)] +#![allow( + clippy::module_name_repetitions, + clippy::if_not_else, + clippy::too_many_lines, + clippy::must_use_candidate, + clippy::type_repetition_in_bounds, + clippy::filter_map, + clippy::missing_errors_doc +)] +#![deny(unsafe_code)] mod cloudfront; mod file; @@ -144,7 +173,7 @@ pub enum AwsError { NoSuchBucket(String), NoSuchKey(String), HttpDispatch(rusoto_core::request::HttpDispatchError), - Credentials(rusoto_credential::CredentialsError), + Credentials(CredentialsError), Validation(String), ParseError(String), Unknown(rusoto_core::request::BufferedHttpResponse), @@ -156,8 +185,8 @@ impl Clone for AwsError { Self::NoSuchBucket(err) => Self::NoSuchBucket(err.clone()), Self::NoSuchKey(err) => Self::NoSuchKey(err.clone()), Self::HttpDispatch(err) => Self::HttpDispatch(err.clone()), - Self::Credentials(rusoto_credential::CredentialsError { message }) => { - Self::Credentials(rusoto_credential::CredentialsError { + Self::Credentials(CredentialsError { message }) => { + Self::Credentials(CredentialsError { message: message.clone(), }) } @@ -179,12 +208,12 @@ impl Clone for AwsError { impl PartialEq for AwsError { fn eq(&self, other: &Self) -> bool { match (self, other) { - (Self::NoSuchBucket(a), Self::NoSuchBucket(b)) => a == b, - (Self::NoSuchKey(a), Self::NoSuchKey(b)) => a == b, + (Self::NoSuchBucket(a), Self::NoSuchBucket(b)) + | (Self::NoSuchKey(a), Self::NoSuchKey(b)) + | (Self::Validation(a), Self::Validation(b)) + | (Self::ParseError(a), Self::ParseError(b)) => a == b, (Self::HttpDispatch(a), Self::HttpDispatch(b)) => a == b, (Self::Credentials(a), Self::Credentials(b)) => a == b, - (Self::Validation(a), Self::Validation(b)) => a == b, - (Self::ParseError(a), Self::ParseError(b)) => a == b, (Self::Unknown(a), Self::Unknown(b)) => a == b, (Self::Io(a), Self::Io(b)) => a == b, _ => false, @@ -195,12 +224,12 @@ impl error::Error for AwsError {} impl Display for AwsError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - Self::NoSuchBucket(err) => err.fmt(f), - Self::NoSuchKey(err) => err.fmt(f), + Self::NoSuchBucket(err) + | Self::NoSuchKey(err) + | Self::Validation(err) + | Self::ParseError(err) => err.fmt(f), Self::HttpDispatch(err) => err.fmt(f), Self::Credentials(err) => err.fmt(f), - Self::Validation(err) => err.fmt(f), - Self::ParseError(err) => err.fmt(f), Self::Unknown(err) => fmt::Debug::fmt(err, f), Self::Io(err) => err.fmt(f), } diff --git a/amadeus-commoncrawl/src/commoncrawl.rs b/amadeus-commoncrawl/src/commoncrawl.rs index 299da35d..07e26857 100644 --- a/amadeus-commoncrawl/src/commoncrawl.rs +++ b/amadeus-commoncrawl/src/commoncrawl.rs @@ -14,7 +14,7 @@ const CHOMP: usize = 2 << 13; // 8 KiB #[pin_project] #[derive(Clone, Debug)] -pub struct WarcParser { +pub(crate) struct WarcParser { #[pin] input: I, state: WarcParserState, @@ -30,7 +30,7 @@ enum WarcParserState { Done, } impl WarcParser { - pub fn new(input: I) -> WarcParser { + pub(crate) fn new(input: I) -> WarcParser { WarcParser { input, state: WarcParserState::Info, @@ -43,7 +43,7 @@ impl WarcParser where I: Read, { - pub fn next_borrowed(&mut self) -> Result>, io::Error> { + pub(crate) fn next_borrowed(&mut self) -> Result>, io::Error> { if let WarcParserState::Done = self.state { return Ok(None); } @@ -68,7 +68,7 @@ where } loop { - self.res.splice(..self.offset, iter::empty()); + let _ = self.res.splice(..self.offset, iter::empty()); self.offset = 0; if self.offset == self.res.len() { continue 'chomp; @@ -124,7 +124,7 @@ impl WarcParser where I: AsyncRead, { - pub fn poll_next_borrowed( + pub(crate) fn poll_next_borrowed( self: Pin<&mut Self>, cx: &mut Context, ) -> Poll>, io::Error>> { let mut self_ = self.project(); @@ -149,7 +149,7 @@ where } loop { - self_.res.splice(..*self_.offset, iter::empty()); + let _ = self_.res.splice(..*self_.offset, iter::empty()); *self_.offset = 0; if *self_.offset == self_.res.len() { continue 'chomp; diff --git a/amadeus-commoncrawl/src/lib.rs b/amadeus-commoncrawl/src/lib.rs index c996b0db..4a68833d 100644 --- a/amadeus-commoncrawl/src/lib.rs +++ b/amadeus-commoncrawl/src/lib.rs @@ -1,5 +1,30 @@ +//! Harmonious distributed data processing & analysis in Rust. +//! +//!

+//! 📦  Crates.io  │  📑  GitHub  │  💬  Chat +//!

+//! +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.2/amadeus/source/index.html). + #![doc(html_root_url = "https://docs.rs/amadeus-commoncrawl/0.2.5")] #![feature(type_alias_impl_trait)] +#![warn( + // missing_copy_implementations, + // missing_debug_implementations, + // missing_docs, + trivial_numeric_casts, + unused_import_braces, + unused_qualifications, + unused_results, + unreachable_pub, + clippy::pedantic, +)] +#![allow( + clippy::doc_markdown, + clippy::inline_always, + clippy::missing_errors_doc +)] +#![deny(unsafe_code)] mod commoncrawl; mod parser; @@ -7,7 +32,7 @@ mod parser; use async_compression::futures::bufread::GzipDecoder; // TODO: use stream or https://github.com/alexcrichton/flate2-rs/pull/214 use futures::{io::BufReader, AsyncBufReadExt, FutureExt, StreamExt, TryStreamExt}; use reqwest_resume::ClientExt; -use serde_closure::*; +use serde_closure::FnMut; use std::{io, time}; use amadeus_core::{ diff --git a/amadeus-commoncrawl/src/parser.rs b/amadeus-commoncrawl/src/parser.rs index f589a33d..82ec7919 100644 --- a/amadeus-commoncrawl/src/parser.rs +++ b/amadeus-commoncrawl/src/parser.rs @@ -1,11 +1,11 @@ //! Web ARChive format parser //! //! Takes data and separates records in headers and content. -use nom::{space, Err, IResult, Needed, *}; +use nom::{complete, do_parse, many1, map_res, named, opt, space, tag, Err, IResult, Needed}; use std::{fmt, str}; #[derive(Clone, Copy, PartialEq, Eq)] -pub enum RecordType { +pub(crate) enum RecordType { WARCInfo, Response, Resource, @@ -16,7 +16,7 @@ pub enum RecordType { Continuation, } impl RecordType { - pub fn parse(x: &str) -> RecordType { + pub(crate) fn parse(x: &str) -> RecordType { match x { "warcinfo" => RecordType::WARCInfo, "response" => RecordType::Response, @@ -33,14 +33,14 @@ impl RecordType { /// The WArc `Record` struct // #[derive(Clone)] -pub struct Record<'a> { - // lazy design should not use pub +pub(crate) struct Record<'a> { + // lazy design should not use pub(crate) /// WArc headers - pub type_: RecordType, - pub target_uri: Option<&'a str>, - pub ip_address: Option<&'a str>, + pub(crate) type_: RecordType, + pub(crate) target_uri: Option<&'a str>, + pub(crate) ip_address: Option<&'a str>, /// Content for call in a raw format - pub content: &'a [u8], + pub(crate) content: &'a [u8], } impl<'a> fmt::Debug for Record<'a> { @@ -154,7 +154,7 @@ named!(warc_header<&[u8], ((&str, &str), Vec<(&str,&str)>) >, /// } /// ``` #[inline(always)] -pub fn record(input: &[u8]) -> IResult<&[u8], Record> { +pub(crate) fn record(input: &[u8]) -> IResult<&[u8], Record> { // TODO if the stream parser does not get all the header it fails . // like a default size of 10 doesnt for for a producer warc_header(input).and_then(|(mut i, tuple_vec)| { @@ -165,13 +165,13 @@ pub fn record(input: &[u8]) -> IResult<&[u8], Record> { let mut type_ = None; let mut target_uri = None; let mut ip_address = None; - for &(k, v) in headers.iter() { + for &(k, v) in &headers { match k { "Content-Length" => { let length_number = v.parse::().unwrap(); if length_number <= i.len() { - content = Some(&i[0..length_number as usize]); - i = &i[length_number as usize..]; + content = Some(&i[0..length_number]); + i = &i[length_number..]; bytes_needed = 0; } else { bytes_needed = length_number - i.len(); @@ -232,4 +232,4 @@ named!(record_complete <&[u8], Record >, // } // } // ``` -named!(pub records<&[u8], Vec >, many1!(record_complete)); +named!(pub(crate) records<&[u8], Vec >, many1!(record_complete)); diff --git a/amadeus-core/src/file.rs b/amadeus-core/src/file.rs index e0773d1c..366a1504 100644 --- a/amadeus-core/src/file.rs +++ b/amadeus-core/src/file.rs @@ -305,7 +305,7 @@ where let mut self_ = self.project(); if self_.pending.is_none() { let start = *self_.offset; - let len = (self_.page.len() - start).min(buf.len() as u64) as usize; + let len = usize::try_from((self_.page.len() - start).min(buf.len() as u64)).unwrap(); let len = len.min(PAGE_SIZE); let pending = self_.page.read(start, len); *self_.pending = Some(pending); diff --git a/amadeus-core/src/into_par_stream/slice.rs b/amadeus-core/src/into_par_stream/slice.rs index 88e460e3..a47b95ab 100644 --- a/amadeus-core/src/into_par_stream/slice.rs +++ b/amadeus-core/src/into_par_stream/slice.rs @@ -1,7 +1,7 @@ use futures::Stream; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::{ - iter, pin::Pin, slice, task::{Context, Poll} + convert::Infallible, iter, pin::Pin, slice, task::{Context, Poll} }; use super::{ @@ -53,9 +53,7 @@ impl_par_dist_rename! { } } -enum NeverInner {} - -pub struct Never(NeverInner); +pub struct Never(Infallible); impl StreamTask for Never { type Item = Self; diff --git a/amadeus-core/src/lib.rs b/amadeus-core/src/lib.rs index 3ca597c9..1a3d97b3 100644 --- a/amadeus-core/src/lib.rs +++ b/amadeus-core/src/lib.rs @@ -1,19 +1,55 @@ +//! Harmonious distributed data processing & analysis in Rust. +//! +//!

+//! 📦  Crates.io  │  📑  GitHub  │  💬  Chat +//!

+//! +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. All functionality is re-exposed in [`amadeus`](https://docs.rs/amadeus/0.2/amadeus/). + #![doc(html_root_url = "https://docs.rs/amadeus-core/0.2.5")] #![recursion_limit = "25600"] +#![warn( + // missing_copy_implementations, + // missing_debug_implementations, + // missing_docs, + trivial_numeric_casts, + unused_import_braces, + unused_qualifications, + unused_results, + unreachable_pub, + clippy::pedantic, +)] +#![allow( + clippy::module_name_repetitions, + clippy::if_not_else, + clippy::similar_names, + clippy::type_repetition_in_bounds, + clippy::missing_errors_doc, + clippy::must_use_candidate, + clippy::unsafe_derive_deserialize, + clippy::inline_always, + clippy::option_option, + clippy::default_trait_access, + clippy::filter_map, + clippy::wildcard_imports, + clippy::needless_pass_by_value +)] +#![deny(unsafe_code)] macro_rules! impl_par_dist { ($($body:tt)*) => { $($body)* const _: () = { - use crate::impl_par_dist::*; + use $crate::impl_par_dist::*; #[allow(unused_imports)] - use crate::impl_par_dist::{combiner_par_sink,folder_par_sink}; + use $crate::impl_par_dist::{combiner_par_sink,folder_par_sink}; $($body)* }; } } mod impl_par_dist { - pub use crate::{ + #[allow(unused_imports)] + pub(crate) use crate::{ combiner_dist_sink as combiner_par_sink, folder_dist_sink as folder_par_sink, par_pipe::DistributedPipe as ParallelPipe, par_sink::{DistributedSink as ParallelSink, FromDistributedStream as FromParallelStream}, par_stream::DistributedStream as ParallelStream, pool::ProcessSend as Send }; } diff --git a/amadeus-core/src/par_pipe.rs b/amadeus-core/src/par_pipe.rs index 3630121d..3973e289 100644 --- a/amadeus-core/src/par_pipe.rs +++ b/amadeus-core/src/par_pipe.rs @@ -89,13 +89,13 @@ impl_par_dist_rename! { assert_parallel_sink(super::par_sink::Pipe::new(self, sink)) } - fn fork(self, sink: A, sink_ref: B) -> super::par_sink::Fork + fn fork(self, sink: A, sink_ref: B) -> Fork where A: ParallelSink, B: for<'a> ParallelSink<&'a Self::Item>, Self: Sized, { - assert_parallel_sink(super::par_sink::Fork::new(self, sink, sink_ref)) + assert_parallel_sink(Fork::new(self, sink, sink_ref)) } fn for_each(self, f: F) -> ForEach diff --git a/amadeus-core/src/par_sink/combiner.rs b/amadeus-core/src/par_sink/combiner.rs index 502823ab..92b2f6d8 100644 --- a/amadeus-core/src/par_sink/combiner.rs +++ b/amadeus-core/src/par_sink/combiner.rs @@ -1,3 +1,5 @@ +#![allow(unused_imports,clippy::single_component_path_imports)] + use super::FolderSync; mod macros { @@ -39,11 +41,11 @@ mod macros { } }; } - pub use combiner_dist_sink; - pub use combiner_par_sink; + pub(crate) use combiner_dist_sink; + pub(crate) use combiner_par_sink; } -pub use macros::{combiner_dist_sink, combiner_par_sink}; +pub(crate) use macros::{combiner_dist_sink, combiner_par_sink}; pub trait CombinerSync { type Output; diff --git a/amadeus-core/src/par_sink/folder.rs b/amadeus-core/src/par_sink/folder.rs index e45d975d..0fef0d84 100644 --- a/amadeus-core/src/par_sink/folder.rs +++ b/amadeus-core/src/par_sink/folder.rs @@ -1,4 +1,4 @@ -#![allow(clippy::type_complexity)] +#![allow(unused_imports,clippy::single_component_path_imports)] use derive_new::new; use educe::Educe; @@ -53,11 +53,11 @@ mod macros { } }; } - pub use folder_dist_sink; - pub use folder_par_sink; + pub(crate) use folder_dist_sink; + pub(crate) use folder_par_sink; } -pub use macros::{folder_dist_sink, folder_par_sink}; +pub(crate) use macros::{folder_dist_sink, folder_par_sink}; pub trait FolderSync { type Output; diff --git a/amadeus-core/src/par_sink/fork.rs b/amadeus-core/src/par_sink/fork.rs index cd5b80cc..f6d3ff50 100644 --- a/amadeus-core/src/par_sink/fork.rs +++ b/amadeus-core/src/par_sink/fork.rs @@ -1,3 +1,6 @@ +// TODO: document why this is sound +#![allow(unsafe_code)] + use derive_new::new; use futures::{pin_mut, stream, Stream, StreamExt as _}; use pin_project::pin_project; @@ -177,8 +180,8 @@ pub struct JoinStreamTaskAsync { impl Stream for JoinStreamTaskAsync where A: Stream, - B: crate::pipe::Pipe, - C: crate::pipe::Pipe, + B: Pipe, + C: Pipe, { type Item = Sum2; @@ -239,8 +242,8 @@ where impl Pipe for JoinStreamTaskAsync where A: Pipe, - B: crate::pipe::Pipe, - C: crate::pipe::Pipe, + B: Pipe, + C: Pipe, { type Item = Sum2; diff --git a/amadeus-core/src/par_sink/tuple.rs b/amadeus-core/src/par_sink/tuple.rs index 8583399e..2840188d 100644 --- a/amadeus-core/src/par_sink/tuple.rs +++ b/amadeus-core/src/par_sink/tuple.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use std::{ future::Future, pin::Pin, task::{Context, Poll} }; -use sum::*; +use sum::{Sum0, Sum1, Sum2, Sum3, Sum4, Sum5, Sum6, Sum7, Sum8}; use super::{ DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend @@ -153,7 +153,7 @@ macro_rules! impl_tuple { *self_.pending = match stream.as_mut().poll_next(cx) { Poll::Ready(x) => Some(x), Poll::Pending => None}; } $({ - let pending = &mut self_.pending; + let pending = &mut *self_.pending; let given = &mut self_.given.$num; let stream_ = stream::poll_fn(|cx| { if !*given && pending.is_some() { diff --git a/amadeus-core/src/par_stream.rs b/amadeus-core/src/par_stream.rs index a85978f7..46d7ca99 100644 --- a/amadeus-core/src/par_stream.rs +++ b/amadeus-core/src/par_stream.rs @@ -1,6 +1,8 @@ // TODO: P: Pool -> impl Pool: async_trait triggers https://github.com/rust-lang/rust/issues/71869 // TODO: how to dedup?? +#![allow(clippy::too_many_lines)] + mod chain; mod cloned; mod filter; diff --git a/amadeus-core/src/par_stream/cloned.rs b/amadeus-core/src/par_stream/cloned.rs index e5f65758..281ee8d7 100644 --- a/amadeus-core/src/par_stream/cloned.rs +++ b/amadeus-core/src/par_stream/cloned.rs @@ -94,6 +94,6 @@ where self.project() .task .poll_next(cx, stream) - .map(|item| item.cloned()) + .map(Option::<&_>::cloned) } } diff --git a/amadeus-core/src/util.rs b/amadeus-core/src/util.rs index fa3cb7d5..487d2b2f 100644 --- a/amadeus-core/src/util.rs +++ b/amadeus-core/src/util.rs @@ -223,6 +223,7 @@ where /// # Safety /// /// Not. +#[allow(unsafe_code)] pub unsafe fn transmute(a: A) -> B { use std::mem; assert_eq!( diff --git a/amadeus-derive/src/lib.rs b/amadeus-derive/src/lib.rs index 4e0dcae3..953a8b13 100644 --- a/amadeus-derive/src/lib.rs +++ b/amadeus-derive/src/lib.rs @@ -1,8 +1,30 @@ +//! Harmonious distributed data processing & analysis in Rust. +//! +//!

+//! 📦  Crates.io  │  📑  GitHub  │  💬  Chat +//!

+//! +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. This macro is re-exposed as [`amadeus::data::Data`](https://docs.rs/amadeus/0.2/amadeus/data/derive.Data.html). + #![doc(html_root_url = "https://docs.rs/amadeus-derive/0.2.5")] #![recursion_limit = "400"] -#![allow(clippy::useless_let_if_seq)] - -extern crate proc_macro; +#![warn( + missing_copy_implementations, + missing_debug_implementations, + missing_docs, + trivial_numeric_casts, + unused_import_braces, + unused_qualifications, + unused_results, + unreachable_pub, + clippy::pedantic +)] +#![allow( + clippy::doc_markdown, + clippy::too_many_lines, + clippy::useless_let_if_seq +)] +#![deny(unsafe_code)] use proc_macro2::{Span, TokenStream}; use quote::{quote, ToTokens}; @@ -121,12 +143,13 @@ fn impl_struct( let (impl_generics, ty_generics, where_clause) = ast.generics.split_for_impl(); - let where_clause = where_clause - .map(Clone::clone) - .unwrap_or_else(|| WhereClause { + let where_clause = where_clause.map_or_else( + || WhereClause { where_token: ::default(), predicates: Punctuated::new(), - }); + }, + Clone::clone, + ); let mut where_clause_with_data = where_clause.clone(); for TypeParam { ident, .. } in ast.generics.type_params() { where_clause_with_data @@ -639,9 +662,9 @@ fn impl_struct( fn impl_tuple_struct( ast: &DeriveInput, fields: &Punctuated, ) -> Result { - let _name = &ast.ident; - let _schema_name = Ident::new(&format!("{}Schema", _name), Span::call_site()); - let _reader_name = Ident::new(&format!("{}Reader", _name), Span::call_site()); + let name = &ast.ident; + let _schema_name = Ident::new(&format!("{}Schema", name), Span::call_site()); + let _reader_name = Ident::new(&format!("{}Reader", name), Span::call_site()); let (_impl_generics, _ty_generics, _where_clause) = ast.generics.split_for_impl(); diff --git a/amadeus-parquet/src/lib.rs b/amadeus-parquet/src/lib.rs index 8686e5e2..f6829a6e 100644 --- a/amadeus-parquet/src/lib.rs +++ b/amadeus-parquet/src/lib.rs @@ -1,3 +1,11 @@ +//! Harmonious distributed data processing & analysis in Rust. +//! +//!

+//! 📦  Crates.io  │  📑  GitHub  │  💬  Chat +//!

+//! +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.2/amadeus/source/index.html). + #![doc(html_root_url = "https://docs.rs/amadeus-parquet/0.2.5")] #![feature(bufreader_seek_relative)] #![feature(read_initializer)] @@ -8,16 +16,24 @@ // missing_copy_implementations, // missing_debug_implementations, // missing_docs, - // trivial_casts, // trivial_numeric_casts, unused_import_braces, unused_qualifications, unused_results, // unreachable_pub, - // clippy::pedantic + // clippy::pedantic, +)] +#![allow( + clippy::module_name_repetitions, + clippy::similar_names, + clippy::if_not_else, + clippy::must_use_candidate, + clippy::missing_errors_doc, + dead_code, + clippy::all, + incomplete_features )] -// from https://github.com/rust-unofficial/patterns/blob/master/anti_patterns/deny-warnings.md -#![allow(dead_code, incomplete_features, clippy::all)] +// #![deny(unsafe_code)] #[cfg(test)] extern crate test; diff --git a/amadeus-postgres/src/impls.rs b/amadeus-postgres/src/impls.rs index 0a243e9f..ec2e8875 100644 --- a/amadeus-postgres/src/impls.rs +++ b/amadeus-postgres/src/impls.rs @@ -18,7 +18,7 @@ where } default fn decode( type_: &::postgres::types::Type, buf: Option<&[u8]>, - ) -> Result> { + ) -> Result> { T::decode(type_, buf).map(Box::new) } } diff --git a/amadeus-postgres/src/lib.rs b/amadeus-postgres/src/lib.rs index 4b656b0d..6fc018e9 100644 --- a/amadeus-postgres/src/lib.rs +++ b/amadeus-postgres/src/lib.rs @@ -1,12 +1,39 @@ -// TODO: -// Check types? These might work?? -// select column_name, is_nullable, data_type, character_maximum_length, * from information_schema.columns where table_name = 'weather' order by ordinal_position; -// select attname, atttypid, atttypmod, attnotnull, attndims from pg_attribute where attrelid = 'public.weather'::regclass and attnum > 0 and not attisdropped; +//! Harmonious distributed data processing & analysis in Rust. +//! +//!

+//! 📦  Crates.io  │  📑  GitHub  │  💬  Chat +//!

+//! +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.2/amadeus/source/index.html). #![doc(html_root_url = "https://docs.rs/amadeus-postgres/0.2.5")] #![feature(specialization)] #![feature(type_alias_impl_trait)] -#![allow(incomplete_features)] +#![warn( + // missing_copy_implementations, + // missing_debug_implementations, + // missing_docs, + trivial_numeric_casts, + unused_import_braces, + unused_qualifications, + unused_results, + unreachable_pub, + clippy::pedantic, +)] +#![allow( + clippy::module_name_repetitions, + clippy::similar_names, + clippy::if_not_else, + clippy::must_use_candidate, + clippy::missing_errors_doc, + incomplete_features +)] +#![deny(unsafe_code)] + +// TODO: +// Check types? These might work?? +// select column_name, is_nullable, data_type, character_maximum_length, * from information_schema.columns where table_name = 'weather' order by ordinal_position; +// select attname, atttypid, atttypmod, attnotnull, attndims from pg_attribute where attrelid = 'public.weather'::regclass and attnum > 0 and not attisdropped; mod impls; @@ -19,7 +46,7 @@ use futures::{ready, stream, FutureExt, Stream, StreamExt, TryStreamExt}; use pin_project::pin_project; use postgres::{CopyOutStream, Error as InternalPostgresError}; use serde::{Deserialize, Serialize}; -use serde_closure::*; +use serde_closure::FnMut; use std::{ convert::TryFrom, error, fmt::{self, Debug, Display}, io, io::Cursor, marker::PhantomData, ops::Fn, path::PathBuf, pin::Pin, str, sync::Arc, task::{Context, Poll}, time::Duration }; @@ -219,7 +246,7 @@ where .connect(postgres::tls::NoTls) .await .expect("Error handling not yet implemented. Tracking at https://github.com/constellation-rs/amadeus/issues/63"); - tokio::spawn(async move { + let _ = tokio::spawn(async move { let _ = connection.await; }); let client = Arc::new(client); @@ -241,7 +268,7 @@ where .map_ok(|row| { Row::decode( &postgres::types::Type::RECORD, - row.as_ref().map(|bytes| bytes.as_ref()), + row.as_ref().map(AsRef::as_ref), ) .expect("Error handling not yet implemented. Tracking at https://github.com/constellation-rs/amadeus/issues/63") }) @@ -317,22 +344,24 @@ impl Stream for BinaryCopyOutStream { } check_remaining(&chunk, 2)?; - let len = chunk.get_i16(); - if len == -1 { + let row_len = chunk.get_i16(); + if row_len == -1 { return Poll::Ready(None); } - assert_eq!(len, 1); + assert_eq!(row_len, 1); check_remaining(&chunk, 4)?; - let len = chunk.get_i32(); - if len == -1 { + let field_len = chunk.get_i32(); + if field_len == -1 { Poll::Ready(Some(Ok(None))) } else { - let len = len as usize; - check_remaining(&chunk, len)?; - let start = chunk.position() as usize; - Poll::Ready(Some(Ok(Some(chunk.into_inner().slice(start..start + len))))) + let field_len = usize::try_from(field_len).unwrap(); + check_remaining(&chunk, field_len)?; + let start = usize::try_from(chunk.position()).unwrap(); + Poll::Ready(Some(Ok(Some( + chunk.into_inner().slice(start..start + field_len), + )))) } } } @@ -444,7 +473,7 @@ impl DisplayFmt where F: Fn(&mut fmt::Formatter) -> fmt::Result, { - pub fn new(f: F) -> Self { + fn new(f: F) -> Self { Self(f) } } diff --git a/amadeus-serde/src/csv.rs b/amadeus-serde/src/csv.rs index 0599281e..bd2a0a9c 100644 --- a/amadeus-serde/src/csv.rs +++ b/amadeus-serde/src/csv.rs @@ -2,7 +2,7 @@ use csv::Error as InternalCsvError; use educe::Educe; use futures::{pin_mut, stream, AsyncReadExt, FutureExt, StreamExt}; use serde::{Deserialize, Serialize}; -use serde_closure::*; +use serde_closure::FnMut; use std::{ error, fmt::{self, Display}, io::Cursor, marker::PhantomData }; @@ -152,13 +152,13 @@ where mod csverror { use serde::{Deserializer, Serializer}; - pub fn serialize(_t: &T, _serializer: S) -> Result + pub(crate) fn serialize(_t: &T, _serializer: S) -> Result where S: Serializer, { unimplemented!() } - pub fn deserialize<'de, T, D>(_deserializer: D) -> Result + pub(crate) fn deserialize<'de, T, D>(_deserializer: D) -> Result where D: Deserializer<'de>, { diff --git a/amadeus-serde/src/impls.rs b/amadeus-serde/src/impls.rs index d6334b3f..b88fc087 100644 --- a/amadeus-serde/src/impls.rs +++ b/amadeus-serde/src/impls.rs @@ -1,3 +1,5 @@ +#![allow(clippy::too_many_lines)] + use linked_hash_map::LinkedHashMap; use serde::{ de::{self, MapAccess, SeqAccess, Visitor}, ser::{SerializeSeq, SerializeStruct, SerializeTupleStruct}, Deserializer, Serializer @@ -479,6 +481,7 @@ macro_rules! array { S: Serializer, { let self_: *const Self = self; + #[allow(unsafe_code)] let self_: &[SerdeSerialize; $i] = unsafe{ &*(self_ as *const _)}; serde::Serialize::serialize(self_, serializer) } @@ -530,6 +533,7 @@ macro_rules! array { D: Deserializer<'de>, { let self_: Box<[SerdeDeserialize; $i]> = serde::Deserialize::deserialize(deserializer)?; + #[allow(unsafe_code)] Ok(unsafe { Box::from_raw(Box::into_raw(self_) as *mut [T; $i]) }) } } diff --git a/amadeus-serde/src/json.rs b/amadeus-serde/src/json.rs index bec8c36f..c67afd07 100644 --- a/amadeus-serde/src/json.rs +++ b/amadeus-serde/src/json.rs @@ -1,7 +1,7 @@ use educe::Educe; use futures::{pin_mut, stream, AsyncReadExt, FutureExt, StreamExt}; use serde::{Deserialize, Serialize}; -use serde_closure::*; +use serde_closure::FnMut; use serde_json::Error as InternalJsonError; use std::{ error, fmt::{self, Debug, Display}, io::{self, Cursor}, marker::PhantomData @@ -106,13 +106,13 @@ where mod jsonerror { use serde::{Deserializer, Serializer}; - pub fn serialize(_t: &T, _serializer: S) -> Result + pub(crate) fn serialize(_t: &T, _serializer: S) -> Result where S: Serializer, { unimplemented!() } - pub fn deserialize<'de, T, D>(_deserializer: D) -> Result + pub(crate) fn deserialize<'de, T, D>(_deserializer: D) -> Result where D: Deserializer<'de>, { diff --git a/amadeus-serde/src/lib.rs b/amadeus-serde/src/lib.rs index ad9ad23b..1ff749c7 100644 --- a/amadeus-serde/src/lib.rs +++ b/amadeus-serde/src/lib.rs @@ -1,7 +1,36 @@ +//! Harmonious distributed data processing & analysis in Rust. +//! +//!

+//! 📦  Crates.io  │  📑  GitHub  │  💬  Chat +//!

+//! +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.2/amadeus/source/index.html). + #![doc(html_root_url = "https://docs.rs/amadeus-serde/0.2.5")] #![feature(specialization)] #![feature(type_alias_impl_trait)] -#![allow(incomplete_features)] +#![warn( + // missing_copy_implementations, + // missing_debug_implementations, + // missing_docs, + trivial_numeric_casts, + unused_import_braces, + unused_qualifications, + unused_results, + unreachable_pub, + clippy::pedantic, +)] +#![allow( + clippy::module_name_repetitions, + clippy::similar_names, + clippy::if_not_else, + clippy::must_use_candidate, + clippy::missing_errors_doc, + clippy::needless_pass_by_value, + clippy::default_trait_access, + incomplete_features +)] +#![deny(unsafe_code)] mod csv; mod impls; diff --git a/amadeus-types/src/data.rs b/amadeus-types/src/data.rs index 7be5cc8c..f7a71c08 100644 --- a/amadeus-types/src/data.rs +++ b/amadeus-types/src/data.rs @@ -8,7 +8,7 @@ use super::*; // + AmadeusOrd + DowncastFrom + Into pub trait Data: Clone + Debug + Send + Sized + 'static { - type Vec: list::ListVec; + type Vec: ListVec; type DynamicType; fn new_vec(_type: Self::DynamicType) -> Self::Vec; } diff --git a/amadeus-types/src/lib.rs b/amadeus-types/src/lib.rs index c78b9362..646174c0 100644 --- a/amadeus-types/src/lib.rs +++ b/amadeus-types/src/lib.rs @@ -1,7 +1,36 @@ -#![doc(html_root_url = "https://docs.rs/amadeus-types/0.2.5")] +//! Harmonious distributed data processing & analysis in Rust. +//! +//!

+//! 📦  Crates.io  │  📑  GitHub  │  💬  Chat +//!

+//! +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::data`](https://docs.rs/amadeus/0.2/amadeus/data/index.html). -//! Implementations of Rust types that correspond to Parquet logical types. -//! [`Record`](super::Record) is implemented for each of them. +#![doc(html_root_url = "https://docs.rs/amadeus-types/0.2.5")] +#![warn( + // missing_copy_implementations, + // missing_debug_implementations, + // missing_docs, + trivial_numeric_casts, + unused_import_braces, + unused_qualifications, + unused_results, + // unreachable_pub, + clippy::pedantic, +)] +#![allow( + clippy::module_name_repetitions, + clippy::similar_names, + clippy::if_not_else, + clippy::must_use_candidate, + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::wildcard_imports, + clippy::default_trait_access, + clippy::inline_always, + clippy::too_many_lines +)] +#![deny(unsafe_code)] #[macro_export] macro_rules! array { diff --git a/amadeus-types/src/list.rs b/amadeus-types/src/list.rs index 66e71730..9d358ef6 100644 --- a/amadeus-types/src/list.rs +++ b/amadeus-types/src/list.rs @@ -227,7 +227,9 @@ impl UnwindSafe for List where T: UnwindSafe {} impl RefUnwindSafe for List where T: RefUnwindSafe {} impl Unpin for List where T: Unpin {} // Implementers of ListVec must be Send/Sync if T is Send/Sync! +#[allow(unsafe_code)] unsafe impl Send for List where T: Send {} +#[allow(unsafe_code)] unsafe impl Sync for List where T: Sync {} impl Deref for List { @@ -413,8 +415,8 @@ mod test { #[test] fn test() { let mut list: List = List::new(); - list.push(0u8); - list.push(1u8); + list.push(0_u8); + list.push(1_u8); println!("{:#?}", list); let mut list: List = List::new(); list.push(Value::U8(0)); diff --git a/amadeus-types/src/time.rs b/amadeus-types/src/time.rs index 2695419a..b3411f4b 100644 --- a/amadeus-types/src/time.rs +++ b/amadeus-types/src/time.rs @@ -174,11 +174,11 @@ impl Timezone { -7200 => Some(Etc::GMTPlus2), -10800 => Some(Etc::GMTPlus3), -14400 => Some(Etc::GMTPlus4), - -18000 => Some(Etc::GMTPlus4), - -21600 => Some(Etc::GMTPlus5), - -25200 => Some(Etc::GMTPlus6), - -28800 => Some(Etc::GMTPlus7), - -32400 => Some(Etc::GMTPlus8), + -18000 => Some(Etc::GMTPlus5), + -21600 => Some(Etc::GMTPlus6), + -25200 => Some(Etc::GMTPlus7), + -28800 => Some(Etc::GMTPlus8), + -32400 => Some(Etc::GMTPlus9), -36000 => Some(Etc::GMTPlus10), -39600 => Some(Etc::GMTPlus11), -43200 => Some(Etc::GMTPlus12), diff --git a/amadeus-types/src/value.rs b/amadeus-types/src/value.rs index 1de6d01b..d5912d52 100644 --- a/amadeus-types/src/value.rs +++ b/amadeus-types/src/value.rs @@ -212,118 +212,118 @@ impl Hash for Value { fn hash(&self, state: &mut H) { match self { Self::Bool(value) => { - 0u8.hash(state); + 0_u8.hash(state); value.hash(state); } Self::U8(value) => { - 1u8.hash(state); + 1_u8.hash(state); value.hash(state); } Self::I8(value) => { - 2u8.hash(state); + 2_u8.hash(state); value.hash(state); } Self::U16(value) => { - 3u8.hash(state); + 3_u8.hash(state); value.hash(state); } Self::I16(value) => { - 4u8.hash(state); + 4_u8.hash(state); value.hash(state); } Self::U32(value) => { - 5u8.hash(state); + 5_u8.hash(state); value.hash(state); } Self::I32(value) => { - 6u8.hash(state); + 6_u8.hash(state); value.hash(state); } Self::U64(value) => { - 7u8.hash(state); + 7_u8.hash(state); value.hash(state); } Self::I64(value) => { - 8u8.hash(state); + 8_u8.hash(state); value.hash(state); } Self::F32(_value) => { - 9u8.hash(state); + 9_u8.hash(state); } Self::F64(_value) => { - 10u8.hash(state); + 10_u8.hash(state); } Self::Date(value) => { - 11u8.hash(state); + 11_u8.hash(state); value.hash(state); } Self::DateWithoutTimezone(value) => { - 11u8.hash(state); + 11_u8.hash(state); value.hash(state); } Self::Time(value) => { - 12u8.hash(state); + 12_u8.hash(state); value.hash(state); } Self::TimeWithoutTimezone(value) => { - 12u8.hash(state); + 12_u8.hash(state); value.hash(state); } Self::DateTime(value) => { - 13u8.hash(state); + 13_u8.hash(state); value.hash(state); } Self::DateTimeWithoutTimezone(value) => { - 13u8.hash(state); + 13_u8.hash(state); value.hash(state); } Self::Timezone(value) => { - 13u8.hash(state); + 13_u8.hash(state); value.hash(state); } Self::Decimal(_value) => { - 14u8.hash(state); + 14_u8.hash(state); } Self::Bson(value) => { - 15u8.hash(state); + 15_u8.hash(state); value.hash(state); } Self::String(value) => { - 16u8.hash(state); + 16_u8.hash(state); value.hash(state); } Self::Json(value) => { - 17u8.hash(state); + 17_u8.hash(state); value.hash(state); } Self::Enum(value) => { - 18u8.hash(state); + 18_u8.hash(state); value.hash(state); } Self::Url(value) => { - 19u8.hash(state); + 19_u8.hash(state); value.hash(state); } Self::Webpage(value) => { - 20u8.hash(state); + 20_u8.hash(state); value.hash(state); } Self::IpAddr(value) => { - 21u8.hash(state); + 21_u8.hash(state); value.hash(state); } Self::List(value) => { - 22u8.hash(state); + 22_u8.hash(state); value.hash(state); } Self::Map(_value) => { - 23u8.hash(state); + 23_u8.hash(state); } Self::Group(_value) => { - 24u8.hash(state); + 24_u8.hash(state); } Self::Option(value) => { - 25u8.hash(state); + 25_u8.hash(state); value.hash(state); } } diff --git a/amadeus-types/src/value_required.rs b/amadeus-types/src/value_required.rs index c6dd3f69..9f1a7474 100644 --- a/amadeus-types/src/value_required.rs +++ b/amadeus-types/src/value_required.rs @@ -83,115 +83,115 @@ impl Hash for ValueRequired { fn hash(&self, state: &mut H) { match self { Self::Bool(value) => { - 0u8.hash(state); + 0_u8.hash(state); value.hash(state); } Self::U8(value) => { - 1u8.hash(state); + 1_u8.hash(state); value.hash(state); } Self::I8(value) => { - 2u8.hash(state); + 2_u8.hash(state); value.hash(state); } Self::U16(value) => { - 3u8.hash(state); + 3_u8.hash(state); value.hash(state); } Self::I16(value) => { - 4u8.hash(state); + 4_u8.hash(state); value.hash(state); } Self::U32(value) => { - 5u8.hash(state); + 5_u8.hash(state); value.hash(state); } Self::I32(value) => { - 6u8.hash(state); + 6_u8.hash(state); value.hash(state); } Self::U64(value) => { - 7u8.hash(state); + 7_u8.hash(state); value.hash(state); } Self::I64(value) => { - 8u8.hash(state); + 8_u8.hash(state); value.hash(state); } Self::F32(_value) => { - 9u8.hash(state); + 9_u8.hash(state); } Self::F64(_value) => { - 10u8.hash(state); + 10_u8.hash(state); } Self::Date(value) => { - 11u8.hash(state); + 11_u8.hash(state); value.hash(state); } Self::DateWithoutTimezone(value) => { - 11u8.hash(state); + 11_u8.hash(state); value.hash(state); } Self::Time(value) => { - 12u8.hash(state); + 12_u8.hash(state); value.hash(state); } Self::TimeWithoutTimezone(value) => { - 12u8.hash(state); + 12_u8.hash(state); value.hash(state); } Self::DateTime(value) => { - 13u8.hash(state); + 13_u8.hash(state); value.hash(state); } Self::DateTimeWithoutTimezone(value) => { - 13u8.hash(state); + 13_u8.hash(state); value.hash(state); } Self::Timezone(value) => { - 13u8.hash(state); + 13_u8.hash(state); value.hash(state); } Self::Decimal(_value) => { - 14u8.hash(state); + 14_u8.hash(state); } Self::Bson(value) => { - 15u8.hash(state); + 15_u8.hash(state); value.hash(state); } Self::String(value) => { - 16u8.hash(state); + 16_u8.hash(state); value.hash(state); } Self::Json(value) => { - 17u8.hash(state); + 17_u8.hash(state); value.hash(state); } Self::Enum(value) => { - 18u8.hash(state); + 18_u8.hash(state); value.hash(state); } Self::Url(value) => { - 19u8.hash(state); + 19_u8.hash(state); value.hash(state); } Self::Webpage(value) => { - 20u8.hash(state); + 20_u8.hash(state); value.hash(state); } Self::IpAddr(value) => { - 21u8.hash(state); + 21_u8.hash(state); value.hash(state); } Self::List(value) => { - 22u8.hash(state); + 22_u8.hash(state); value.hash(state); } Self::Map(_value) => { - 23u8.hash(state); + 23_u8.hash(state); } Self::Group(_value) => { - 24u8.hash(state); + 24_u8.hash(state); } } } diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 41490317..c163301d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,7 +20,7 @@ jobs: endpoint: alecmocatta default: rust_toolchain: nightly - rust_lint_toolchain: nightly-2020-07-05 + rust_lint_toolchain: nightly-2020-07-06 rust_flags: '' rust_features_clippy: ';aws;commoncrawl;parquet;postgres;csv;json;constellation aws commoncrawl parquet postgres csv json' rust_features: 'constellation aws commoncrawl parquet postgres csv json' @@ -44,7 +44,7 @@ jobs: endpoint: alecmocatta default: rust_toolchain: nightly - rust_lint_toolchain: nightly-2020-07-05 + rust_lint_toolchain: nightly-2020-07-06 rust_flags: '' rust_features_clippy: ';aws;commoncrawl;parquet;postgres;csv;json;aws commoncrawl parquet postgres csv json' rust_features: 'aws commoncrawl parquet postgres csv json' diff --git a/examples/cloudfront_logs.rs b/examples/cloudfront_logs.rs index 83de457d..900f2f4c 100644 --- a/examples/cloudfront_logs.rs +++ b/examples/cloudfront_logs.rs @@ -27,30 +27,19 @@ async fn main() { .await .unwrap(); - let histogram = rows // (sample, (histogram, count)) = rows + let histogram = rows .par_stream() .map(Result::unwrap) .pipe( pool, - // Identity.sample_unstable(10), - // ( Identity .map(|row: CloudfrontRow| (row.time.truncate_minutes(60), ())) - .group_by( - Identity.count() - // || 0, - // |count, fold| count + fold.map_left(|()| 1).into_inner(), - ), - // Identity.map(|row: CloudfrontRow| (row.time.truncate_minutes(60), ())).group_by(Identity.count()), - // Identity.count(), - // ), + .group_by(Identity.count()), ) .await; let mut histogram = histogram.into_iter().collect::>(); histogram.sort(); - // println!("{} log lines analysed.", count); - // println!("sample: {:#?}", sample); println!( "histogram:\n {}", histogram diff --git a/tests/into_par_stream.rs b/tests/into_par_stream.rs index 86aa59c1..d20dbbfb 100644 --- a/tests/into_par_stream.rs +++ b/tests/into_par_stream.rs @@ -15,7 +15,7 @@ async fn into_par_stream() { assert_eq!(res, 6); let slice = [ - 0usize, 1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, + 0_usize, 1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, ]; for i in 0..slice.len() { @@ -24,7 +24,7 @@ async fn into_par_stream() { .into_par_stream() .fold( &pool, - || 0usize, + || 0_usize, |a: usize, b: Either| a + b.into_inner(), ) .await; diff --git a/tests/into_par_stream_dist.rs b/tests/into_par_stream_dist.rs index 82640c87..383d60d6 100644 --- a/tests/into_par_stream_dist.rs +++ b/tests/into_par_stream_dist.rs @@ -15,7 +15,7 @@ async fn main() { assert_eq!(res, 6); let slice = [ - 0usize, 1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, + 0_usize, 1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, ]; for i in 0..slice.len() { @@ -24,7 +24,7 @@ async fn main() { .into_dist_stream() .fold( &pool, - FnMut!(|| 0usize), + FnMut!(|| 0_usize), FnMut!(|a: usize, b: Either| a + b.into_inner()), ) .await; From 33cd62f049b322a7696568bc30e1d69ee473e319 Mon Sep 17 00:00:00 2001 From: alecmocatta Date: Mon, 6 Jul 2020 10:59:34 +0100 Subject: [PATCH 8/8] v0.3.0 --- Cargo.toml | 20 ++++++++++---------- README.md | 2 +- amadeus-aws/Cargo.toml | 8 ++++---- amadeus-aws/src/lib.rs | 4 ++-- amadeus-commoncrawl/Cargo.toml | 8 ++++---- amadeus-commoncrawl/src/lib.rs | 4 ++-- amadeus-core/Cargo.toml | 4 ++-- amadeus-core/src/lib.rs | 4 ++-- amadeus-derive/Cargo.toml | 4 ++-- amadeus-derive/src/lib.rs | 4 ++-- amadeus-parquet/Cargo.toml | 8 ++++---- amadeus-parquet/src/lib.rs | 4 ++-- amadeus-postgres/Cargo.toml | 8 ++++---- amadeus-postgres/src/lib.rs | 4 ++-- amadeus-serde/Cargo.toml | 8 ++++---- amadeus-serde/src/lib.rs | 4 ++-- amadeus-types/Cargo.toml | 6 +++--- amadeus-types/src/lib.rs | 4 ++-- src/lib.rs | 2 +- 19 files changed, 55 insertions(+), 55 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9c7fe567..1c36e521 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "amadeus" -version = "0.2.5" +version = "0.3.0" license = "Apache-2.0" authors = ["Alec Mocatta "] categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"] @@ -14,7 +14,7 @@ parquet postgres aws s3 cloudfront elb json csv logs hadoop hdfs arrow common cr """ repository = "https://github.com/alecmocatta/amadeus" homepage = "https://github.com/alecmocatta/amadeus" -documentation = "https://docs.rs/amadeus/0.2.5" +documentation = "https://docs.rs/amadeus/0.3.0" readme = "README.md" edition = "2018" @@ -35,14 +35,14 @@ json = ["amadeus-serde", "amadeus-derive/serde"] features = ["constellation", "aws", "commoncrawl", "parquet", "postgres", "csv", "json"] [dependencies] -amadeus-core = { version = "=0.2.5", path = "amadeus-core" } -amadeus-derive = { version = "=0.2.5", path = "amadeus-derive" } -amadeus-types = { version = "=0.2.5", path = "amadeus-types" } -amadeus-aws = { version = "=0.2.5", path = "amadeus-aws", optional = true } -amadeus-commoncrawl = { version = "=0.2.5", path = "amadeus-commoncrawl", optional = true } -amadeus-parquet = { version = "=0.2.5", path = "amadeus-parquet", optional = true } -amadeus-postgres = { version = "=0.2.5", path = "amadeus-postgres", optional = true } -amadeus-serde = { version = "=0.2.5", path = "amadeus-serde", optional = true } +amadeus-core = { version = "=0.3.0", path = "amadeus-core" } +amadeus-derive = { version = "=0.3.0", path = "amadeus-derive" } +amadeus-types = { version = "=0.3.0", path = "amadeus-types" } +amadeus-aws = { version = "=0.3.0", path = "amadeus-aws", optional = true } +amadeus-commoncrawl = { version = "=0.3.0", path = "amadeus-commoncrawl", optional = true } +amadeus-parquet = { version = "=0.3.0", path = "amadeus-parquet", optional = true } +amadeus-postgres = { version = "=0.3.0", path = "amadeus-postgres", optional = true } +amadeus-serde = { version = "=0.3.0", path = "amadeus-serde", optional = true } async-std = { version = "1.6", features = ["unstable"] } constellation-rs = { version = "0.1", default-features = false, optional = true } derive-new = "0.5" diff --git a/README.md b/README.md index 0572df95..d1346372 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ We aim to create a community that is welcoming and helpful to anyone that is int Amadeus has deep, pluggable, integration with various file formats, databases and interfaces: -| Data format | [`Source`](https://docs.rs/amadeus/0.2/amadeus/trait.Source.html) | [`Destination`](https://docs.rs/amadeus/0.2/amadeus/trait.Destination.html) | +| Data format | [`Source`](https://docs.rs/amadeus/0.3/amadeus/trait.Source.html) | [`Destination`](https://docs.rs/amadeus/0.3/amadeus/trait.Destination.html) | |---|---|---| | CSV | ✔ | ✔ | | JSON | ✔ | ✔ | diff --git a/amadeus-aws/Cargo.toml b/amadeus-aws/Cargo.toml index e3f62d05..d8ff4a39 100644 --- a/amadeus-aws/Cargo.toml +++ b/amadeus-aws/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "amadeus-aws" -version = "0.2.5" +version = "0.3.0" license = "Apache-2.0" authors = ["Alec Mocatta "] categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"] @@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust. """ repository = "https://github.com/alecmocatta/amadeus" homepage = "https://github.com/alecmocatta/amadeus" -documentation = "https://docs.rs/amadeus/0.2.5" +documentation = "https://docs.rs/amadeus/0.3.0" readme = "README.md" edition = "2018" @@ -19,8 +19,8 @@ azure-devops = { project = "alecmocatta/amadeus", pipeline = "tests" } maintenance = { status = "actively-developed" } [dependencies] -amadeus-core = { version = "=0.2.5", path = "../amadeus-core" } -amadeus-types = { version = "=0.2.5", path = "../amadeus-types" } +amadeus-core = { version = "=0.3.0", path = "../amadeus-core" } +amadeus-types = { version = "=0.3.0", path = "../amadeus-types" } async-compression = { version = "0.3.3", features = ["gzip", "futures-bufread"] } async-trait = "0.1" chrono = { version = "0.4", default-features = false } diff --git a/amadeus-aws/src/lib.rs b/amadeus-aws/src/lib.rs index a143ee7e..8ca9a86e 100644 --- a/amadeus-aws/src/lib.rs +++ b/amadeus-aws/src/lib.rs @@ -4,9 +4,9 @@ //! 📦  Crates.io  │  📑  GitHub  │  💬  Chat //!

//! -//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.2/amadeus/source/index.html). +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.3/amadeus/source/index.html). -#![doc(html_root_url = "https://docs.rs/amadeus-aws/0.2.5")] +#![doc(html_root_url = "https://docs.rs/amadeus-aws/0.3.0")] #![feature(type_alias_impl_trait)] #![warn( // missing_copy_implementations, diff --git a/amadeus-commoncrawl/Cargo.toml b/amadeus-commoncrawl/Cargo.toml index f1061287..cb1aad0b 100644 --- a/amadeus-commoncrawl/Cargo.toml +++ b/amadeus-commoncrawl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "amadeus-commoncrawl" -version = "0.2.5" +version = "0.3.0" license = "MIT OR Apache-2.0" authors = ["Stephen Becker IV ", "Alec Mocatta "] categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"] @@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust. """ repository = "https://github.com/alecmocatta/amadeus" homepage = "https://github.com/alecmocatta/amadeus" -documentation = "https://docs.rs/amadeus/0.2.5" +documentation = "https://docs.rs/amadeus/0.3.0" readme = "README.md" edition = "2018" @@ -19,8 +19,8 @@ azure-devops = { project = "alecmocatta/amadeus", pipeline = "tests" } maintenance = { status = "actively-developed" } [dependencies] -amadeus-core = { version = "=0.2.5", path = "../amadeus-core" } -amadeus-types = { version = "=0.2.5", path = "../amadeus-types" } +amadeus-core = { version = "=0.3.0", path = "../amadeus-core" } +amadeus-types = { version = "=0.3.0", path = "../amadeus-types" } async-compression = { version = "0.3.3", features = ["gzip", "futures-bufread"] } futures = "0.3" nom = "4.2.3" diff --git a/amadeus-commoncrawl/src/lib.rs b/amadeus-commoncrawl/src/lib.rs index 4a68833d..6cca6ae0 100644 --- a/amadeus-commoncrawl/src/lib.rs +++ b/amadeus-commoncrawl/src/lib.rs @@ -4,9 +4,9 @@ //! 📦  Crates.io  │  📑  GitHub  │  💬  Chat //!

//! -//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.2/amadeus/source/index.html). +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.3/amadeus/source/index.html). -#![doc(html_root_url = "https://docs.rs/amadeus-commoncrawl/0.2.5")] +#![doc(html_root_url = "https://docs.rs/amadeus-commoncrawl/0.3.0")] #![feature(type_alias_impl_trait)] #![warn( // missing_copy_implementations, diff --git a/amadeus-core/Cargo.toml b/amadeus-core/Cargo.toml index 36de531d..88247864 100644 --- a/amadeus-core/Cargo.toml +++ b/amadeus-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "amadeus-core" -version = "0.2.5" +version = "0.3.0" license = "Apache-2.0" authors = ["Alec Mocatta "] categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"] @@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust. """ repository = "https://github.com/alecmocatta/amadeus" homepage = "https://github.com/alecmocatta/amadeus" -documentation = "https://docs.rs/amadeus/0.2.5" +documentation = "https://docs.rs/amadeus/0.3.0" readme = "README.md" edition = "2018" diff --git a/amadeus-core/src/lib.rs b/amadeus-core/src/lib.rs index 1a3d97b3..9104029d 100644 --- a/amadeus-core/src/lib.rs +++ b/amadeus-core/src/lib.rs @@ -4,9 +4,9 @@ //! 📦  Crates.io  │  📑  GitHub  │  💬  Chat //!

//! -//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. All functionality is re-exposed in [`amadeus`](https://docs.rs/amadeus/0.2/amadeus/). +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. All functionality is re-exposed in [`amadeus`](https://docs.rs/amadeus/0.3/amadeus/). -#![doc(html_root_url = "https://docs.rs/amadeus-core/0.2.5")] +#![doc(html_root_url = "https://docs.rs/amadeus-core/0.3.0")] #![recursion_limit = "25600"] #![warn( // missing_copy_implementations, diff --git a/amadeus-derive/Cargo.toml b/amadeus-derive/Cargo.toml index cc3aea2e..03437613 100644 --- a/amadeus-derive/Cargo.toml +++ b/amadeus-derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "amadeus-derive" -version = "0.2.5" +version = "0.3.0" license = "Apache-2.0" authors = ["Alec Mocatta "] categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"] @@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust. """ repository = "https://github.com/alecmocatta/amadeus" homepage = "https://github.com/alecmocatta/amadeus" -documentation = "https://docs.rs/amadeus/0.2.5" +documentation = "https://docs.rs/amadeus/0.3.0" readme = "README.md" edition = "2018" diff --git a/amadeus-derive/src/lib.rs b/amadeus-derive/src/lib.rs index 953a8b13..b365595d 100644 --- a/amadeus-derive/src/lib.rs +++ b/amadeus-derive/src/lib.rs @@ -4,9 +4,9 @@ //! 📦  Crates.io  │  📑  GitHub  │  💬  Chat //!

//! -//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. This macro is re-exposed as [`amadeus::data::Data`](https://docs.rs/amadeus/0.2/amadeus/data/derive.Data.html). +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. This macro is re-exposed as [`amadeus::data::Data`](https://docs.rs/amadeus/0.3/amadeus/data/derive.Data.html). -#![doc(html_root_url = "https://docs.rs/amadeus-derive/0.2.5")] +#![doc(html_root_url = "https://docs.rs/amadeus-derive/0.3.0")] #![recursion_limit = "400"] #![warn( missing_copy_implementations, diff --git a/amadeus-parquet/Cargo.toml b/amadeus-parquet/Cargo.toml index b0af1260..30bf1b68 100644 --- a/amadeus-parquet/Cargo.toml +++ b/amadeus-parquet/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "amadeus-parquet" -version = "0.2.5" +version = "0.3.0" license = "Apache-2.0" authors = ["Alec Mocatta ", "Apache Arrow "] categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"] @@ -10,7 +10,7 @@ An Apache Parquet implementation in Rust. """ repository = "https://github.com/alecmocatta/amadeus" homepage = "https://github.com/alecmocatta/amadeus" -documentation = "https://docs.rs/amadeus/0.2.5" +documentation = "https://docs.rs/amadeus/0.3.0" readme = "README.md" edition = "2018" @@ -19,8 +19,8 @@ azure-devops = { project = "alecmocatta/amadeus", pipeline = "tests" } maintenance = { status = "actively-developed" } [dependencies] -amadeus-core = { version = "=0.2.5", path = "../amadeus-core" } -amadeus-types = { version = "=0.2.5", path = "../amadeus-types" } +amadeus-core = { version = "=0.3.0", path = "../amadeus-core" } +amadeus-types = { version = "=0.3.0", path = "../amadeus-types" } async-trait = "0.1" brotli = "3.3" byteorder = "1.2" diff --git a/amadeus-parquet/src/lib.rs b/amadeus-parquet/src/lib.rs index f6829a6e..229eef94 100644 --- a/amadeus-parquet/src/lib.rs +++ b/amadeus-parquet/src/lib.rs @@ -4,9 +4,9 @@ //! 📦  Crates.io  │  📑  GitHub  │  💬  Chat //!

//! -//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.2/amadeus/source/index.html). +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.3/amadeus/source/index.html). -#![doc(html_root_url = "https://docs.rs/amadeus-parquet/0.2.5")] +#![doc(html_root_url = "https://docs.rs/amadeus-parquet/0.3.0")] #![feature(bufreader_seek_relative)] #![feature(read_initializer)] #![feature(specialization)] diff --git a/amadeus-postgres/Cargo.toml b/amadeus-postgres/Cargo.toml index ce7993b9..9ec510a1 100644 --- a/amadeus-postgres/Cargo.toml +++ b/amadeus-postgres/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "amadeus-postgres" -version = "0.2.5" +version = "0.3.0" license = "Apache-2.0" authors = ["Alec Mocatta "] categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"] @@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust. """ repository = "https://github.com/alecmocatta/amadeus" homepage = "https://github.com/alecmocatta/amadeus" -documentation = "https://docs.rs/amadeus/0.2.5" +documentation = "https://docs.rs/amadeus/0.3.0" readme = "README.md" edition = "2018" @@ -19,8 +19,8 @@ azure-devops = { project = "alecmocatta/amadeus", pipeline = "tests" } maintenance = { status = "actively-developed" } [dependencies] -amadeus-core = { version = "=0.2.5", path = "../amadeus-core" } -amadeus-types = { version = "=0.2.5", path = "../amadeus-types" } +amadeus-core = { version = "=0.3.0", path = "../amadeus-core" } +amadeus-types = { version = "=0.3.0", path = "../amadeus-types" } bytes = "0.5" chrono = { version = "0.4", default-features = false } educe = "0.4" diff --git a/amadeus-postgres/src/lib.rs b/amadeus-postgres/src/lib.rs index 6fc018e9..cce5670c 100644 --- a/amadeus-postgres/src/lib.rs +++ b/amadeus-postgres/src/lib.rs @@ -4,9 +4,9 @@ //! 📦  Crates.io  │  📑  GitHub  │  💬  Chat //!

//! -//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.2/amadeus/source/index.html). +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.3/amadeus/source/index.html). -#![doc(html_root_url = "https://docs.rs/amadeus-postgres/0.2.5")] +#![doc(html_root_url = "https://docs.rs/amadeus-postgres/0.3.0")] #![feature(specialization)] #![feature(type_alias_impl_trait)] #![warn( diff --git a/amadeus-serde/Cargo.toml b/amadeus-serde/Cargo.toml index eb248b78..b8f124fc 100644 --- a/amadeus-serde/Cargo.toml +++ b/amadeus-serde/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "amadeus-serde" -version = "0.2.5" +version = "0.3.0" license = "Apache-2.0" authors = ["Alec Mocatta "] categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"] @@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust. """ repository = "https://github.com/alecmocatta/amadeus" homepage = "https://github.com/alecmocatta/amadeus" -documentation = "https://docs.rs/amadeus/0.2.5" +documentation = "https://docs.rs/amadeus/0.3.0" readme = "README.md" edition = "2018" @@ -19,8 +19,8 @@ azure-devops = { project = "alecmocatta/amadeus", pipeline = "tests" } maintenance = { status = "actively-developed" } [dependencies] -amadeus-core = { version = "=0.2.5", path = "../amadeus-core" } -amadeus-types = { version = "=0.2.5", path = "../amadeus-types" } +amadeus-core = { version = "=0.3.0", path = "../amadeus-core" } +amadeus-types = { version = "=0.3.0", path = "../amadeus-types" } chrono = { version = "0.4", default-features = false, features = ["serde"] } csv = "1.0" educe = "0.4" diff --git a/amadeus-serde/src/lib.rs b/amadeus-serde/src/lib.rs index 1ff749c7..59a9cfeb 100644 --- a/amadeus-serde/src/lib.rs +++ b/amadeus-serde/src/lib.rs @@ -4,9 +4,9 @@ //! 📦  Crates.io  │  📑  GitHub  │  💬  Chat //!

//! -//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.2/amadeus/source/index.html). +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::source`](https://docs.rs/amadeus/0.3/amadeus/source/index.html). -#![doc(html_root_url = "https://docs.rs/amadeus-serde/0.2.5")] +#![doc(html_root_url = "https://docs.rs/amadeus-serde/0.3.0")] #![feature(specialization)] #![feature(type_alias_impl_trait)] #![warn( diff --git a/amadeus-types/Cargo.toml b/amadeus-types/Cargo.toml index 00760e9b..5b4e53f5 100644 --- a/amadeus-types/Cargo.toml +++ b/amadeus-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "amadeus-types" -version = "0.2.5" +version = "0.3.0" license = "Apache-2.0" authors = ["Alec Mocatta "] categories = ["concurrency", "science", "database", "date-and-time", "data-structures"] @@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust. """ repository = "https://github.com/alecmocatta/amadeus" homepage = "https://github.com/alecmocatta/amadeus" -documentation = "https://docs.rs/amadeus/0.2.5" +documentation = "https://docs.rs/amadeus/0.3.0" readme = "README.md" edition = "2018" @@ -19,7 +19,7 @@ azure-devops = { project = "alecmocatta/amadeus", pipeline = "tests" } maintenance = { status = "actively-developed" } [dependencies] -amadeus-core = { version = "=0.2.5", path = "../amadeus-core" } +amadeus-core = { version = "=0.3.0", path = "../amadeus-core" } chrono = { version = "0.4", default-features = false, features = ["std", "serde"] } chrono-tz = { version = "0.5", features = ["serde"] } fxhash = "0.2" diff --git a/amadeus-types/src/lib.rs b/amadeus-types/src/lib.rs index 646174c0..a46955ac 100644 --- a/amadeus-types/src/lib.rs +++ b/amadeus-types/src/lib.rs @@ -4,9 +4,9 @@ //! 📦  Crates.io  │  📑  GitHub  │  💬  Chat //!

//! -//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::data`](https://docs.rs/amadeus/0.2/amadeus/data/index.html). +//! This is a support crate of [Amadeus](https://github.com/constellation-rs/amadeus) and is not intended to be used directly. These types are re-exposed in [`amadeus::data`](https://docs.rs/amadeus/0.3/amadeus/data/index.html). -#![doc(html_root_url = "https://docs.rs/amadeus-types/0.2.5")] +#![doc(html_root_url = "https://docs.rs/amadeus-types/0.3.0")] #![warn( // missing_copy_implementations, // missing_debug_implementations, diff --git a/src/lib.rs b/src/lib.rs index 442db61d..2aa37381 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ //! 📦  Crates.io  │  📑  GitHub  │  💬  Chat //!

-#![doc(html_root_url = "https://docs.rs/amadeus/0.2.5")] +#![doc(html_root_url = "https://docs.rs/amadeus/0.3.0")] #![doc( html_logo_url = "https://raw.githubusercontent.com/constellation-rs/amadeus/master/logo.svg?sanitize=true" )]

where P: DerefMut + Unpin, - P::Target: ReducerAsync, + P::Target: ReducerAsync, { - type Item = ::Item; - type Output = ::Output; + type Output = >::Output; - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - self.get_mut().as_mut().poll_forward(cx, stream) - } - fn poll_output(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - self.get_mut().as_mut().poll_output(cx) + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + self.get_mut().as_mut().output() } } -impl ReducerAsync for &mut T +impl ReducerAsync for &mut T where - T: ReducerAsync + Unpin, + T: ReducerAsync + Unpin, { - type Item = T::Item; type Output = T::Output; - fn poll_forward( - mut self: Pin<&mut Self>, cx: &mut Context, - stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - Pin::new(&mut **self).poll_forward(cx, stream) - } - fn poll_output(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { - Pin::new(&mut **self).poll_output(cx) + fn output<'a>(mut self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { Pin::new(&mut **self).output().await }) } } @@ -92,8 +76,11 @@ where pub trait ParallelSink { type Output; type Pipe: ParallelPipe; - type ReduceA: ReducerSend>::Item> + Clone + Send; - type ReduceC: Reducer::Output, Output = Self::Output>; + type ReduceA: ReducerSend<>::Item> + Clone + Send; + type ReduceC: Reducer< + >::Item>>::Output, + Output = Self::Output, + >; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC); } @@ -107,14 +94,20 @@ pub(crate) fn assert_parallel_sink, Source>(r: R) -> R { pub trait DistributedSink { type Output; type Pipe: DistributedPipe; - type ReduceA: ReducerSend>::Item> + type ReduceA: ReducerSend<>::Item> + Clone + ProcessSend + Send; - type ReduceB: ReducerProcessSend::Output> - + Clone + type ReduceB: ReducerProcessSend< + >::Item>>::Output, + > + Clone + ProcessSend; - type ReduceC: Reducer::Output, Output = Self::Output>; + type ReduceC: Reducer< + >::Item>>::Output, + >>::Output, + Output = Self::Output, + >; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC); } diff --git a/amadeus-core/src/par_sink/all.rs b/amadeus-core/src/par_sink/all.rs index 6f8357ce..cd46f3d1 100644 --- a/amadeus-core/src/par_sink/all.rs +++ b/amadeus-core/src/par_sink/all.rs @@ -4,13 +4,13 @@ use futures::{ready, Stream}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - marker::PhantomData, pin::Pin, task::{Context, Poll} + future::Future, marker::PhantomData, pin::Pin, task::{Context, Poll} }; use super::{ DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend }; -use crate::pool::ProcessSend; +use crate::{pipe::Sink, pool::ProcessSend}; #[derive(new)] #[must_use] @@ -60,11 +60,10 @@ where )] pub struct AllReducer(F, PhantomData A>); -impl Reducer for AllReducer +impl Reducer for AllReducer where F: FnMut(A) -> bool, { - type Item = A; type Output = bool; type Async = AllReducerAsync; @@ -72,13 +71,13 @@ where AllReducerAsync(self.0, true, PhantomData) } } -impl ReducerProcessSend for AllReducer +impl ReducerProcessSend for AllReducer where F: FnMut(A) -> bool, { type Output = bool; } -impl ReducerSend for AllReducer +impl ReducerSend for AllReducer where F: FnMut(A) -> bool, { @@ -93,17 +92,13 @@ where )] pub struct AllReducerAsync(F, bool, PhantomData A>); -impl ReducerAsync for AllReducerAsync +impl Sink for AllReducerAsync where F: FnMut(A) -> bool, { - type Item = A; - type Output = bool; - #[inline(always)] - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, + fn poll_pipe( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, ) -> Poll<()> { let self_ = self.project(); while *self_.1 { @@ -115,16 +110,22 @@ where } Poll::Ready(()) } - fn poll_output(self: Pin<&mut Self>, _cx: &mut Context) -> Poll { - Poll::Ready(self.1) +} +impl ReducerAsync for AllReducerAsync +where + F: FnMut(A) -> bool, +{ + type Output = bool; + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { self.1 }) } } #[derive(Clone, Serialize, Deserialize)] pub struct BoolAndReducer; -impl Reducer for BoolAndReducer { - type Item = bool; +impl Reducer for BoolAndReducer { type Output = bool; type Async = BoolAndReducerAsync; @@ -132,23 +133,19 @@ impl Reducer for BoolAndReducer { BoolAndReducerAsync(true) } } -impl ReducerProcessSend for BoolAndReducer { +impl ReducerProcessSend for BoolAndReducer { type Output = bool; } -impl ReducerSend for BoolAndReducer { +impl ReducerSend for BoolAndReducer { type Output = bool; } #[pin_project] pub struct BoolAndReducerAsync(bool); -impl ReducerAsync for BoolAndReducerAsync { - type Item = bool; - type Output = bool; - +impl Sink for BoolAndReducerAsync { #[inline(always)] - fn poll_forward( - mut self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, + fn poll_pipe( + mut self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, ) -> Poll<()> { while self.0 { if let Some(item) = ready!(stream.as_mut().poll_next(cx)) { @@ -159,7 +156,11 @@ impl ReducerAsync for BoolAndReducerAsync { } Poll::Ready(()) } - fn poll_output(self: Pin<&mut Self>, _cx: &mut Context) -> Poll { - Poll::Ready(self.0) +} +impl ReducerAsync for BoolAndReducerAsync { + type Output = bool; + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { self.0 }) } } diff --git a/amadeus-core/src/par_sink/any.rs b/amadeus-core/src/par_sink/any.rs index 3af65580..a6b8ab48 100644 --- a/amadeus-core/src/par_sink/any.rs +++ b/amadeus-core/src/par_sink/any.rs @@ -4,13 +4,13 @@ use futures::{ready, Stream}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - marker::PhantomData, pin::Pin, task::{Context, Poll} + future::Future, marker::PhantomData, pin::Pin, task::{Context, Poll} }; use super::{ DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend }; -use crate::pool::ProcessSend; +use crate::{pipe::Sink, pool::ProcessSend}; #[derive(new)] #[must_use] @@ -60,11 +60,10 @@ where )] pub struct AnyReducer(F, PhantomData A>); -impl Reducer for AnyReducer +impl Reducer for AnyReducer where F: FnMut(A) -> bool, { - type Item = A; type Output = bool; type Async = AnyReducerAsync; @@ -72,13 +71,13 @@ where AnyReducerAsync(self.0, true, PhantomData) } } -impl ReducerProcessSend for AnyReducer +impl ReducerProcessSend for AnyReducer where F: FnMut(A) -> bool, { type Output = bool; } -impl ReducerSend for AnyReducer +impl ReducerSend for AnyReducer where F: FnMut(A) -> bool, { @@ -93,17 +92,13 @@ where )] pub struct AnyReducerAsync(F, bool, PhantomData A>); -impl ReducerAsync for AnyReducerAsync +impl Sink for AnyReducerAsync where F: FnMut(A) -> bool, { - type Item = A; - type Output = bool; - #[inline(always)] - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, + fn poll_pipe( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, ) -> Poll<()> { let self_ = self.project(); while *self_.1 { @@ -115,16 +110,22 @@ where } Poll::Ready(()) } - fn poll_output(self: Pin<&mut Self>, _cx: &mut Context) -> Poll { - Poll::Ready(!self.1) +} +impl ReducerAsync for AnyReducerAsync +where + F: FnMut(A) -> bool, +{ + type Output = bool; + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { !self.1 }) } } #[derive(Clone, Serialize, Deserialize)] pub struct BoolOrReducer; -impl Reducer for BoolOrReducer { - type Item = bool; +impl Reducer for BoolOrReducer { type Output = bool; type Async = BoolOrReducerAsync; @@ -132,10 +133,10 @@ impl Reducer for BoolOrReducer { BoolOrReducerAsync(true) } } -impl ReducerProcessSend for BoolOrReducer { +impl ReducerProcessSend for BoolOrReducer { type Output = bool; } -impl ReducerSend for BoolOrReducer { +impl ReducerSend for BoolOrReducer { type Output = bool; } @@ -143,14 +144,10 @@ impl ReducerSend for BoolOrReducer { #[derive(Serialize, Deserialize)] pub struct BoolOrReducerAsync(bool); -impl ReducerAsync for BoolOrReducerAsync { - type Item = bool; - type Output = bool; - +impl Sink for BoolOrReducerAsync { #[inline(always)] - fn poll_forward( - mut self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, + fn poll_pipe( + mut self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, ) -> Poll<()> { while self.0 { if let Some(item) = ready!(stream.as_mut().poll_next(cx)) { @@ -161,7 +158,11 @@ impl ReducerAsync for BoolOrReducerAsync { } Poll::Ready(()) } - fn poll_output(self: Pin<&mut Self>, _cx: &mut Context) -> Poll { - Poll::Ready(!self.0) +} +impl ReducerAsync for BoolOrReducerAsync { + type Output = bool; + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { !self.0 }) } } diff --git a/amadeus-core/src/par_sink/collect.rs b/amadeus-core/src/par_sink/collect.rs index 3b48e5d3..815cc6b9 100644 --- a/amadeus-core/src/par_sink/collect.rs +++ b/amadeus-core/src/par_sink/collect.rs @@ -4,13 +4,13 @@ use futures::{pin_mut, ready, Stream, StreamExt}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, HashSet, LinkedList, VecDeque}, hash::{BuildHasher, Hash}, iter, marker::PhantomData, pin::Pin, task::{Context, Poll} + collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, HashSet, LinkedList, VecDeque}, future::Future, hash::{BuildHasher, Hash}, iter, marker::PhantomData, pin::Pin, task::{Context, Poll} }; use super::{ DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend }; -use crate::pool::ProcessSend; +use crate::{pipe::Sink, pool::ProcessSend}; #[derive(new)] #[must_use] @@ -48,18 +48,21 @@ impl, Source, T: FromDistributedStream> Dist } pub trait FromParallelStream: Sized { - type ReduceA: ReducerSend + Clone + Send; - type ReduceC: Reducer::Output, Output = Self>; + type ReduceA: ReducerSend + Clone + Send; + type ReduceC: Reducer<>::Output, Output = Self>; fn reducers() -> (Self::ReduceA, Self::ReduceC); } pub trait FromDistributedStream: Sized { - type ReduceA: ReducerSend + Clone + ProcessSend; - type ReduceB: ReducerProcessSend::Output> + type ReduceA: ReducerSend + Clone + ProcessSend; + type ReduceB: ReducerProcessSend<>::Output> + Clone + ProcessSend; - type ReduceC: Reducer::Output, Output = Self>; + type ReduceC: Reducer< + >::Output>>::Output, + Output = Self, + >; fn reducers() -> (Self::ReduceA, Self::ReduceB, Self::ReduceC); // fn from_dist_stream(dist_stream: I, pool: &Pool) -> Self where T: Serialize + DeserializeOwned + Send + 'static, I: IntoDistributedStream, <::Iter as DistributedStream>::Task: Serialize + DeserializeOwned + Send + 'static; @@ -70,8 +73,7 @@ pub trait FromDistributedStream: Sized { #[serde(bound = "")] pub struct PushReducer(PhantomData (T, A)>); -impl> Reducer for PushReducer { - type Item = A; +impl> Reducer for PushReducer { type Output = T; type Async = PushReducerAsync; @@ -79,13 +81,13 @@ impl> Reducer for PushReducer { PushReducerAsync(Some(Default::default()), PhantomData) } } -impl> ReducerProcessSend for PushReducer +impl> ReducerProcessSend for PushReducer where T: ProcessSend + 'static, { type Output = T; } -impl> ReducerSend for PushReducer +impl> ReducerSend for PushReducer where T: Send + 'static, { @@ -94,14 +96,10 @@ where #[pin_project] pub struct PushReducerAsync(Option, PhantomData A>); -impl> ReducerAsync for PushReducerAsync { - type Item = A; - type Output = T; - +impl> Sink for PushReducerAsync { #[inline] - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, + fn poll_pipe( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, ) -> Poll<()> { let self_ = self.project(); while let Some(item) = ready!(stream.as_mut().poll_next(cx)) { @@ -109,8 +107,12 @@ impl> ReducerAsync for PushReducerAsync { } Poll::Ready(()) } - fn poll_output(mut self: Pin<&mut Self>, _cx: &mut Context) -> Poll { - Poll::Ready(self.0.take().unwrap()) +} +impl> ReducerAsync for PushReducerAsync { + type Output = T; + + fn output<'a>(mut self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { self.0.take().unwrap() }) } } @@ -118,8 +120,7 @@ impl> ReducerAsync for PushReducerAsync { #[educe(Clone, Default)] #[serde(bound = "")] pub struct ExtendReducer(PhantomData (T, A)>); -impl, T: Default + Extend, B> Reducer for ExtendReducer { - type Item = A; +impl, T: Default + Extend, B> Reducer for ExtendReducer { type Output = T; type Async = ExtendReducerAsync; @@ -127,14 +128,14 @@ impl, T: Default + Extend, B> Reducer for ExtendRed ExtendReducerAsync(Some(T::default()), PhantomData) } } -impl, T: Default + Extend, B> ReducerProcessSend +impl, T: Default + Extend, B> ReducerProcessSend for ExtendReducer where T: ProcessSend + 'static, { type Output = T; } -impl, T: Default + Extend, B> ReducerSend for ExtendReducer +impl, T: Default + Extend, B> ReducerSend for ExtendReducer where T: Send + 'static, { @@ -143,14 +144,10 @@ where #[pin_project] pub struct ExtendReducerAsync(Option, PhantomData A>); -impl, T: Extend, B> ReducerAsync for ExtendReducerAsync { - type Item = A; - type Output = T; - +impl, T: Extend, B> Sink for ExtendReducerAsync { #[inline] - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, + fn poll_pipe( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, ) -> Poll<()> { let self_ = self.project(); while let Some(item) = ready!(stream.as_mut().poll_next(cx)) { @@ -158,31 +155,27 @@ impl, T: Extend, B> ReducerAsync for ExtendReducerA } Poll::Ready(()) } - fn poll_output(mut self: Pin<&mut Self>, _cx: &mut Context) -> Poll { - Poll::Ready(self.0.take().unwrap()) +} +impl, T: Extend, B> ReducerAsync for ExtendReducerAsync { + type Output = T; + + fn output<'a>(mut self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { self.0.take().unwrap() }) } } #[derive(Educe, Serialize, Deserialize)] -#[educe(Clone(bound = "R: Clone"))] //, Default(bound = "R: Default"))] +#[educe(Clone(bound = "R: Clone"), Default(bound = "R: Default"))] #[serde( bound(serialize = "R: Serialize"), bound(deserialize = "R: Deserialize<'de>") )] pub struct IntoReducer(R, PhantomData T>); -impl Default for IntoReducer -where - R: Default, -{ - fn default() -> Self { - Self(R::default(), PhantomData) - } -} -impl Reducer for IntoReducer + +impl, T, Item> Reducer for IntoReducer where R::Output: Into, { - type Item = R::Item; type Output = T; type Async = IntoReducerAsync; @@ -194,30 +187,33 @@ where #[pin_project] pub struct IntoReducerAsync(#[pin] R, PhantomData T>); -impl ReducerAsync for IntoReducerAsync +impl, T, Item> Sink for IntoReducerAsync where R::Output: Into, { - type Item = R::Item; - type Output = T; - #[inline] - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, + fn poll_pipe( + self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, ) -> Poll<()> { let stream = stream.map(Into::into); pin_mut!(stream); - self.project().0.poll_forward(cx, stream) + self.project().0.poll_pipe(cx, stream) } - fn poll_output(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - self.project().0.poll_output(cx).map(Into::into) +} +impl, T, Item> ReducerAsync for IntoReducerAsync +where + R::Output: Into, +{ + type Output = T; + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { self.project().0.output().await.into() }) } } #[derive(Clone, Default, Serialize, Deserialize)] pub struct OptionReducer(R); -impl Reducer for OptionReducer { - type Item = Option; +impl, Item> Reducer> for OptionReducer { type Output = Option; type Async = OptionReducerAsync; @@ -225,13 +221,13 @@ impl Reducer for OptionReducer { OptionReducerAsync(Some(self.0.into_async())) } } -impl ReducerProcessSend for OptionReducer +impl, Item> ReducerProcessSend> for OptionReducer where R::Output: ProcessSend + 'static, { type Output = Option; } -impl ReducerSend for OptionReducer +impl, Item> ReducerSend> for OptionReducer where R::Output: Send + 'static, { @@ -241,13 +237,11 @@ where #[pin_project] pub struct OptionReducerAsync(#[pin] Option); -impl ReducerAsync for OptionReducerAsync { - type Item = Option; - type Output = Option; - +impl, Item> Sink> for OptionReducerAsync { #[inline] - fn poll_forward( - self: Pin<&mut Self>, _cx: &mut Context, _stream: Pin<&mut impl Stream>, + fn poll_pipe( + self: Pin<&mut Self>, _cx: &mut Context, + _stream: Pin<&mut impl Stream>>, ) -> Poll<()> { todo!("Tracking at https://github.com/constellation-rs/amadeus/projects/3#card-40276549"); // let self_ = self.project(); @@ -259,32 +253,24 @@ impl ReducerAsync for OptionReducerAsync { // } // self_.0.is_some() } - fn poll_output(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - self.project() - .0 - .as_pin_mut() - .map(|r| r.poll_output(cx).map(Some)) - .unwrap_or(Poll::Ready(None)) +} +impl, Item> ReducerAsync> for OptionReducerAsync { + type Output = Option; + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { Some(self.project().0.as_pin_mut()?.output().await) }) } } #[derive(Educe, Serialize, Deserialize)] -#[educe(Clone(bound = "R: Clone"))] // , Default(bound = "R: Default"))] +#[educe(Clone(bound = "R: Clone"), Default(bound = "R: Default"))] #[serde( bound(serialize = "R: Serialize"), bound(deserialize = "R: Deserialize<'de>") )] pub struct ResultReducer(R, PhantomData E>); -impl Default for ResultReducer -where - R: Default, -{ - fn default() -> Self { - Self(R::default(), PhantomData) - } -} -impl Reducer for ResultReducer { - type Item = Result; + +impl, E, Item> Reducer> for ResultReducer { type Output = Result; type Async = ResultReducerAsync; @@ -292,14 +278,14 @@ impl Reducer for ResultReducer { ResultReducerAsync::Ok(self.0.into_async()) } } -impl ReducerProcessSend for ResultReducer +impl, E, Item> ReducerProcessSend> for ResultReducer where R::Output: ProcessSend + 'static, E: ProcessSend + 'static, { type Output = Result; } -impl ReducerSend for ResultReducer +impl, E, Item> ReducerSend> for ResultReducer where R::Output: Send + 'static, E: Send + 'static, @@ -312,13 +298,11 @@ pub enum ResultReducerAsync { Ok(#[pin] R), Err(Option), } -impl ReducerAsync for ResultReducerAsync { - type Item = Result; - type Output = Result; - +impl, E, Item> Sink> for ResultReducerAsync { #[inline] - fn poll_forward( - self: Pin<&mut Self>, _cx: &mut Context, _stream: Pin<&mut impl Stream>, + fn poll_pipe( + self: Pin<&mut Self>, _cx: &mut Context, + _stream: Pin<&mut impl Stream>>, ) -> Poll<()> { todo!("Tracking at https://github.com/constellation-rs/amadeus/projects/3#card-40276549"); // let self_ = self.project(); @@ -331,11 +315,17 @@ impl ReducerAsync for ResultReducerAsync { // } // self_.0.is_ok() } - fn poll_output(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - match self.project() { - ResultReducerAsyncProj::Ok(a) => a.poll_output(cx).map(Ok), - ResultReducerAsyncProj::Err(b) => Poll::Ready(Err(b.take().unwrap())), - } +} +impl, E, Item> ReducerAsync> for ResultReducerAsync { + type Output = Result; + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { + match self.project() { + ResultReducerAsyncProj::Ok(a) => Ok(a.output().await), + ResultReducerAsyncProj::Err(b) => Err(b.take().unwrap()), + } + }) } } diff --git a/amadeus-core/src/par_sink/combiner.rs b/amadeus-core/src/par_sink/combiner.rs index d4b1d7d3..502823ab 100644 --- a/amadeus-core/src/par_sink/combiner.rs +++ b/amadeus-core/src/par_sink/combiner.rs @@ -4,10 +4,10 @@ mod macros { #[macro_export] macro_rules! combiner_par_sink { ($combiner:ty, $self:ident, $init:expr) => { - type Output = ::Output; + type Output = >::Output>>::Output; type Pipe = I; type ReduceA = FolderSyncReducer; - type ReduceC = FolderSyncReducer<::Output, $combiner>; + type ReduceC = FolderSyncReducer<>::Output, $combiner>; fn reducers($self) -> (I, Self::ReduceA, Self::ReduceC) { let init = $init; @@ -22,11 +22,11 @@ mod macros { #[macro_export] macro_rules! combiner_dist_sink { ($combiner:ty, $self:ident, $init:expr) => { - type Output = ::Output; + type Output = >::Output>>::Output>>::Output; type Pipe = I; type ReduceA = FolderSyncReducer; - type ReduceB = FolderSyncReducer<::Output, $combiner>; - type ReduceC = FolderSyncReducer<::Output, $combiner>; + type ReduceB = FolderSyncReducer<>::Output, $combiner>; + type ReduceC = FolderSyncReducer<>::Output>>::Output, $combiner>; fn reducers($self) -> (I, Self::ReduceA, Self::ReduceB, Self::ReduceC) { let init = $init; diff --git a/amadeus-core/src/par_sink/folder.rs b/amadeus-core/src/par_sink/folder.rs index bf34d3a1..e45d975d 100644 --- a/amadeus-core/src/par_sink/folder.rs +++ b/amadeus-core/src/par_sink/folder.rs @@ -6,20 +6,20 @@ use futures::{ready, Stream}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - marker::PhantomData, pin::Pin, task::{Context, Poll} + future::Future, marker::PhantomData, pin::Pin, task::{Context, Poll} }; use super::{Reducer, ReducerAsync, ReducerProcessSend, ReducerSend}; -use crate::pool::ProcessSend; +use crate::{pipe::Sink, pool::ProcessSend}; mod macros { #[macro_export] macro_rules! folder_par_sink { ($folder_a:ty, $folder_b:ty, $self:ident, $init_a:expr, $init_b:expr) => { - type Output = ::Output; + type Output = >::Output>>::Output; type Pipe = I; type ReduceA = FolderSyncReducer; - type ReduceC = FolderSyncReducer<::Output, $folder_b>; + type ReduceC = FolderSyncReducer<>::Output, $folder_b>; fn reducers($self) -> (I, Self::ReduceA, Self::ReduceC) { let init_a = $init_a; @@ -35,11 +35,11 @@ mod macros { #[macro_export] macro_rules! folder_dist_sink { ($folder_a:ty, $folder_b:ty, $self:ident, $init_a:expr, $init_b:expr) => { - type Output = ::Output; + type Output = >::Output>>::Output>>::Output; type Pipe = I; type ReduceA = FolderSyncReducer; - type ReduceB = FolderSyncReducer<::Output, $folder_b>; - type ReduceC = FolderSyncReducer<::Output, $folder_b>; + type ReduceB = FolderSyncReducer<>::Output, $folder_b>; + type ReduceC = FolderSyncReducer<>::Output>>::Output, $folder_b>; fn reducers($self) -> (I, Self::ReduceA, Self::ReduceB, Self::ReduceC) { let init_a = $init_a; @@ -77,11 +77,10 @@ pub struct FolderSyncReducer { marker: PhantomData A>, } -impl Reducer for FolderSyncReducer +impl Reducer for FolderSyncReducer where C: FolderSync, { - type Item = A; type Output = C::Output; type Async = FolderSyncReducerAsync; @@ -93,14 +92,14 @@ where } } } -impl ReducerProcessSend for FolderSyncReducer +impl ReducerProcessSend for FolderSyncReducer where C: FolderSync, C::Output: ProcessSend + 'static, { type Output = C::Output; } -impl ReducerSend for FolderSyncReducer +impl ReducerSend for FolderSyncReducer where C: FolderSync, C::Output: Send + 'static, @@ -114,17 +113,13 @@ pub struct FolderSyncReducerAsync { folder: C, marker: PhantomData A>, } -impl ReducerAsync for FolderSyncReducerAsync +impl Sink for FolderSyncReducerAsync where C: FolderSync, { - type Item = A; - type Output = C::Output; - #[inline(always)] - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, + fn poll_pipe( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, ) -> Poll<()> { let self_ = self.project(); let folder = self_.folder; @@ -133,7 +128,14 @@ where } Poll::Ready(()) } - fn poll_output(self: Pin<&mut Self>, _cx: &mut Context) -> Poll { - Poll::Ready(self.project().state.take().unwrap()) +} +impl ReducerAsync for FolderSyncReducerAsync +where + C: FolderSync, +{ + type Output = C::Output; + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { self.project().state.take().unwrap() }) } } diff --git a/amadeus-core/src/par_sink/for_each.rs b/amadeus-core/src/par_sink/for_each.rs index 381243f0..302972d4 100644 --- a/amadeus-core/src/par_sink/for_each.rs +++ b/amadeus-core/src/par_sink/for_each.rs @@ -4,13 +4,13 @@ use futures::{ready, Stream}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - marker::PhantomData, pin::Pin, task::{Context, Poll} + future::Future, marker::PhantomData, pin::Pin, task::{Context, Poll} }; use super::{ DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PushReducer, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend }; -use crate::pool::ProcessSend; +use crate::{pipe::Sink, pool::ProcessSend}; #[derive(new)] #[must_use] @@ -65,11 +65,10 @@ where )] pub struct ForEachReducer(F, PhantomData A>); -impl Reducer for ForEachReducer +impl Reducer for ForEachReducer where F: FnMut(A) + Clone, { - type Item = A; type Output = (); type Async = Self; @@ -77,30 +76,26 @@ where self } } -impl ReducerProcessSend for ForEachReducer +impl ReducerProcessSend for ForEachReducer where F: FnMut(A) + Clone, { type Output = (); } -impl ReducerSend for ForEachReducer +impl ReducerSend for ForEachReducer where F: FnMut(A) + Clone, { type Output = (); } -impl ReducerAsync for ForEachReducer +impl Sink for ForEachReducer where F: FnMut(A) + Clone, { - type Item = A; - type Output = (); - #[inline(always)] - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, + fn poll_pipe( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, ) -> Poll<()> { let self_ = self.project(); while let Some(item) = ready!(stream.as_mut().poll_next(cx)) { @@ -108,7 +103,14 @@ where } Poll::Ready(()) } - fn poll_output(self: Pin<&mut Self>, _cx: &mut Context) -> Poll { - Poll::Ready(()) +} +impl ReducerAsync for ForEachReducer +where + F: FnMut(A) + Clone, +{ + type Output = (); + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move {}) } } diff --git a/amadeus-core/src/par_sink/fork.rs b/amadeus-core/src/par_sink/fork.rs new file mode 100644 index 00000000..cd5b80cc --- /dev/null +++ b/amadeus-core/src/par_sink/fork.rs @@ -0,0 +1,301 @@ +use derive_new::new; +use futures::{pin_mut, stream, Stream, StreamExt as _}; +use pin_project::pin_project; +use serde::{Deserialize, Serialize}; +use std::{ + marker::PhantomData, pin::Pin, task::{Context, Poll} +}; +use sum::Sum2; + +use super::{ + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask, ReduceA2, ReduceC2 +}; +use crate::{ + par_stream::{ParallelStream, StreamTask}, pipe::Pipe, util::transmute +}; + +#[derive(new)] +#[must_use] +pub struct Fork { + a: A, + b: B, + c: C, + marker: PhantomData RefAItem>, +} + +impl_par_dist! { + impl ParallelStream for Fork + where + A: ParallelStream, + B: ParallelPipe, + C: ParallelPipe, + RefAItem: 'static, + { + type Item = Sum2; + type Task = JoinTask; + + fn size_hint(&self) -> (usize, Option) { + self.a.size_hint() + } + fn next_task(&mut self) -> Option { + self.a + .next_task() + .map(|task| JoinTask{stream:task, pipe:self.b.task(), pipe_ref:self.c.task(), marker:PhantomData}) + } + } + impl ParallelPipe for Fork + where + A: ParallelPipe, + B: ParallelPipe, + C: ParallelPipe, + RefAItem: 'static, + { + type Item = Sum2; + type Task = JoinTask; + + fn task(&self) -> Self::Task { + let stream = self.a.task(); + let pipe = self.b.task(); + let pipe_ref = self.c.task(); + JoinTask { stream, pipe, pipe_ref, marker: PhantomData } + } + } +} + +impl ParallelSink for Fork +where + A: ParallelPipe, + B: ParallelSink, + C: ParallelSink, + RefAItem: 'static, +{ + type Output = (B::Output, C::Output); + type Pipe = Fork; + type ReduceA = ReduceA2; + type ReduceC = ReduceC2; + + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { + let (iterator_a, reducer_a_a, reducer_a_c) = self.b.reducers(); + let (iterator_b, reducer_b_a, reducer_b_c) = self.c.reducers(); + ( + Fork::new(self.a, iterator_a, iterator_b), + ReduceA2::new(reducer_a_a, reducer_b_a), + ReduceC2::new(reducer_a_c, reducer_b_c), + ) + } +} +impl DistributedSink for Fork +where + A: DistributedPipe, + B: DistributedSink, + C: DistributedSink, + RefAItem: 'static, +{ + type Output = (B::Output, C::Output); + type Pipe = Fork; + type ReduceA = ReduceA2; + type ReduceB = ReduceC2; + type ReduceC = ReduceC2; + + fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { + let (iterator_a, reducer_a_a, reducer_a_b, reducer_a_c) = self.b.reducers(); + let (iterator_b, reducer_b_a, reducer_b_b, reducer_b_c) = self.c.reducers(); + ( + Fork::new(self.a, iterator_a, iterator_b), + ReduceA2::new(reducer_a_a, reducer_b_a), + ReduceC2::new(reducer_a_b, reducer_b_b), + ReduceC2::new(reducer_a_c, reducer_b_c), + ) + } +} + +#[derive(Serialize, Deserialize)] +#[serde( + bound(serialize = "A: Serialize, B: Serialize, C: Serialize"), + bound(deserialize = "A: Deserialize<'de>, B: Deserialize<'de>, C: Deserialize<'de>") +)] +pub struct JoinTask { + stream: A, + pipe: B, + pipe_ref: C, + marker: PhantomData RefAItem>, +} +impl StreamTask for JoinTask +where + A: StreamTask, + B: PipeTask, + C: PipeTask, +{ + type Item = Sum2; + type Async = JoinStreamTaskAsync; + + fn into_async(self) -> Self::Async { + JoinStreamTaskAsync { + stream: self.stream.into_async(), + pipe: self.pipe.into_async(), + pipe_ref: self.pipe_ref.into_async(), + ref_given: false, + pending: None, + marker: PhantomData, + } + } +} +impl PipeTask for JoinTask +where + A: PipeTask, + B: PipeTask, + C: PipeTask, +{ + type Item = Sum2; + type Async = JoinStreamTaskAsync; + + fn into_async(self) -> Self::Async { + JoinStreamTaskAsync { + stream: self.stream.into_async(), + pipe: self.pipe.into_async(), + pipe_ref: self.pipe_ref.into_async(), + ref_given: false, + pending: None, + marker: PhantomData, + } + } +} + +#[pin_project] +pub struct JoinStreamTaskAsync { + #[pin] + stream: A, + #[pin] + pipe: B, + #[pin] + pipe_ref: C, + ref_given: bool, + pending: Option>, + marker: PhantomData RefAItem>, +} + +impl Stream for JoinStreamTaskAsync +where + A: Stream, + B: crate::pipe::Pipe, + C: crate::pipe::Pipe, +{ + type Item = Sum2; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let mut self_ = self.project(); + loop { + if self_.pending.is_none() { + *self_.pending = match self_.stream.as_mut().poll_next(cx) { + Poll::Ready(x) => Some(x), + Poll::Pending => None, + }; + } + let ref_given = &mut *self_.ref_given; + let pending = &mut *self_.pending; + let mut progress = false; + { + let stream = stream::poll_fn(|_cx| match pending { + Some(x) if !*ref_given => { + *ref_given = true; + progress = true; + Poll::Ready(unsafe { transmute(x.as_ref()) }) + } + _ => Poll::Pending, + }) + .fuse(); + pin_mut!(stream); + if let Poll::Ready(item) = self_.pipe_ref.as_mut().poll_next(cx, stream) { + return Poll::Ready(item.map(|item| Sum2::B(unsafe { transmute(item) }))); + } + } + { + let stream = stream::poll_fn(|_cx| { + if !*ref_given { + return Poll::Pending; + } + match pending.take() { + Some(x) => { + *ref_given = false; + progress = true; + Poll::Ready(x) + } + None => Poll::Pending, + } + }) + .fuse(); + pin_mut!(stream); + if let Poll::Ready(item) = self_.pipe.as_mut().poll_next(cx, stream) { + return Poll::Ready(item.map(Sum2::A)); + } + } + if !progress { + break Poll::Pending; + } + } + } +} + +impl Pipe for JoinStreamTaskAsync +where + A: Pipe, + B: crate::pipe::Pipe, + C: crate::pipe::Pipe, +{ + type Item = Sum2; + + fn poll_next( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, + ) -> Poll> { + let mut self_ = self.project(); + loop { + if self_.pending.is_none() { + *self_.pending = match self_.stream.as_mut().poll_next(cx, stream.as_mut()) { + Poll::Ready(x) => Some(x), + Poll::Pending => None, + }; + } + let ref_given = &mut *self_.ref_given; + let pending = &mut *self_.pending; + let mut progress = false; + { + let stream = stream::poll_fn(|_cx| match pending { + Some(x) if !*ref_given => { + *ref_given = true; + progress = true; + Poll::Ready(unsafe { transmute(x.as_ref()) }) + } + _ => Poll::Pending, + }) + .fuse(); + pin_mut!(stream); + if let Poll::Ready(item) = self_.pipe_ref.as_mut().poll_next(cx, stream) { + return Poll::Ready(item.map(|item| Sum2::B(unsafe { transmute(item) }))); + } + } + { + let stream = stream::poll_fn(|_cx| { + if !*ref_given { + return Poll::Pending; + } + match pending.take() { + Some(x) => { + *ref_given = false; + progress = true; + Poll::Ready(x) + } + None => Poll::Pending, + } + }) + .fuse(); + pin_mut!(stream); + if let Poll::Ready(item) = self_.pipe.as_mut().poll_next(cx, stream) { + return Poll::Ready(item.map(Sum2::A)); + } + } + if !progress { + break Poll::Pending; + } + } + } +} diff --git a/amadeus-core/src/par_sink/group_by.rs b/amadeus-core/src/par_sink/group_by.rs index b2fc625e..999b0b29 100644 --- a/amadeus-core/src/par_sink/group_by.rs +++ b/amadeus-core/src/par_sink/group_by.rs @@ -2,17 +2,19 @@ use derive_new::new; use educe::Educe; -use futures::{pin_mut, ready, stream, Stream, StreamExt}; +use futures::{future::join_all, pin_mut, ready, stream, Stream, StreamExt}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - collections::HashMap, hash::Hash, marker::PhantomData, mem, pin::Pin, task::{Context, Poll} + collections::HashMap, future::Future, hash::Hash, marker::PhantomData, mem, pin::Pin, task::{Context, Poll} }; use super::{ - DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend +}; +use crate::{ + pipe::{Pipe, Sink, StreamExt as _}, pool::ProcessSend }; -use crate::{pool::ProcessSend, util::type_coerce}; #[derive(new)] #[must_use] @@ -25,19 +27,27 @@ impl, B: ParallelSink, Source, T, U> P for GroupBy where T: Eq + Hash + Send + 'static, - B::Pipe: Clone + Send + 'static, + >::Task: Clone + Send + 'static, B::ReduceA: Clone + Send + 'static, B::ReduceC: Clone, B::Output: Send + 'static, { type Output = HashMap; type Pipe = A; - type ReduceA = GroupByReducerA; - type ReduceC = GroupByReducerB::Output>; + type ReduceA = GroupByReducerA<>::Task, B::ReduceA, T, U>; + type ReduceC = GroupByReducerB< + B::ReduceC, + T, + >::Item>>::Output, + >; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceC) { let (a, b, c) = self.b.reducers(); - (self.a, GroupByReducerA::new(a, b), GroupByReducerB::new(c)) + ( + self.a, + GroupByReducerA::new(a.task(), b), + GroupByReducerB::new(c), + ) } } @@ -45,7 +55,7 @@ impl, B: DistributedSink, Source, T DistributedSink for GroupBy where T: Eq + Hash + ProcessSend + 'static, - B::Pipe: Clone + ProcessSend + 'static, + >::Task: Clone + ProcessSend + 'static, B::ReduceA: Clone + ProcessSend + 'static, B::ReduceB: Clone, B::ReduceC: Clone, @@ -53,15 +63,25 @@ where { type Output = HashMap; type Pipe = A; - type ReduceA = GroupByReducerA; - type ReduceB = GroupByReducerB::Output>; - type ReduceC = GroupByReducerB::Output>; + type ReduceA = GroupByReducerA<>::Task, B::ReduceA, T, U>; + type ReduceB = GroupByReducerB< + B::ReduceB, + T, + >::Item>>::Output, + >; + type ReduceC = GroupByReducerB< + B::ReduceC, + T, + >::Item>>::Output, + >>::Output, + >; fn reducers(self) -> (Self::Pipe, Self::ReduceA, Self::ReduceB, Self::ReduceC) { let (a, b, c, d) = self.b.reducers(); ( self.a, - GroupByReducerA::new(a, b), + GroupByReducerA::new(a.task(), b), GroupByReducerB::new(c), GroupByReducerB::new(d), ) @@ -76,30 +96,32 @@ where )] pub struct GroupByReducerA(P, R, PhantomData (R, T, U)>); -impl Reducer for GroupByReducerA +impl Reducer<(T, U)> for GroupByReducerA where - R: Reducer + Clone, + P: PipeTask, + R: Reducer + Clone, T: Eq + Hash, { - type Item = (T, U); type Output = HashMap; - type Async = GroupByReducerAAsync; + type Async = GroupByReducerAAsync; fn into_async(self) -> Self::Async { - GroupByReducerAAsync::new(self.0, self.1) + GroupByReducerAAsync::new(self.0.into_async(), self.1) } } -impl ReducerProcessSend for GroupByReducerA +impl ReducerProcessSend<(T, U)> for GroupByReducerA where - R: Reducer + Clone, + P: PipeTask, + R: Reducer + Clone, T: Eq + Hash + ProcessSend + 'static, R::Output: ProcessSend + 'static, { type Output = HashMap; } -impl ReducerSend for GroupByReducerA +impl ReducerSend<(T, U)> for GroupByReducerA where - R: Reducer + Clone, + P: PipeTask, + R: Reducer + Clone, T: Eq + Hash + Send + 'static, R::Output: Send + 'static, { @@ -109,6 +131,7 @@ where #[pin_project] #[derive(new)] pub struct GroupByReducerAAsync { + #[pin] pipe: P, factory: R, #[new(default)] @@ -118,20 +141,17 @@ pub struct GroupByReducerAAsync { marker: PhantomData (R, U)>, } -impl ReducerAsync for GroupByReducerAAsync +impl Sink<(T, U)> for GroupByReducerAAsync where - R: Reducer + Clone, + P: Pipe, + R: Reducer + Clone, T: Eq + Hash, { - type Item = (T, U); - type Output = HashMap; - #[inline(always)] - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, + fn poll_pipe( + self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, ) -> Poll<()> { - let self_ = self.project(); + let mut self_ = self.project(); loop { if !self_.pending.is_some() { *self_.pending = Some(ready!(stream.as_mut().poll_next(cx)).map(|(t, u)| { @@ -146,16 +166,17 @@ where if let Some((t, u, r)) = self_.pending.as_mut().unwrap() { let stream = stream::poll_fn(|_cx| { if let Some(u) = u.take() { - Poll::Ready(Some(type_coerce(u))) + Poll::Ready(Some(u)) } else { Poll::Pending } }) - .fuse(); + .fuse() + .pipe(self_.pipe.as_mut()); pin_mut!(stream); let map = &mut *self_.map; let r_ = r.as_mut().unwrap_or_else(|| map.get_mut(&t).unwrap()); - let _ = r_.as_mut().poll_forward(cx, stream); + let _ = r_.as_mut().poll_pipe(cx, stream); if u.is_some() { return Poll::Pending; } @@ -167,29 +188,33 @@ where for r in self_.map.values_mut() { let stream = stream::empty(); pin_mut!(stream); - ready!(r.as_mut().poll_forward(cx, stream)); + ready!(r.as_mut().poll_pipe(cx, stream)); } return Poll::Ready(()); } } } - fn poll_output(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let self_ = self.project(); - Poll::Ready( - mem::take(self_.map) - .into_iter() - .map(|(k, mut v)| { - ( - k, - if let Poll::Ready(x) = v.as_mut().poll_output(cx) { - x - } else { - todo!() - }, - ) - }) - .collect(), - ) +} +impl ReducerAsync<(T, U)> for GroupByReducerAAsync +where + P: Pipe, + R: Reducer + Clone, + T: Eq + Hash, +{ + type Output = HashMap; + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { + let self_ = self.project(); + join_all( + mem::take(self_.map) + .into_iter() + .map(|(k, mut v)| async move { (k, v.as_mut().output().await) }), + ) + .await + .into_iter() + .collect() + }) } } @@ -201,12 +226,11 @@ where )] pub struct GroupByReducerB(R, PhantomData (T, U)>); -impl Reducer for GroupByReducerB +impl Reducer> for GroupByReducerB where - R: Reducer + Clone, + R: Reducer + Clone, T: Eq + Hash, { - type Item = HashMap; type Output = HashMap; type Async = GroupByReducerBAsync; @@ -214,17 +238,17 @@ where GroupByReducerBAsync::new(self.0) } } -impl ReducerProcessSend for GroupByReducerB +impl ReducerProcessSend> for GroupByReducerB where - R: Reducer + Clone, + R: Reducer + Clone, T: Eq + Hash + ProcessSend + 'static, R::Output: ProcessSend + 'static, { type Output = HashMap; } -impl ReducerSend for GroupByReducerB +impl ReducerSend> for GroupByReducerB where - R: Reducer + Clone, + R: Reducer + Clone, T: Eq + Hash + Send + 'static, R::Output: Send + 'static, { @@ -242,19 +266,15 @@ pub struct GroupByReducerBAsync { marker: PhantomData (T, U)>, } -impl ReducerAsync for GroupByReducerBAsync +impl Sink> for GroupByReducerBAsync where - R: Reducer + Clone, - R::Async: ReducerAsync, + R: Reducer + Clone, T: Eq + Hash, { - type Item = HashMap; - type Output = HashMap; - #[inline(always)] - fn poll_forward( + fn poll_pipe( self: Pin<&mut Self>, cx: &mut Context, - mut stream: Pin<&mut impl Stream>, + mut stream: Pin<&mut impl Stream>>, ) -> Poll<()> { let self_ = self.project(); loop { @@ -274,11 +294,11 @@ where } let pending = self_.pending.as_mut().unwrap(); if let Some(pending) = pending { - while let Some((k, (v, mut r))) = remove_arbitrary(pending) { + while let Some((k, (v, mut r))) = pop(pending) { let mut v = Some(v); let stream = stream::poll_fn(|_cx| { if let Some(v) = v.take() { - Poll::Ready(Some(type_coerce::(v))) + Poll::Ready(Some(v)) } else { Poll::Pending } @@ -286,7 +306,7 @@ where pin_mut!(stream); let map = &mut *self_.map; let r_ = r.as_mut().unwrap_or_else(|| map.get_mut(&k).unwrap()); - let _ = r_.as_mut().poll_forward(cx, stream); + let _ = r_.as_mut().poll_pipe(cx, stream); if let Some(v) = v { let _ = pending.insert(k, (v, r)); return Poll::Pending; @@ -300,34 +320,38 @@ where for r in self_.map.values_mut() { let stream = stream::empty(); pin_mut!(stream); - ready!(r.as_mut().poll_forward(cx, stream)); + ready!(r.as_mut().poll_pipe(cx, stream)); } return Poll::Ready(()); } } } - fn poll_output(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let self_ = self.project(); - Poll::Ready( - mem::take(self_.map) - .into_iter() - .map(|(k, mut v)| { - ( - k, - if let Poll::Ready(x) = ReducerAsync::poll_output(v.as_mut(), cx) { - x - } else { - todo!() - }, - ) - }) - .collect(), - ) +} +impl ReducerAsync> for GroupByReducerBAsync +where + R: Reducer + Clone, + T: Eq + Hash, +{ + type Output = HashMap; + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { + let self_ = self.project(); + join_all( + mem::take(self_.map) + .into_iter() + .map(|(k, mut v)| async move { (k, v.as_mut().output().await) }), + ) + .await + .into_iter() + .collect() + }) } } +// https://github.com/rust-lang/rfcs/issues/1800#issuecomment-653757340 #[allow(clippy::unnecessary_filter_map)] -fn remove_arbitrary(map: &mut HashMap) -> Option<(K, V)> +fn pop(map: &mut HashMap) -> Option<(K, V)> where K: Eq + Hash, { diff --git a/amadeus-core/src/par_sink/pipe.rs b/amadeus-core/src/par_sink/pipe.rs index f5a0a301..d0bf1a41 100644 --- a/amadeus-core/src/par_sink/pipe.rs +++ b/amadeus-core/src/par_sink/pipe.rs @@ -1,15 +1,11 @@ use derive_new::new; -use futures::{pin_mut, Stream}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; -use std::{ - marker::PhantomData, pin::Pin, task::{Context, Poll} -}; -use super::{ - DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask, PipeTaskAsync +use super::{DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask}; +use crate::{ + par_stream::{ParallelStream, StreamTask}, pipe::{Pipe as _, PipePipe, StreamExt, StreamPipe} }; -use crate::sink::Sink; #[derive(new)] #[must_use] @@ -19,6 +15,20 @@ pub struct Pipe { } impl_par_dist! { + impl> ParallelStream for Pipe { + type Item = B::Item; + type Task = JoinTask; + + fn next_task(&mut self) -> Option { + self.a.next_task().map(|a| { + let b = self.b.task(); + JoinTask { a, b } + }) + } + fn size_hint(&self) -> (usize, Option) { + self.a.size_hint() + } + } impl, B: ParallelPipe, Source> ParallelPipe for Pipe { type Item = B::Item; type Task = JoinTask; @@ -68,46 +78,20 @@ pub struct JoinTask { b: B, } -impl, B: PipeTask, Source> PipeTask for JoinTask { +impl> StreamTask for JoinTask { type Item = B::Item; - type Async = JoinTask; + type Async = StreamPipe; + fn into_async(self) -> Self::Async { - JoinTask { - a: self.a.into_async(), - b: self.b.into_async(), - } + self.a.into_async().pipe(self.b.into_async()) } } -impl, B: PipeTaskAsync, Source> PipeTaskAsync - for JoinTask -{ +impl, B: PipeTask, Source> PipeTask for JoinTask { type Item = B::Item; + type Async = PipePipe; - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>, - sink: Pin<&mut impl Sink>, - ) -> Poll<()> { - #[pin_project] - struct Proxy<'a, I, B, Item>(#[pin] I, Pin<&'a mut B>, PhantomData Item>); - impl<'a, I, B, Item> Sink for Proxy<'a, I, B, Item> - where - I: Sink, - B: PipeTaskAsync, - { - type Item = Item; - - fn poll_forward( - self: Pin<&mut Self>, cx: &mut Context, - stream: Pin<&mut impl Stream>, - ) -> Poll<()> { - let self_ = self.project(); - self_.1.as_mut().poll_run(cx, stream, self_.0) - } - } - let self_ = self.project(); - let sink = Proxy(sink, self_.b, PhantomData); - pin_mut!(sink); - self_.a.poll_run(cx, stream, sink) + fn into_async(self) -> Self::Async { + self.a.into_async().pipe(self.b.into_async()) } } diff --git a/amadeus-core/src/par_sink/tuple.rs b/amadeus-core/src/par_sink/tuple.rs index d716ed42..8583399e 100644 --- a/amadeus-core/src/par_sink/tuple.rs +++ b/amadeus-core/src/par_sink/tuple.rs @@ -1,19 +1,23 @@ #![allow(non_snake_case, clippy::type_complexity, irrefutable_let_patterns, clippy::new_without_default, unused_mut, unreachable_code, clippy::too_many_arguments)] use derive_new::new; -use futures::{pin_mut, ready, stream, Stream, StreamExt}; +use futures::{ + future::{ + join as Join2, join3 as Join3, join4 as Join4, join5 as Join5, maybe_done, FusedFuture, MaybeDone + }, pin_mut, ready, stream, Stream, StreamExt +}; use pin_project::pin_project; use serde::{Deserialize, Serialize}; use std::{ - pin::Pin, task::{Context, Poll} + future::Future, pin::Pin, task::{Context, Poll} }; use sum::*; use super::{ - DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask, PipeTaskAsync, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend + DistributedPipe, DistributedSink, ParallelPipe, ParallelSink, PipeTask, Reducer, ReducerAsync, ReducerProcessSend, ReducerSend }; use crate::{ - pool::ProcessSend, sink::{Sink, SinkMap} + pipe::{Pipe, Sink}, pool::ProcessSend }; fn substream<'a, 'b, S, F1, F2, O>( @@ -40,7 +44,7 @@ where } macro_rules! impl_tuple { - ($reducea:ident $reduceaasync:ident $reduceb:ident $reducebasync:ident $async:ident $enum:ident $($copy:ident)? : $($i:ident $r:ident $o:ident $c:ident $iterator:ident $reducera:ident $reducerb:ident $num:tt $t:ident $($copyb:ident)? , $comma:tt)*) => ( + ($reducea:ident $reduceaasync:ident $reduceb:ident $reducebasync:ident $async:ident $enum:ident $join:ident $($copy:ident)? : $($num:tt $t:ident $s:ident $i:ident $r:ident $o:ident $c:ident $iterator:ident $reducera:ident $reducerb:ident $($copyb:ident)? , $comma:tt)*) => ( impl< Source, $($r: ParallelSink,)* @@ -132,60 +136,47 @@ macro_rules! impl_tuple { } #[allow(unused_variables)] - impl,)*> PipeTaskAsync for $async + impl,)*> Pipe for $async where Source: $($copy)*, { type Item = $enum<$($c::Item,)*>; #[allow(non_snake_case)] - fn poll_run( + fn poll_next( self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>, - mut sink: Pin<&mut impl Sink>, - ) -> Poll<()> { + ) -> Poll> { let mut self_ = self.project(); // buffer, copy to each - let mut ready = ($(false $comma)*); loop { - let mut progress = false; if self_.pending.is_none() { *self_.pending = match stream.as_mut().poll_next(cx) { Poll::Ready(x) => Some(x), Poll::Pending => None}; } - $( - if !ready.$num { - let sink_ = SinkMap::new(sink.as_mut(), $enum::$t); - pin_mut!(sink_); - let pending = &mut self_.pending; - let given = &mut self_.given.$num; - let stream_ = stream::poll_fn(|cx| { - if !*given && pending.is_some() { - *given = true; - progress = true; - $( - return Poll::Ready(*pending.as_ref().unwrap()); - let $copyb = (); - )? - Poll::Ready(pending.take().unwrap().take()) - } else { - Poll::Pending - } - }).fuse(); - pin_mut!(stream_); - ready.$num = self_.$t.as_mut().poll_run(cx, stream_, sink_).is_ready(); - if ready.$num { - self_.given.$num = true; + $({ + let pending = &mut self_.pending; + let given = &mut self_.given.$num; + let stream_ = stream::poll_fn(|cx| { + if !*given && pending.is_some() { + *given = true; + $( + return Poll::Ready(*pending.as_ref().unwrap()); + let $copyb = (); + )? + Poll::Ready(pending.take().unwrap().take()) + } else { + Poll::Pending } + }).fuse(); + pin_mut!(stream_); + if let Poll::Ready(item) = self_.$t.as_mut().poll_next(cx, stream_) { + return Poll::Ready(item.map($enum::$t)); } - )* - if $(ready.$num &&)* true { - break Poll::Ready(()); - } - if !progress { - break Poll::Pending; - } + })* if $(self_.given.$num &&)* true { $(self_.given.$num = false;)* *self_.pending = None; + } else { + break Poll::Pending; } } } @@ -195,10 +186,9 @@ macro_rules! impl_tuple { pub struct $reducea<$($t,)*> { $($t: $t,)* } - impl<$($t: Reducer,)*> Reducer for $reducea<$($t,)*> { - type Item = $enum<$($t::Item,)*>; + impl<$($t: Reducer<$s>,)* $($s,)*> Reducer<$enum<$($s,)*>> for $reducea<$($t,)*> { type Output = ($($t::Output,)*); - type Async = $reduceaasync<$($t::Async,)*>; + type Async = $reduceaasync<$($t::Async,)* $($s,)*>; fn into_async(self) -> Self::Async { $reduceaasync{ @@ -208,18 +198,21 @@ macro_rules! impl_tuple { } } } + impl<$($t: Reducer<$s>,)* $($s,)*> ReducerProcessSend<$enum<$($s,)*>> for $reducea<$($t,)*> where $($t::Output: ProcessSend + 'static,)* { + type Output = ($($t::Output,)*); + } + impl<$($t: Reducer<$s>,)* $($s,)*> ReducerSend<$enum<$($s,)*>> for $reducea<$($t,)*> where $($t::Output: Send + 'static,)* { + type Output = ($($t::Output,)*); + } #[pin_project] - pub struct $reduceaasync<$($t,)*> where $($t: ReducerAsync,)* { + pub struct $reduceaasync<$($t,)* $($s,)*> where $($t: ReducerAsync<$s>,)* { $(#[pin] $t: $t,)* - peeked: Option<$enum<$($t::Item,)*>>, + peeked: Option<$enum<$($s,)*>>, output: ($(Option<$t::Output>,)*), } #[allow(unused_variables)] - impl<$($t: ReducerAsync,)*> ReducerAsync for $reduceaasync<$($t,)*> { - type Item = $enum<$($t::Item,)*>; - type Output = ($($t::Output,)*); - - fn poll_forward(self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>) -> Poll<()> { + impl<$($t: ReducerAsync<$s>,)* $($s,)*> Sink<$enum<$($s,)*>> for $reduceaasync<$($t,)* $($s,)*> { + fn poll_pipe(self: Pin<&mut Self>, cx: &mut Context, mut stream: Pin<&mut impl Stream>>) -> Poll<()> { let mut self_ = self.project(); let mut ready = ($(false $comma)*); loop { @@ -229,7 +222,7 @@ macro_rules! impl_tuple { pin_mut!(stream); let stream_ = substream(stream, |item| if let $enum::$t(_) = item { true } else { false }, |item| { progress = true; if let $enum::$t(item) = item { item } else { unreachable!() } }); pin_mut!(stream_); - ready.$num = self_.$t.as_mut().poll_forward(cx, stream_).is_ready(); + ready.$num = self_.$t.as_mut().poll_pipe(cx, stream_).is_ready(); })* if $(ready.$num &&)* true { break Poll::Ready(()) @@ -239,40 +232,26 @@ macro_rules! impl_tuple { } } } - fn poll_output(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let mut self_ = self.project(); - let mut pending = false; - $( - if self_.output.$num.is_none() { - if let Poll::Ready(output) = self_.$t.poll_output(cx) { - self_.output.$num = Some(output); - } else { - pending = true; - } - } - )* - if !pending { - Poll::Ready(($(self_.output.$num.take().unwrap(),)*)) - } else { - Poll::Pending - } - } } - impl<$($t: Reducer,)*> ReducerProcessSend for $reducea<$($t,)*> where $($t::Output: ProcessSend + 'static,)* { - type Output = ($($t::Output,)*); - } - impl<$($t: Reducer,)*> ReducerSend for $reducea<$($t,)*> where $($t::Output: Send + 'static,)* { + #[allow(unused_variables)] + impl<$($t: ReducerAsync<$s>,)* $($s,)*> ReducerAsync<$enum<$($s,)*>> for $reduceaasync<$($t,)* $($s,)*> { type Output = ($($t::Output,)*); + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { + let self_ = self.project(); + $join($(self_.$t.output(),)*).await + }) + } } #[derive(Clone, Serialize, Deserialize, new)] pub struct $reduceb<$($t,)*> { $($t: $t,)* } - impl<$($t: Reducer,)*> Reducer for $reduceb<$($t,)*> { - type Item = ($($t::Item,)*); + impl<$($t: Reducer<$s>,)* $($s,)*> Reducer<($($s,)*)> for $reduceb<$($t,)*> { type Output = ($($t::Output,)*); - type Async = $reducebasync<$($t::Async,)*>; + type Async = $reducebasync<$($t::Async,)* $($s,)*>; fn into_async(self) -> Self::Async { $reducebasync{ @@ -282,24 +261,21 @@ macro_rules! impl_tuple { } } } - impl<$($t: ReducerProcessSend,)*> ReducerProcessSend for $reduceb<$($t,)*> { - type Output = ($(<$t as ReducerProcessSend>::Output,)*); + impl<$($t: ReducerProcessSend<$s>,)* $($s,)*> ReducerProcessSend<($($s,)*)> for $reduceb<$($t,)*> { + type Output = ($(<$t as ReducerProcessSend<$s>>::Output,)*); } - impl<$($t: ReducerSend,)*> ReducerSend for $reduceb<$($t,)*> { - type Output = ($(<$t as ReducerSend>::Output,)*); + impl<$($t: ReducerSend<$s>,)* $($s,)*> ReducerSend<($($s,)*)> for $reduceb<$($t,)*> { + type Output = ($(<$t as ReducerSend<$s>>::Output,)*); } #[pin_project] - pub struct $reducebasync<$($t,)*> where $($t: ReducerAsync,)* { + pub struct $reducebasync<$($t,)* $($s,)*> where $($t: ReducerAsync<$s>,)* { $(#[pin] $t: $t,)* - peeked: Option<($(Option<$t::Item>,)*)>, + peeked: Option<($(Option<$s>,)*)>, output: ($(Option<$t::Output>,)*), } #[allow(unused_variables)] - impl<$($t: ReducerAsync,)*> ReducerAsync for $reducebasync<$($t,)*> { - type Item = ($($t::Item,)*); - type Output = ($($t::Output,)*); - - fn poll_forward(self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>) -> Poll<()> { + impl<$($t: ReducerAsync<$s>,)* $($s,)*> Sink<($($s,)*)> for $reducebasync<$($t,)* $($s,)*> { + fn poll_pipe(self: Pin<&mut Self>, cx: &mut Context, stream: Pin<&mut impl Stream>) -> Poll<()> { let mut self_ = self.project(); let mut ready = ($(false $comma)*); let stream = stream.map(|item| ($(Some(item.$num),)*)); @@ -321,7 +297,7 @@ macro_rules! impl_tuple { None => Poll::Ready(None), }).fuse(); pin_mut!(stream); - ready.$num = self_.$t.as_mut().poll_forward(cx, stream).is_ready(); + ready.$num = self_.$t.as_mut().poll_pipe(cx, stream).is_ready(); })* if let Some(peeked) = self_.peeked { if $(peeked.$num.is_none() &&)* true { @@ -337,36 +313,29 @@ macro_rules! impl_tuple { } } } - fn poll_output(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let mut self_ = self.project(); - let mut pending = false; - $( - if self_.output.$num.is_none() { - if let Poll::Ready(output) = self_.$t.poll_output(cx) { - self_.output.$num = Some(output); - } else { - pending = true; - } - } - )* - if !pending { - Poll::Ready(($(self_.output.$num.take().unwrap(),)*)) - } else { - Poll::Pending - } + } + #[allow(unused_variables)] + impl<$($t: ReducerAsync<$s>,)* $($s,)*> ReducerAsync<($($s,)*)> for $reducebasync<$($t,)* $($s,)*> { + type Output = ($($t::Output,)*); + + fn output<'a>(self: Pin<&'a mut Self>) -> Pin + 'a>> { + Box::pin(async move { + let self_ = self.project(); + $join($(self_.$t.output(),)*).await + }) } } ); } -impl_tuple!(ReduceA0 ReduceA0Async ReduceC0 ReduceC0Async AsyncTuple0 Sum0:); -impl_tuple!(ReduceA1 ReduceA1Async ReduceC1 ReduceC1Async AsyncTuple1 Sum1: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A,,); -impl_tuple!(ReduceA2 ReduceA2Async ReduceC2 ReduceC2Async AsyncTuple2 Sum2 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,,); -impl_tuple!(ReduceA3 ReduceA3Async ReduceC3 ReduceC3Async AsyncTuple3 Sum3 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,,); -impl_tuple!(ReduceA4 ReduceA4Async ReduceC4 ReduceC4Async AsyncTuple4 Sum4 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,,); -impl_tuple!(ReduceA5 ReduceA5Async ReduceC5 ReduceC5Async AsyncTuple5 Sum5 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,,); -impl_tuple!(ReduceA6 ReduceA6Async ReduceC6 ReduceC6Async AsyncTuple6 Sum6 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,, I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 5 F Copy,,); -impl_tuple!(ReduceA7 ReduceA7Async ReduceC7 ReduceC7Async AsyncTuple7 Sum7 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,, I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 5 F Copy,, I6 R6 O6 C6 iterator_6 reducer_a_6 reducer_b_6 6 G Copy,,); -impl_tuple!(ReduceA8 ReduceA8Async ReduceC8 ReduceC8Async AsyncTuple8 Sum8 Copy: I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 0 A Copy,, I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 1 B Copy,, I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 2 C Copy,, I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 3 D Copy,, I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 4 E Copy,, I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 5 F Copy,, I6 R6 O6 C6 iterator_6 reducer_a_6 reducer_b_6 6 G Copy,, I7 R7 O7 C7 iterator_7 reducer_a_7 reducer_b_7 7 H Copy,,); +impl_tuple!(ReduceA0 ReduceA0Async ReduceC0 ReduceC0Async AsyncTuple0 Sum0 Join0:); +impl_tuple!(ReduceA1 ReduceA1Async ReduceC1 ReduceC1Async AsyncTuple1 Sum1 Join1: 0 A S0 I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0,,); +impl_tuple!(ReduceA2 ReduceA2Async ReduceC2 ReduceC2Async AsyncTuple2 Sum2 Join2 Copy: 0 A S0 I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 Copy,, 1 B S1 I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 Copy,,); +impl_tuple!(ReduceA3 ReduceA3Async ReduceC3 ReduceC3Async AsyncTuple3 Sum3 Join3 Copy: 0 A S0 I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 Copy,, 1 B S1 I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 Copy,, 2 C S2 I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 Copy,,); +impl_tuple!(ReduceA4 ReduceA4Async ReduceC4 ReduceC4Async AsyncTuple4 Sum4 Join4 Copy: 0 A S0 I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 Copy,, 1 B S1 I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 Copy,, 2 C S2 I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 Copy,, 3 D S3 I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 Copy,,); +impl_tuple!(ReduceA5 ReduceA5Async ReduceC5 ReduceC5Async AsyncTuple5 Sum5 Join5 Copy: 0 A S0 I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 Copy,, 1 B S1 I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 Copy,, 2 C S2 I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 Copy,, 3 D S3 I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 Copy,, 4 E S4 I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 Copy,,); +impl_tuple!(ReduceA6 ReduceA6Async ReduceC6 ReduceC6Async AsyncTuple6 Sum6 Join6 Copy: 0 A S0 I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 Copy,, 1 B S1 I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 Copy,, 2 C S2 I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 Copy,, 3 D S3 I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 Copy,, 4 E S4 I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 Copy,, 5 F S5 I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 Copy,,); +impl_tuple!(ReduceA7 ReduceA7Async ReduceC7 ReduceC7Async AsyncTuple7 Sum7 Join7 Copy: 0 A S0 I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 Copy,, 1 B S1 I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 Copy,, 2 C S2 I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 Copy,, 3 D S3 I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 Copy,, 4 E S4 I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 Copy,, 5 F S5 I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 Copy,, 6 G S6 I6 R6 O6 C6 iterator_6 reducer_a_6 reducer_b_6 Copy,,); +impl_tuple!(ReduceA8 ReduceA8Async ReduceC8 ReduceC8Async AsyncTuple8 Sum8 Join8 Copy: 0 A S0 I0 R0 O0 C0 iterator_0 reducer_a_0 reducer_b_0 Copy,, 1 B S1 I1 R1 O1 C1 iterator_1 reducer_a_1 reducer_b_1 Copy,, 2 C S2 I2 R2 O2 C2 iterator_2 reducer_a_2 reducer_b_2 Copy,, 3 D S3 I3 R3 O3 C3 iterator_3 reducer_a_3 reducer_b_3 Copy,, 4 E S4 I4 R4 O4 C4 iterator_4 reducer_a_4 reducer_b_4 Copy,, 5 F S5 I5 R5 O5 C5 iterator_5 reducer_a_5 reducer_b_5 Copy,, 6 G S6 I6 R6 O6 C6 iterator_6 reducer_a_6 reducer_b_6 Copy,, 7 H S7 I7 R7 O7 C7 iterator_7 reducer_a_7 reducer_b_7 Copy,,); #[pin_project(project = PeekableProj)] #[derive(Debug)] @@ -415,3 +384,60 @@ impl<'a, S: Stream> Stream for Peekable<'a, S> { (lower, upper) } } + +// from https://github.com/rust-lang/futures-rs/blob/19831dbecd13961fbd4243c1114663e24299c33d/futures-util/src/future/join.rs +macro_rules! generate { + ($( + ($Join:ident, <$($Fut:ident),*>), + )*) => ($( + #[pin_project] + struct $Join<$($Fut: Future),*> { + $(#[pin] $Fut: MaybeDone<$Fut>,)* + dummy: (), + } + + fn $Join<$($Fut: Future),*>($($Fut: $Fut),*) -> $Join<$($Fut),*> { + $Join { + $($Fut: maybe_done($Fut),)* + dummy: (), + } + } + + #[allow(unused_variables)] + impl<$($Fut: Future),*> Future for $Join<$($Fut),*> { + type Output = ($($Fut::Output,)*); + + fn poll( + self: Pin<&mut Self>, cx: &mut Context<'_> + ) -> Poll { + let mut all_done = true; + let mut futures = self.project(); + $( + all_done &= futures.$Fut.as_mut().poll(cx).is_ready(); + )* + + if all_done { + Poll::Ready(($(futures.$Fut.take_output().unwrap(),)*)) + } else { + Poll::Pending + } + } + } + + impl<$($Fut: FusedFuture),*> FusedFuture for $Join<$($Fut),*> { + fn is_terminated(&self) -> bool { + $( + self.$Fut.is_terminated() && + )* true + } + } + )*) +} + +generate! { + (Join0, <>), + (Join1, ), + (Join6, ), + (Join7, ), + (Join8, ), +} diff --git a/amadeus-core/src/par_stream.rs b/amadeus-core/src/par_stream.rs index f93e17ef..a85978f7 100644 --- a/amadeus-core/src/par_stream.rs +++ b/amadeus-core/src/par_stream.rs @@ -11,19 +11,14 @@ mod map; mod sum_type; mod update; -use ::sum::*; use async_trait::async_trait; use either::Either; -use futures::{pin_mut, ready, stream, stream::StreamExt, Stream}; -use pin_project::pin_project; -use serde::{Deserialize, Serialize}; +use futures::{pin_mut, stream::StreamExt as _, Stream}; use serde_closure::*; -use std::{ - cmp::Ordering, collections::HashMap, future::Future, hash::Hash, iter, marker::PhantomData, ops::{DerefMut, FnMut}, pin::Pin, task::{Context, Poll}, vec -}; +use std::{cmp::Ordering, collections::HashMap, hash::Hash, iter, ops::FnMut, vec}; use crate::{ - into_par_stream::IntoDistributedStream, pool::{ProcessPool, ProcessSend, ThreadPool}, sink::{Sink, SinkMap}, util::type_coerce + into_par_stream::IntoDistributedStream, pipe::StreamExt, pool::{ProcessPool, ProcessSend, ThreadPool} }; pub use self::{ @@ -35,44 +30,10 @@ use super::{par_pipe::*, par_sink::*}; #[must_use] pub trait StreamTask { type Item; - type Async: StreamTaskAsync; + type Async: Stream; fn into_async(self) -> Self::Async; } -#[must_use] -pub trait StreamTaskAsync { - type Item; - - fn poll_run( - self: Pin<&mut Self>, cx: &mut Context, sink: Pin<&mut impl Sink>, - ) -> Poll<()>; -} - -impl