From d83d725c397fc6625cbc85c09c4008a0452d2c34 Mon Sep 17 00:00:00 2001 From: Beno!t POLASZEK Date: Mon, 13 Nov 2023 11:22:20 +0100 Subject: [PATCH] feature: improve chaining --- README.md | 38 +++++++++++++ src/Extractor/ChainExtractor.php | 8 +++ src/Internal/EtlBuilderTrait.php | 32 ++++------- src/Loader/ChainLoader.php | 8 +++ src/Transformer/ChainTransformer.php | 8 +++ src/functions.php | 38 ++++++++----- tests/Unit/Extractor/ChainExtractorTest.php | 13 ++--- tests/Unit/Loader/ChainLoaderTest.php | 54 ++++++++++--------- .../Unit/Transformer/ChainTransformerTest.php | 35 ++++++------ 9 files changed, 154 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 5ab91f5..acc1abe 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,44 @@ $report = withRecipe(new LoggerRecipe($logger)) ->process(['foo', 'bar']); ``` +Chaining extractors / transformers / loaders +------------------------------------------- + +Instead of replacing existing extractors / transformers / loaders inside your `EtlExecutor`, +you can decorate them by using the `chain` function: + +```php +use BenTools\ETL\EtlExecutor; +use ArrayObject; + +use function BenTools\ETL\chain; +use function implode; +use function str_split; +use function strtoupper; + +$a = new ArrayObject(); +$executor = (new EtlExecutor()) + ->extractFrom(fn () => yield 'foo') + ->transformWith(fn (string $value) => strtoupper($value)) + ->loadInto(fn (string $value) => $a->append($value)); + +$b = new ArrayObject(); +$executor = $executor + ->extractFrom( + chain($executor->extractor)->with(fn () => ['bar']) + ) + ->transformWith( + chain($executor->transformer)->with(fn (string $value) => implode('-', str_split($value))) + ) + ->loadInto( + chain($executor->loader)->with(fn (string $value) => $b->append($value)) + ); + +$executor->process(); +var_dump([...$a]); // ['F-O-O', 'B-A-R'] +var_dump([...$b]); // ['F-O-O', 'B-A-R'] + +``` Contribute ---------- diff --git a/src/Extractor/ChainExtractor.php b/src/Extractor/ChainExtractor.php index 6f06d08..705cbda 100644 --- a/src/Extractor/ChainExtractor.php +++ b/src/Extractor/ChainExtractor.php @@ -39,4 +39,12 @@ public function extract(EtlState $state): iterable } } } + + public static function from(ExtractorInterface $extractor): self + { + return match ($extractor instanceof self) { + true => $extractor, + false => new self($extractor), + }; + } } diff --git a/src/Internal/EtlBuilderTrait.php b/src/Internal/EtlBuilderTrait.php index e60899d..7642a72 100644 --- a/src/Internal/EtlBuilderTrait.php +++ b/src/Internal/EtlBuilderTrait.php @@ -16,8 +16,6 @@ use BenTools\ETL\Transformer\ChainTransformer; use BenTools\ETL\Transformer\TransformerInterface; -use function count; - /** * @internal * @@ -30,8 +28,10 @@ trait EtlBuilderTrait */ use EtlEventListenersTrait; - public function extractFrom(ExtractorInterface|callable $extractor, ExtractorInterface|callable ...$extractors): self - { + public function extractFrom( + ExtractorInterface|callable $extractor, + ExtractorInterface|callable ...$extractors + ): self { $extractors = [$extractor, ...$extractors]; foreach ($extractors as $e => $_extractor) { @@ -40,15 +40,13 @@ public function extractFrom(ExtractorInterface|callable $extractor, ExtractorInt } } - if (count($extractors) > 1) { - return $this->cloneWith(['extractor' => new ChainExtractor(...$extractors)]); - } - - return $this->cloneWith(['extractor' => $extractors[0]]); + return $this->cloneWith(['extractor' => new ChainExtractor(...$extractors)]); } - public function transformWith(TransformerInterface|callable $transformer, TransformerInterface|callable ...$transformers): self - { + public function transformWith( + TransformerInterface|callable $transformer, + TransformerInterface|callable ...$transformers + ): self { $transformers = [$transformer, ...$transformers]; foreach ($transformers as $t => $_transformer) { @@ -57,11 +55,7 @@ public function transformWith(TransformerInterface|callable $transformer, Transf } } - if (count($transformers) > 1) { - return $this->cloneWith(['transformer' => new ChainTransformer(...$transformers)]); - } - - return $this->cloneWith(['transformer' => $transformers[0]]); + return $this->cloneWith(['transformer' => new ChainTransformer(...$transformers)]); } public function loadInto(LoaderInterface|callable $loader, LoaderInterface|callable ...$loaders): self @@ -74,11 +68,7 @@ public function loadInto(LoaderInterface|callable $loader, LoaderInterface|calla } } - if (count($loaders) > 1) { - return $this->cloneWith(['loader' => new ChainLoader(...$loaders)]); - } - - return $this->cloneWith(['loader' => $loaders[0]]); + return $this->cloneWith(['loader' => new ChainLoader(...$loaders)]); } public function withOptions(EtlConfiguration $configuration): self diff --git a/src/Loader/ChainLoader.php b/src/Loader/ChainLoader.php index cb2f87a..1610537 100644 --- a/src/Loader/ChainLoader.php +++ b/src/Loader/ChainLoader.php @@ -51,4 +51,12 @@ public function flush(bool $isPartial, EtlState $state): mixed return $output ?? null; } + + public static function from(LoaderInterface $loader): self + { + return match ($loader instanceof self) { + true => $loader, + false => new self($loader), + }; + } } diff --git a/src/Transformer/ChainTransformer.php b/src/Transformer/ChainTransformer.php index f598bc2..43df97b 100644 --- a/src/Transformer/ChainTransformer.php +++ b/src/Transformer/ChainTransformer.php @@ -51,4 +51,12 @@ public function doTransform(mixed $item, EtlState $state): mixed return $item; } + + public static function from(TransformerInterface $transformer): self + { + return match ($transformer instanceof self) { + true => $transformer, + false => new self($transformer), + }; + } } diff --git a/src/functions.php b/src/functions.php index 870f2da..edeff72 100644 --- a/src/functions.php +++ b/src/functions.php @@ -4,10 +4,13 @@ namespace BenTools\ETL; +use BenTools\ETL\Extractor\ChainExtractor; use BenTools\ETL\Extractor\ExtractorInterface; use BenTools\ETL\Internal\Ref; +use BenTools\ETL\Loader\ChainLoader; use BenTools\ETL\Loader\LoaderInterface; use BenTools\ETL\Recipe\Recipe; +use BenTools\ETL\Transformer\ChainTransformer; use BenTools\ETL\Transformer\TransformerInterface; use function array_fill_keys; @@ -16,13 +19,13 @@ use function func_get_args; /** - * @internal - * * @param list $keys * @param array $values * @param array ...$extraValues * * @return array + * + * @internal */ function array_fill_from(array $keys, array $values, array ...$extraValues): array { @@ -33,13 +36,13 @@ function array_fill_from(array $keys, array $values, array ...$extraValues): arr } /** - * @internal - * - * @template T - * * @param T $value * * @return Ref + * + * @internal + * + * @template T */ function ref(mixed $value): Ref { @@ -47,13 +50,13 @@ function ref(mixed $value): Ref } /** - * @internal - * - * @template T - * * @param Ref $ref * * @return T + * + * @internal + * + * @template T */ function unref(Ref $ref): mixed { @@ -65,8 +68,10 @@ function extractFrom(ExtractorInterface|callable $extractor, ExtractorInterface| return (new EtlExecutor())->extractFrom(...func_get_args()); } -function transformWith(TransformerInterface|callable $transformer, TransformerInterface|callable ...$transformers): EtlExecutor -{ +function transformWith( + TransformerInterface|callable $transformer, + TransformerInterface|callable ...$transformers +): EtlExecutor { return (new EtlExecutor())->transformWith(...func_get_args()); } @@ -79,3 +84,12 @@ function withRecipe(Recipe|callable $recipe): EtlExecutor { return (new EtlExecutor())->withRecipe(...func_get_args()); } + +function chain(ExtractorInterface|TransformerInterface|LoaderInterface $service, +): ChainExtractor|ChainTransformer|ChainLoader { + return match (true) { + $service instanceof ExtractorInterface => ChainExtractor::from($service), + $service instanceof TransformerInterface => ChainTransformer::from($service), + $service instanceof LoaderInterface => ChainLoader::from($service), + }; +} diff --git a/tests/Unit/Extractor/ChainExtractorTest.php b/tests/Unit/Extractor/ChainExtractorTest.php index 2f23be3..93b5854 100644 --- a/tests/Unit/Extractor/ChainExtractorTest.php +++ b/tests/Unit/Extractor/ChainExtractorTest.php @@ -5,18 +5,19 @@ namespace BenTools\ETL\Tests\Unit\Extractor; use BenTools\ETL\EtlExecutor; -use BenTools\ETL\Extractor\ChainExtractor; +use BenTools\ETL\Extractor\CallableExtractor; +use function BenTools\ETL\chain; use function BenTools\ETL\extractFrom; use function expect; it('chains extractors', function () { // Given - $extractor = (new ChainExtractor( - fn () => 'banana', - fn () => yield from ['apple', 'strawberry'], - ))->with(fn () => ['raspberry', 'peach']); - $executor = (new EtlExecutor($extractor)); + $executor = new EtlExecutor(new CallableExtractor(fn () => 'banana')); + $executor = $executor->extractFrom(chain($executor->extractor) + ->with(fn () => yield from ['apple', 'strawberry']) + ->with(fn () => ['raspberry', 'peach'])) + ; // When $report = $executor->process(); diff --git a/tests/Unit/Loader/ChainLoaderTest.php b/tests/Unit/Loader/ChainLoaderTest.php index 0f6f108..e98b549 100644 --- a/tests/Unit/Loader/ChainLoaderTest.php +++ b/tests/Unit/Loader/ChainLoaderTest.php @@ -7,9 +7,10 @@ use ArrayObject; use BenTools\ETL\EtlExecutor; use BenTools\ETL\EtlState; -use BenTools\ETL\Loader\ChainLoader; +use BenTools\ETL\Loader\CallableLoader; use BenTools\ETL\Loader\ConditionalLoaderInterface; +use function BenTools\ETL\chain; use function expect; it('chains loaders', function () { @@ -17,36 +18,39 @@ $a = new ArrayObject(); $b = new ArrayObject(); $c = new ArrayObject(); - $loader = (new ChainLoader( + + $executor = new EtlExecutor(loader: new CallableLoader( fn (string $item) => $a[] = $item, // @phpstan-ignore-line - fn (string $item) => $b[] = $item, // @phpstan-ignore-line - )) - ->with( - new class() implements ConditionalLoaderInterface { - public function supports(mixed $item, EtlState $state): bool - { - return 'foo' !== $item; - } - - public function load(mixed $item, EtlState $state): void - { - $state->context[__CLASS__][] = $item; - } - - public function flush(bool $isPartial, EtlState $state): mixed - { - foreach ($state->context[__CLASS__] as $item) { - $state->context['storage'][] = $item; + )); + $executor = $executor->loadInto( + chain($executor->loader) + ->with(fn (string $item) => $b[] = $item) // @phpstan-ignore-line + ->with( + new class() implements ConditionalLoaderInterface { + public function supports(mixed $item, EtlState $state): bool + { + return 'foo' !== $item; + } + + public function load(mixed $item, EtlState $state): void + { + $state->context[__CLASS__][] = $item; } - return $state->context['storage']; - } - }, - ); + public function flush(bool $isPartial, EtlState $state): mixed + { + foreach ($state->context[__CLASS__] as $item) { + $state->context['storage'][] = $item; + } + + return $state->context['storage']; + } + }, + ) + ); // Given $input = ['foo', 'bar']; - $executor = new EtlExecutor(loader: $loader); // When $executor->process($input, context: ['storage' => $c]); diff --git a/tests/Unit/Transformer/ChainTransformerTest.php b/tests/Unit/Transformer/ChainTransformerTest.php index 63ead71..68ae0e6 100644 --- a/tests/Unit/Transformer/ChainTransformerTest.php +++ b/tests/Unit/Transformer/ChainTransformerTest.php @@ -5,9 +5,10 @@ namespace BenTools\ETL\Tests\Unit\Transformer; use BenTools\ETL\EtlExecutor; -use BenTools\ETL\Transformer\ChainTransformer; +use BenTools\ETL\Transformer\CallableTransformer; use Generator; +use function BenTools\ETL\chain; use function expect; use function implode; use function strrev; @@ -16,24 +17,26 @@ it('chains transformers', function () { // Given $input = ['foo', 'bar']; - $transformer = (new ChainTransformer( - fn (string $item): string => strrev($item), - function (string $item): Generator { - yield $item; - yield strtoupper($item); - }, - )) - ->with(fn (Generator $items): array => [...$items]) - ->with(function (array $items): array { - $items[] = 'hey'; + $executor = new EtlExecutor(transformer: new CallableTransformer( + fn (string $item): string => strrev($item) + )); + $executor = $executor->transformWith( + chain($executor->transformer) + ->with(function (string $item): Generator { + yield $item; + yield strtoupper($item); + }) + ->with(fn (Generator $items): array => [...$items]) + ->with(function (array $items): array { + $items[] = 'hey'; - return $items; - }) - ->with(fn (array $items): string => implode('-', $items)); + return $items; + }) + ->with(fn (array $items): string => implode('-', $items)), + ); // When - $report = (new EtlExecutor(transformer: $transformer)) - ->process($input); + $report = $executor->process($input); // Then expect($report->output)->toBe([