Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[8.x] Add Builder@lazy() and Builder@lazyById() methods #36699

Merged
merged 1 commit into from
Mar 23, 2021
Merged

[8.x] Add Builder@lazy() and Builder@lazyById() methods #36699

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions src/Illuminate/Database/Concerns/BuildsQueries.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
use Illuminate\Pagination\LengthAwarePaginator;
use Illuminate\Pagination\Paginator;
use Illuminate\Support\Collection;
use Illuminate\Support\LazyCollection;
use InvalidArgumentException;

trait BuildsQueries
{
Expand Down Expand Up @@ -159,6 +161,76 @@ public function eachById(callable $callback, $count = 1000, $column = null, $ali
}, $column, $alias);
}

/**
* Query lazily, by chunks of the given size.
*
* @param int $chunkSize
* @return \Illuminate\Support\LazyCollection
*/
public function lazy($chunkSize = 1000)
{
if ($chunkSize < 1) {
throw new InvalidArgumentException('The chunk size should be at least 1');
}

$this->enforceOrderBy();

return LazyCollection::make(function () use ($chunkSize) {
$page = 1;

while (true) {
$results = $this->forPage($page++, $chunkSize)->get();

foreach ($results as $result) {
yield $result;
}

if ($results->count() < $chunkSize) {
return;
}
}
});
}

/**
* Query lazily, by chunking the results of a query by comparing IDs.
*
* @param int $count
* @param string|null $column
* @param string|null $alias
* @return \Illuminate\Support\LazyCollection
*/
public function lazyById($chunkSize = 1000, $column = null, $alias = null)
{
if ($chunkSize < 1) {
throw new InvalidArgumentException('The chunk size should be at least 1');
}

$column = $column ?? $this->defaultKeyName();

$alias = $alias ?? $column;

return LazyCollection::make(function () use ($chunkSize, $column, $alias) {
$lastId = null;

while (true) {
$clone = clone $this;

$results = $clone->forPageAfterId($chunkSize, $lastId, $column)->get();

foreach ($results as $result) {
yield $result;
}

if ($results->count() < $chunkSize) {
return;
}

$lastId = $results->last()->{$alias};
}
});
}

/**
* Execute the query and get the first result.
*
Expand Down
112 changes: 112 additions & 0 deletions tests/Database/DatabaseEloquentBuilderTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,118 @@ public function testChunkPaginatesUsingIdWithCountZero()
}, 'someIdField');
}

public function testLazyWithLastChunkComplete()
{
$builder = m::mock(Builder::class.'[forPage,get]', [$this->getMockQueryBuilder()]);
$builder->getQuery()->orders[] = ['column' => 'foobar', 'direction' => 'asc'];

$builder->shouldReceive('forPage')->once()->with(1, 2)->andReturnSelf();
$builder->shouldReceive('forPage')->once()->with(2, 2)->andReturnSelf();
$builder->shouldReceive('forPage')->once()->with(3, 2)->andReturnSelf();
$builder->shouldReceive('get')->times(3)->andReturn(
new Collection(['foo1', 'foo2']),
new Collection(['foo3', 'foo4']),
new Collection([])
);

$this->assertEquals(
['foo1', 'foo2', 'foo3', 'foo4'],
$builder->lazy(2)->all()
);
}

public function testLazyWithLastChunkPartial()
{
$builder = m::mock(Builder::class.'[forPage,get]', [$this->getMockQueryBuilder()]);
$builder->getQuery()->orders[] = ['column' => 'foobar', 'direction' => 'asc'];

$builder->shouldReceive('forPage')->once()->with(1, 2)->andReturnSelf();
$builder->shouldReceive('forPage')->once()->with(2, 2)->andReturnSelf();
$builder->shouldReceive('get')->times(2)->andReturn(
new Collection(['foo1', 'foo2']),
new Collection(['foo3'])
);

$this->assertEquals(
['foo1', 'foo2', 'foo3'],
$builder->lazy(2)->all()
);
}

public function testLazyIsLazy()
{
$builder = m::mock(Builder::class.'[forPage,get]', [$this->getMockQueryBuilder()]);
$builder->getQuery()->orders[] = ['column' => 'foobar', 'direction' => 'asc'];

$builder->shouldReceive('forPage')->once()->with(1, 2)->andReturnSelf();
$builder->shouldReceive('get')->once()->andReturn(new Collection(['foo1', 'foo2']));

$this->assertEquals(['foo1', 'foo2'], $builder->lazy(2)->take(2)->all());
}

public function testLazyByIdWithLastChunkComplete()
{
$builder = m::mock(Builder::class.'[forPageAfterId,get]', [$this->getMockQueryBuilder()]);
$builder->getQuery()->orders[] = ['column' => 'foobar', 'direction' => 'asc'];

$chunk1 = new Collection([(object) ['someIdField' => 1], (object) ['someIdField' => 2]]);
$chunk2 = new Collection([(object) ['someIdField' => 10], (object) ['someIdField' => 11]]);
$chunk3 = new Collection([]);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturnSelf();
$builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturnSelf();
$builder->shouldReceive('forPageAfterId')->once()->with(2, 11, 'someIdField')->andReturnSelf();
$builder->shouldReceive('get')->times(3)->andReturn($chunk1, $chunk2, $chunk3);

$this->assertEquals(
[
(object) ['someIdField' => 1],
(object) ['someIdField' => 2],
(object) ['someIdField' => 10],
(object) ['someIdField' => 11],
],
$builder->lazyById(2, 'someIdField')->all()
);
}

public function testLazyByIdWithLastChunkPartial()
{
$builder = m::mock(Builder::class.'[forPageAfterId,get]', [$this->getMockQueryBuilder()]);
$builder->getQuery()->orders[] = ['column' => 'foobar', 'direction' => 'asc'];

$chunk1 = new Collection([(object) ['someIdField' => 1], (object) ['someIdField' => 2]]);
$chunk2 = new Collection([(object) ['someIdField' => 10]]);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturnSelf();
$builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturnSelf();
$builder->shouldReceive('get')->times(2)->andReturn($chunk1, $chunk2);

$this->assertEquals(
[
(object) ['someIdField' => 1],
(object) ['someIdField' => 2],
(object) ['someIdField' => 10],
],
$builder->lazyById(2, 'someIdField')->all()
);
}

public function testLazyByIdIsLazy()
{
$builder = m::mock(Builder::class.'[forPageAfterId,get]', [$this->getMockQueryBuilder()]);
$builder->getQuery()->orders[] = ['column' => 'foobar', 'direction' => 'asc'];

$chunk1 = new Collection([(object) ['someIdField' => 1], (object) ['someIdField' => 2]]);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturnSelf();
$builder->shouldReceive('get')->once()->andReturn($chunk1);

$this->assertEquals(
[
(object) ['someIdField' => 1],
(object) ['someIdField' => 2],
],
$builder->lazyById(2, 'someIdField')->take(2)->all()
);
}

public function testPluckReturnsTheMutatedAttributesOfAModel()
{
$builder = $this->getBuilder();
Expand Down