Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(clients): chunked batch helper #3154

Merged
merged 6 commits into from
Jun 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions playground/python/app/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,9 @@ async def main():

try:
resp = await client.replace_all_objects(
index_name="test-flag",
objects=[{"name": f"John Doe{i}", "objectID": f"fff2bd4d-bb17-4e21-a0c4-0a8ea5e363f2{i}" } for i in range(1001)],
index_name="test_replace_all_objects",
objects=[{"name": f"John Doe{i}", "objectID": f"fff2bd4d-bb17-4e21-a0c4-0a8ea5e363f2{i}" } for i in range(33)],
batch_size=10
)

print(resp)
Expand Down
1,628 changes: 1,308 additions & 320 deletions playground/python/poetry.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion templates/go/search_helpers.mustache
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,7 @@ func (c *APIClient) ChunkedBatch(indexName string, objects []map[string]any, act
for i, obj := range objects {
requests = append(requests, *NewBatchRequest(*action, obj))

if i%*batchSize == 0 {
if len(requests) == *batchSize || i == len(objects)-1 {
resp, err := c.Batch(c.NewApiBatchRequest(indexName, NewBatchWriteParams(requests)))
if err != nil {
return nil, err
Expand Down
7 changes: 4 additions & 3 deletions templates/javascript/clients/client/api/helpers.mustache
Original file line number Diff line number Diff line change
Expand Up @@ -279,9 +279,10 @@ async chunkedBatch({ indexName, objects, action = 'addObject', waitForTasks, bat
let requests: Array<BatchRequest> = [];
const responses: Array<BatchResponse> = [];

for (const [i, obj] of objects.entries()) {
const objectEntries = objects.entries();
for (const [i, obj] of objectEntries) {
requests.push({action, body: obj});
if (i % batchSize === 0) {
if (requests.length === batchSize || i === objects.length-1) {
responses.push(await this.batch({indexName, batchWriteParams: {requests}}, requestOptions));
requests = [];
}
Expand All @@ -303,7 +304,7 @@ async chunkedBatch({ indexName, objects, action = 'addObject', waitForTasks, bat
* @param replaceAllObjects - The `replaceAllObjects` object.
* @param replaceAllObjects.indexName - The `indexName` to replace `objects` in.
* @param replaceAllObjects.objects - The array of `objects` to store in the given Algolia `indexName`.
* @param replaceAllObjects.batchSize - The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
* @param replaceAllObjects.batchSize - The size of the chunk of `objects`. The number of `batch` calls will be equal to `objects.length / batchSize`. Defaults to 1000.
* @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `getTask` method and merged with the transporter requestOptions.
*/
async replaceAllObjects(
Expand Down
8 changes: 4 additions & 4 deletions templates/php/api.mustache
Original file line number Diff line number Diff line change
Expand Up @@ -476,13 +476,13 @@ use {{invokerPackage}}\Support\Helpers;
'action' => $action,
'body' => $object,
];
$count++;

if ($count === $batchSize) {
if (sizeof($requests) === $batchSize || $count === sizeof($objects)-1) {
$responses[] = $this->batch($indexName, ['requests' => $requests], $requestOptions);
$requests = [];
$count = 0;
}

$count++;
}

if (!empty($requests)) {
Expand Down Expand Up @@ -560,4 +560,4 @@ use {{invokerPackage}}\Support\Helpers;
);
}
}
{{/operations}}
{{/operations}}
4 changes: 2 additions & 2 deletions templates/python/search_helpers.mustache
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@
responses: List[BatchResponse] = []
for i, obj in enumerate(objects):
requests.append(BatchRequest(action=action, body=obj))
if i % batch_size == 0:
if len(requests) == batch_size or i == len(objects) - 1:
responses.append(
await self.batch(
index_name=index_name,
Expand Down Expand Up @@ -292,4 +292,4 @@
"copy_operation_response": copy_operation_response,
"batch_responses": batch_responses,
"move_operation_response": move_operation_response,
}
}
Loading