Skip to content

Commit

Permalink
Fix cloudfoundry#1222: Chunk request with large number of matched res…
Browse files Browse the repository at this point in the history
…ources

* CAPI, with v3 API, is limited to returning responses with max 5000 max matched resources; so we chunk them into request with max 5000 matched resources
* update code that calls the request instead of the request code itself
  • Loading branch information
anthonydahanne committed Feb 28, 2024
1 parent 2fd0cc6 commit 7529618
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 92 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@
import java.util.Collection;
import java.util.Enumeration;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.cloudfoundry.client.CloudFoundryClient;
Expand All @@ -49,6 +47,7 @@ public final class ResourceMatchingUtilsV3 {

private static final Logger LOGGER =
LoggerFactory.getLogger("cloudfoundry-client.resource-matching-v3");
public static final int MAX_RESOURCES_SIZE = 5000;

private ResourceMatchingUtilsV3() {}

Expand All @@ -58,9 +57,9 @@ public static Mono<List<MatchedResource>> getMatchedResources(
? getArtifactMetadataFromDirectory(application)
: getArtifactMetadataFromZip(application))
.collectList()
.flatMap(
(List<ArtifactMetadata> artifactMetadatas) ->
requestListMatchingResources(cloudFoundryClient, artifactMetadatas))
.flatMapMany(Flux::fromIterable)
.buffer(MAX_RESOURCES_SIZE)
.flatMap(chunk -> requestListMatchingResources(cloudFoundryClient, chunk))
.map(ListMatchingResourcesResponse::getResources)
.doOnNext(
matched ->
Expand All @@ -71,6 +70,8 @@ public static Mono<List<MatchedResource>> getMatchedResources(
matched.stream()
.mapToInt(MatchedResource::getSize)
.sum())))
.collectList()
.map(lists -> lists.stream().flatMap(List::stream).collect(Collectors.toList()))
.subscribeOn(Schedulers.boundedElastic());
}

Expand Down Expand Up @@ -128,40 +129,24 @@ private static Flux<ArtifactMetadata> getArtifactMetadataFromZip(Path applicatio
return Flux.fromIterable(artifactMetadatas);
}

static Mono<ListMatchingResourcesResponse> requestListMatchingResources(
private static Mono<ListMatchingResourcesResponse> requestListMatchingResources(
CloudFoundryClient cloudFoundryClient, Collection<ArtifactMetadata> artifactMetadatas) {
Collection<List<ArtifactMetadata>> chunksOfArtifactMetadatas = chunkArtifactMetadatas(artifactMetadatas, 5000);

List<ListMatchingResourcesResponse> matchingResourcesResponse = new ArrayList<>();

chunksOfArtifactMetadatas.forEach(chunkOfArtifactMetadatas -> {
ListMatchingResourcesRequest request =
chunkOfArtifactMetadatas.stream()
.reduce(
ListMatchingResourcesRequest.builder(),
(builder, artifactMetadata) ->
builder.resource(
MatchedResource.builder()
.checksum(artifactMetadata.getChecksum())
.mode(artifactMetadata.getPermissions())
.size(artifactMetadata.getSize())
.path(artifactMetadata.getPath())
.build()),
(a, b) -> a.addAllResources(b.build().getResources()))
.build();

matchingResourcesResponse.add(cloudFoundryClient.resourceMatchV3().list(request).block());
});
List<MatchedResource> collect = matchingResourcesResponse.stream().map(listMatchingResourcesResponse -> listMatchingResourcesResponse.getResources()).flatMap(Collection::stream).collect(Collectors.toList());
ListMatchingResourcesResponse listMatchingResourcesResponse = ListMatchingResourcesResponse.builder().resources(collect).build();
Mono<ListMatchingResourcesResponse> listMatchingResourcesResponseMono = Mono.just(listMatchingResourcesResponse);
return listMatchingResourcesResponseMono;
}

private static Collection<List<ArtifactMetadata>> chunkArtifactMetadatas(Collection<ArtifactMetadata> artifactMetadatas, int maxNumberOfArtifacts) {
final AtomicInteger counter = new AtomicInteger();
List<ArtifactMetadata> metadataList = artifactMetadatas.stream().collect(Collectors.toList());
return metadataList.stream().collect(Collectors.groupingBy(artifactMetadata -> counter.getAndIncrement() / maxNumberOfArtifacts)).values();
ListMatchingResourcesRequest request =
artifactMetadatas.stream()
.reduce(
ListMatchingResourcesRequest.builder(),
(builder, artifactMetadata) ->
builder.resource(
MatchedResource.builder()
.checksum(artifactMetadata.getChecksum())
.mode(artifactMetadata.getPermissions())
.size(artifactMetadata.getSize())
.path(artifactMetadata.getPath())
.build()),
(a, b) -> a.addAllResources(b.build().getResources()))
.build();

return cloudFoundryClient.resourceMatchV3().list(request);
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,63 +1,57 @@
package org.cloudfoundry.util;

import static org.cloudfoundry.util.ResourceMatchingUtilsV3.getMatchedResources;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import org.cloudfoundry.client.CloudFoundryClient;
import org.cloudfoundry.client.v3.Checksum;
import org.cloudfoundry.client.v3.resourcematch.ListMatchingResourcesRequest;
import org.cloudfoundry.client.v3.resourcematch.ListMatchingResourcesResponse;
import org.cloudfoundry.client.v3.resourcematch.ResourceMatchV3;
import org.cloudfoundry.client.v3.resourcematch.MatchedResource;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.ClassPathResource;
import reactor.core.publisher.Mono;

import java.util.ArrayList;
import java.util.Collection;
import java.util.stream.IntStream;

import static org.cloudfoundry.util.ResourceMatchingUtilsV3.requestListMatchingResources;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

class ResourceMatchingUtilsV3Test {


@Test
void requestListMatchingResources5() {
CloudFoundryClient cloudFoundryClient = mock(CloudFoundryClient.class);
Collection<ResourceMatchingUtilsV3.ArtifactMetadata> artifactMetadatas = new ArrayList<>();

IntStream.range(0,5).forEach(value -> {
artifactMetadatas.add(new ResourceMatchingUtilsV3.ArtifactMetadata(Checksum.builder().value("pif" + value).build(), "path"+value, "0644", value));
});
when(cloudFoundryClient.resourceMatchV3()).thenReturn(new ResourceMatchV3() {
@Override
public Mono<ListMatchingResourcesResponse> list(ListMatchingResourcesRequest request) {
return Mono.just(ListMatchingResourcesResponse.builder().addAllResources(request.getResources()).build());
}
});

ListMatchingResourcesResponse listMatchingResourcesResponse = requestListMatchingResources(cloudFoundryClient, artifactMetadatas).block();
assertEquals(5, listMatchingResourcesResponse.getResources().size());

}

@Test
void requestListMatchingResources15001() {
CloudFoundryClient cloudFoundryClient = mock(CloudFoundryClient.class);
Collection<ResourceMatchingUtilsV3.ArtifactMetadata> artifactMetadatas = new ArrayList<>();

IntStream.range(0,15001).forEach(value -> {
artifactMetadatas.add(new ResourceMatchingUtilsV3.ArtifactMetadata(Checksum.builder().value("pif" + value).build(), "path"+value, "0644", value));
});
when(cloudFoundryClient.resourceMatchV3()).thenReturn(new ResourceMatchV3() {
@Override
public Mono<ListMatchingResourcesResponse> list(ListMatchingResourcesRequest request) {
return Mono.just(ListMatchingResourcesResponse.builder().addAllResources(request.getResources()).build());
}
});

ListMatchingResourcesResponse listMatchingResourcesResponse = requestListMatchingResources(cloudFoundryClient, artifactMetadatas).block();
assertEquals(15001, listMatchingResourcesResponse.getResources().size());

}

}
@Test
void requestListMatchingResources2() throws IOException {
CloudFoundryClient cloudFoundryClient = mock(CloudFoundryClient.class);
when(cloudFoundryClient.resourceMatchV3())
.thenReturn(
request ->
Mono.just(
ListMatchingResourcesResponse.builder()
.addAllResources(request.getResources())
.build()));

Path testApplication = new ClassPathResource("test-application.zip").getFile().toPath();

List<MatchedResource> result =
getMatchedResources(cloudFoundryClient, testApplication).block();
assertNotNull(result);
assertEquals(2, result.size());
}

@Test
void requestListMatchingResources15001() throws IOException {
CloudFoundryClient cloudFoundryClient = mock(CloudFoundryClient.class);
when(cloudFoundryClient.resourceMatchV3())
.thenReturn(
request ->
Mono.just(
ListMatchingResourcesResponse.builder()
.addAllResources(request.getResources())
.build()));
Path testApplication = new ClassPathResource("15001_files.zip").getFile().toPath();

List<MatchedResource> result =
getMatchedResources(cloudFoundryClient, testApplication).block();
assertNotNull(result);
assertEquals(15001, result.size());
}
}
Binary file not shown.
Binary file not shown.

0 comments on commit 7529618

Please sign in to comment.