Fix cloudfoundry#1222: Chunk request with large number of matched res…

…ources

* CAPI, with v3 API, is limited to returning responses with max 5000 max matched resources; so we chunk them into request with max 5000 matched resources
* first try

cloudfoundry-util/src/main/java/org/cloudfoundry/util/ResourceMatchingUtilsV3.java

@@ -24,6 +24,9 @@
 import java.util.Collection;
 import java.util.Enumeration;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
 import org.apache.commons.compress.archivers.zip.ZipFile;
 import org.cloudfoundry.client.CloudFoundryClient;
@@ -125,24 +128,40 @@ private static Flux<ArtifactMetadata> getArtifactMetadataFromZip(Path applicatio
         return Flux.fromIterable(artifactMetadatas);
     }
 
-    private static Mono<ListMatchingResourcesResponse> requestListMatchingResources(
+    static Mono<ListMatchingResourcesResponse> requestListMatchingResources(
             CloudFoundryClient cloudFoundryClient, Collection<ArtifactMetadata> artifactMetadatas) {
-        ListMatchingResourcesRequest request =
-                artifactMetadatas.stream()
-                        .reduce(
-                                ListMatchingResourcesRequest.builder(),
-                                (builder, artifactMetadata) ->
-                                        builder.resource(
-                                                MatchedResource.builder()
-                                                        .checksum(artifactMetadata.getChecksum())
-                                                        .mode(artifactMetadata.getPermissions())
-                                                        .size(artifactMetadata.getSize())
-                                                        .path(artifactMetadata.getPath())
-                                                        .build()),
-                                (a, b) -> a.addAllResources(b.build().getResources()))
-                        .build();
-
-        return cloudFoundryClient.resourceMatchV3().list(request);
+        Collection<List<ArtifactMetadata>> chunksOfArtifactMetadatas = chunkArtifactMetadatas(artifactMetadatas, 5000);
+
+        List<ListMatchingResourcesResponse> matchingResourcesResponse = new ArrayList<>();
+
+        chunksOfArtifactMetadatas.forEach(chunkOfArtifactMetadatas -> {
+            ListMatchingResourcesRequest request =
+                    chunkOfArtifactMetadatas.stream()
+                            .reduce(
+                                    ListMatchingResourcesRequest.builder(),
+                                    (builder, artifactMetadata) ->
+                                            builder.resource(
+                                                    MatchedResource.builder()
+                                                            .checksum(artifactMetadata.getChecksum())
+                                                            .mode(artifactMetadata.getPermissions())
+                                                            .size(artifactMetadata.getSize())
+                                                            .path(artifactMetadata.getPath())
+                                                            .build()),
+                                    (a, b) -> a.addAllResources(b.build().getResources()))
+                            .build();
+
+            matchingResourcesResponse.add(cloudFoundryClient.resourceMatchV3().list(request).block());
+        });
+        List<MatchedResource> collect = matchingResourcesResponse.stream().map(listMatchingResourcesResponse -> listMatchingResourcesResponse.getResources()).flatMap(Collection::stream).collect(Collectors.toList());
+        ListMatchingResourcesResponse listMatchingResourcesResponse = ListMatchingResourcesResponse.builder().resources(collect).build();
+        Mono<ListMatchingResourcesResponse> listMatchingResourcesResponseMono = Mono.just(listMatchingResourcesResponse);
+        return listMatchingResourcesResponseMono;
+    }
+
+    private static Collection<List<ArtifactMetadata>> chunkArtifactMetadatas(Collection<ArtifactMetadata> artifactMetadatas, int maxNumberOfArtifacts) {
+        final AtomicInteger counter = new AtomicInteger();
+        List<ArtifactMetadata> metadataList = artifactMetadatas.stream().collect(Collectors.toList());
+        return metadataList.stream().collect(Collectors.groupingBy(artifactMetadata -> counter.getAndIncrement() / maxNumberOfArtifacts)).values();
     }
 
     /**

cloudfoundry-util/src/test/java/org/cloudfoundry/util/ResourceMatchingUtilsV3Test.java

@@ -0,0 +1,63 @@
+package org.cloudfoundry.util;
+
+import org.cloudfoundry.client.CloudFoundryClient;
+import org.cloudfoundry.client.v3.Checksum;
+import org.cloudfoundry.client.v3.resourcematch.ListMatchingResourcesRequest;
+import org.cloudfoundry.client.v3.resourcematch.ListMatchingResourcesResponse;
+import org.cloudfoundry.client.v3.resourcematch.ResourceMatchV3;
+import org.junit.jupiter.api.Test;
+import reactor.core.publisher.Mono;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.stream.IntStream;
+
+import static org.cloudfoundry.util.ResourceMatchingUtilsV3.requestListMatchingResources;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class ResourceMatchingUtilsV3Test {
+
+
+	@Test
+	void requestListMatchingResources5() {
+		CloudFoundryClient cloudFoundryClient = mock(CloudFoundryClient.class);
+		Collection<ResourceMatchingUtilsV3.ArtifactMetadata> artifactMetadatas = new ArrayList<>();
+
+		IntStream.range(0,5).forEach(value -> {
+			artifactMetadatas.add(new ResourceMatchingUtilsV3.ArtifactMetadata(Checksum.builder().value("pif" + value).build(), "path"+value, "0644", value));
+		});
+		when(cloudFoundryClient.resourceMatchV3()).thenReturn(new ResourceMatchV3() {
+			@Override
+			public Mono<ListMatchingResourcesResponse> list(ListMatchingResourcesRequest request) {
+				return Mono.just(ListMatchingResourcesResponse.builder().addAllResources(request.getResources()).build());
+			}
+		});
+
+		ListMatchingResourcesResponse listMatchingResourcesResponse = requestListMatchingResources(cloudFoundryClient, artifactMetadatas).block();
+		assertEquals(5, listMatchingResourcesResponse.getResources().size());
+
+	}
+
+	@Test
+	void requestListMatchingResources15001() {
+		CloudFoundryClient cloudFoundryClient = mock(CloudFoundryClient.class);
+		Collection<ResourceMatchingUtilsV3.ArtifactMetadata> artifactMetadatas = new ArrayList<>();
+
+		IntStream.range(0,15001).forEach(value -> {
+			artifactMetadatas.add(new ResourceMatchingUtilsV3.ArtifactMetadata(Checksum.builder().value("pif" + value).build(), "path"+value, "0644", value));
+		});
+		when(cloudFoundryClient.resourceMatchV3()).thenReturn(new ResourceMatchV3() {
+			@Override
+			public Mono<ListMatchingResourcesResponse> list(ListMatchingResourcesRequest request) {
+				return Mono.just(ListMatchingResourcesResponse.builder().addAllResources(request.getResources()).build());
+			}
+		});
+
+		ListMatchingResourcesResponse listMatchingResourcesResponse = requestListMatchingResources(cloudFoundryClient, artifactMetadatas).block();
+		assertEquals(15001, listMatchingResourcesResponse.getResources().size());
+
+	}
+
+}