Merge pull request #526 from wblachowski/feature/sending-urls-in-pack…

…ages

Add sending urls in packets

CHANGELOG.md

@@ -10,6 +10,7 @@ All notable changes to AET will be documented in this file.
 ## Unreleased
 
 **List of changes that are finished but not yet released in any final version.**
+- [PR-526](https://github.com/Cognifide/aet/pull/526) Added sending urls to collectors in packets. ([#431](https://github.com/Cognifide/aet/issues/431))
 
 - [PR-565](https://github.com/Cognifide/aet/pull/565) Fixing styles of note-editor icon ([#371](https://github.com/Cognifide/aet/issues/371))
 - [PR-567](https://github.com/Cognifide/aet/pull/567) Added missing tooltips in the report - "Previous url" and "Next url" ([#566](https://github.com/Cognifide/aet/issues/566))

core/runner/src/main/java/com/cognifide/aet/runner/processing/data/UrlPacket.java

@@ -0,0 +1,66 @@
+/**
+ * AET
+ *
+ * Copyright (C) 2013 Cognifide Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.cognifide.aet.runner.processing.data;
+
+import com.cognifide.aet.communication.api.metadata.Suite;
+import com.cognifide.aet.communication.api.metadata.Test;
+import com.cognifide.aet.communication.api.metadata.Url;
+import java.util.ArrayList;
+import java.util.List;
+
+public class UrlPacket {
+
+  private final Suite suite;
+  private final Test test;
+  private final List<Url> urls = new ArrayList<>();
+
+  public UrlPacket(Suite suite, Test test) {
+    this.suite = suite;
+    this.test = test;
+  }
+
+  public void addUrl(Url url) {
+    urls.add(url);
+  }
+
+  public String getCompany() {
+    return suite.getCompany();
+  }
+
+  public String getProject() {
+    return suite.getProject();
+  }
+
+  public String getName() {
+    return suite.getName();
+  }
+
+  public String getTestName() {
+    return test.getName();
+  }
+
+  public String getProxy() {
+    return test.getProxy();
+  }
+
+  public String getPreferredBrowserId() {
+    return test.getPreferredBrowserId();
+  }
+
+  public List<Url> getUrls() {
+    return urls;
+  }
+}

core/runner/src/main/java/com/cognifide/aet/runner/processing/data/wrappers/RunIndexWrapper.java

@@ -17,8 +17,8 @@
 
 import com.cognifide.aet.communication.api.metadata.Test;
 import com.cognifide.aet.communication.api.metadata.Url;
-import com.cognifide.aet.communication.api.wrappers.MetadataRunDecorator;
 import com.cognifide.aet.communication.api.wrappers.Run;
+import com.cognifide.aet.runner.processing.data.UrlPacket;
 import java.util.List;
 import java.util.Optional;
 
@@ -61,7 +61,7 @@ public Run<T> get() {
     return objectToRunWrapper;
   }
 
-  public abstract List<MetadataRunDecorator<Url>> getUrls();
+  public abstract List<UrlPacket> getUrlPackets();
 
   public abstract int countUrls();
 

core/runner/src/main/java/com/cognifide/aet/runner/processing/data/wrappers/SuiteRunIndexWrapper.java

@@ -18,10 +18,10 @@
 import com.cognifide.aet.communication.api.metadata.Suite;
 import com.cognifide.aet.communication.api.metadata.Test;
 import com.cognifide.aet.communication.api.metadata.Url;
-import com.cognifide.aet.communication.api.wrappers.MetadataRunDecorator;
 import com.cognifide.aet.communication.api.wrappers.Run;
-import com.cognifide.aet.communication.api.wrappers.UrlRunWrapper;
+import com.cognifide.aet.runner.processing.data.UrlPacket;
 import java.util.ArrayList;
+import java.util.List;
 
 public class SuiteRunIndexWrapper extends RunIndexWrapper<Suite> {
 
@@ -30,17 +30,18 @@ public SuiteRunIndexWrapper(Run<Suite> objectToRunWrapper) {
   }
 
   @Override
-  public ArrayList<MetadataRunDecorator<Url>> getUrls() {
+  public List<UrlPacket> getUrlPackets() {
     Suite suite = objectToRunWrapper.getObjectToRun();
-    ArrayList<MetadataRunDecorator<Url>> urls = new ArrayList<>();
+    List<UrlPacket> packets = new ArrayList<>();
     for (Test test : suite.getTests()) {
-      for(Url url : test.getUrls()){
+      UrlPacket packet = new UrlPacket(suite, test);
+      for (Url url : test.getUrls()) {
         cleanUrlFromExecutionData(url);
-        UrlRunWrapper urlRunWrapper = new UrlRunWrapper(url, test);
-        urls.add(new MetadataRunDecorator<>(urlRunWrapper, suite));
+        packet.addUrl(url);
       }
+      packets.add(packet);
     }
-    return urls;
+    return packets;
   }
 
   @Override

core/runner/src/main/java/com/cognifide/aet/runner/processing/data/wrappers/TestRunIndexWrapper.java

@@ -15,12 +15,13 @@
  */
 package com.cognifide.aet.runner.processing.data.wrappers;
 
+import com.cognifide.aet.communication.api.metadata.Suite;
 import com.cognifide.aet.communication.api.metadata.Test;
 import com.cognifide.aet.communication.api.metadata.Url;
-import com.cognifide.aet.communication.api.wrappers.MetadataRunDecorator;
 import com.cognifide.aet.communication.api.wrappers.Run;
-import com.cognifide.aet.communication.api.wrappers.UrlRunWrapper;
-import java.util.ArrayList;
+import com.cognifide.aet.runner.processing.data.UrlPacket;
+import java.util.Collections;
+import java.util.List;
 
 public class TestRunIndexWrapper extends RunIndexWrapper<Test> {
 
@@ -29,15 +30,15 @@ public TestRunIndexWrapper(Run<Test> objectToRunWrapper) {
   }
 
   @Override
-  public ArrayList<MetadataRunDecorator<Url>> getUrls() {
-    ArrayList<MetadataRunDecorator<Url>> urls = new ArrayList<>();
+  public List<UrlPacket> getUrlPackets() {
+    Suite suite = objectToRunWrapper.getRealSuite();
     Test test = objectToRunWrapper.getObjectToRun();
+    UrlPacket packet = new UrlPacket(suite, test);
     for (Url url : test.getUrls()) {
       cleanUrlFromExecutionData(url);
-      UrlRunWrapper urlRunWrapper = new UrlRunWrapper(url, test);
-      urls.add(new MetadataRunDecorator<>(urlRunWrapper, objectToRunWrapper.getRealSuite()));
+      packet.addUrl(url);
     }
-    return urls;
+    return Collections.singletonList(packet);
   }
 
   @Override

core/runner/src/main/java/com/cognifide/aet/runner/processing/data/wrappers/UrlRunIndexWrapper.java

@@ -15,12 +15,13 @@
  */
 package com.cognifide.aet.runner.processing.data.wrappers;
 
+import com.cognifide.aet.communication.api.metadata.Suite;
 import com.cognifide.aet.communication.api.metadata.Test;
 import com.cognifide.aet.communication.api.metadata.Url;
-import com.cognifide.aet.communication.api.wrappers.MetadataRunDecorator;
 import com.cognifide.aet.communication.api.wrappers.Run;
-import com.cognifide.aet.communication.api.wrappers.UrlRunWrapper;
-import java.util.ArrayList;
+import com.cognifide.aet.runner.processing.data.UrlPacket;
+import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
 
 public class UrlRunIndexWrapper extends RunIndexWrapper<Url> {
@@ -30,17 +31,16 @@ public UrlRunIndexWrapper(Run<Url> objectToRunWrapper) {
   }
 
   @Override
-  public ArrayList<MetadataRunDecorator<Url>> getUrls() {
-    ArrayList<MetadataRunDecorator<Url>> urls = new ArrayList<>();
+  public List<UrlPacket> getUrlPackets() {
+    Suite suite = objectToRunWrapper.getRealSuite();
     Optional<Test> test = objectToRunWrapper.getRealSuite()
         .getTest(objectToRunWrapper.getTestName());
     if (test.isPresent()) {
-      Url url = objectToRunWrapper.getObjectToRun();
-      cleanUrlFromExecutionData(url);
-      UrlRunWrapper urlRunWrapper = new UrlRunWrapper(url, test.get());
-      urls.add(new MetadataRunDecorator<>(urlRunWrapper, objectToRunWrapper.getRealSuite()));
+      UrlPacket packet = new UrlPacket(suite, test.get());
+      packet.addUrl(objectToRunWrapper.getObjectToRun());
+      return Collections.singletonList(packet);
     }
-    return urls;
+    return Collections.emptyList();
   }
 
   @Override

core/runner/src/main/java/com/cognifide/aet/runner/processing/steps/CollectDispatcher.java

@@ -19,18 +19,18 @@
 import com.cognifide.aet.communication.api.messages.ProgressLog;
 import com.cognifide.aet.communication.api.metadata.Url;
 import com.cognifide.aet.communication.api.queues.JmsConnection;
-import com.cognifide.aet.communication.api.wrappers.MetadataRunDecorator;
-import com.cognifide.aet.communication.api.wrappers.UrlRunWrapper;
 import com.cognifide.aet.communication.api.queues.QueuesConstant;
 import com.cognifide.aet.runner.RunnerConfiguration;
 import com.cognifide.aet.runner.processing.TimeoutWatch;
+import com.cognifide.aet.runner.processing.data.UrlPacket;
 import com.cognifide.aet.runner.processing.data.wrappers.RunIndexWrapper;
 import com.cognifide.aet.runner.scheduler.CollectorJobSchedulerService;
 import com.cognifide.aet.runner.scheduler.MessageWithDestination;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Queues;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.Deque;
+import java.util.List;
 import javax.jms.JMSException;
 import javax.jms.Message;
 import javax.jms.ObjectMessage;
@@ -52,8 +52,10 @@ public class CollectDispatcher extends StepManager {
 
   public CollectDispatcher(TimeoutWatch timeoutWatch, JmsConnection jmsConnection,
       RunnerConfiguration runnerConfiguration,
-      CollectorJobSchedulerService collectorJobScheduler, RunIndexWrapper<?> runIndexWrapper) throws JMSException {
-    super(timeoutWatch, jmsConnection, runIndexWrapper.get().getCorrelationId(), runnerConfiguration.getMttl());
+      CollectorJobSchedulerService collectorJobScheduler, RunIndexWrapper<?> runIndexWrapper)
+      throws JMSException {
+    super(timeoutWatch, jmsConnection, runIndexWrapper.get().getCorrelationId(),
+        runnerConfiguration.getMttl());
     this.urlPackageSize = runnerConfiguration.getUrlPackageSize();
     this.collectorJobScheduler = collectorJobScheduler;
     this.runIndexWrapper = runIndexWrapper;
@@ -75,15 +77,15 @@ public void process() throws JMSException {
     Deque<MessageWithDestination> messagesQueue = Queues.newArrayDeque();
     LOGGER.info("Starting processing new Test Suite. CorrelationId: {} ", correlationId);
 
-    for (MetadataRunDecorator<Url> metadataRunDecorator : runIndexWrapper.getUrls()) {
-      UrlRunWrapper urlRunWrapper = (UrlRunWrapper) metadataRunDecorator.getRun();
-      final CollectorJobData data = new CollectorJobData(metadataRunDecorator.getCompany(),
-          metadataRunDecorator.getProject(), metadataRunDecorator.getName(), urlRunWrapper.getTestName(),
-          new ArrayList<>(Collections.singleton(urlRunWrapper.getObjectToRun())),
-          urlRunWrapper.getProxy(), urlRunWrapper.getPreferredBrowserId());
-      ObjectMessage message = session.createObjectMessage(data);
-      message.setJMSCorrelationID(correlationId);
-      messagesQueue.add(new MessageWithDestination(getQueueOut(), message, 1));
+    for (UrlPacket urlPacket : runIndexWrapper.getUrlPackets()) {
+      for (List<Url> urls : Lists.partition(urlPacket.getUrls(), urlPackageSize)) {
+        final CollectorJobData data = new CollectorJobData(urlPacket.getCompany(),
+            urlPacket.getProject(), urlPacket.getName(), urlPacket.getTestName(),
+            new ArrayList<>(urls), urlPacket.getProxy(), urlPacket.getPreferredBrowserId());
+        ObjectMessage message = session.createObjectMessage(data);
+        message.setJMSCorrelationID(correlationId);
+        messagesQueue.add(new MessageWithDestination(getQueueOut(), message, urls.size()));
+      }
     }
     collectorJobScheduler.add(messagesQueue, runIndexWrapper.get().getCorrelationId());
     LOGGER.info("MessagesQueue was added to collectorJobScheduler. CorrelationId: {} ",

core/runner/src/test/java/com/cognifide/aet/runner/processing/data/wrappers/SuiteRunIndexWrapperTest.java

@@ -17,13 +17,12 @@
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertThat;
-import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.when;
 
 import com.cognifide.aet.communication.api.metadata.Suite;
 import com.cognifide.aet.communication.api.metadata.Url;
-import com.cognifide.aet.communication.api.wrappers.MetadataRunDecorator;
 import com.cognifide.aet.communication.api.wrappers.Run;
+import com.cognifide.aet.runner.processing.data.UrlPacket;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -63,30 +62,29 @@ public void setUp() throws Exception {
         .of(new com.cognifide.aet.communication.api.metadata.Test("testName", "proxy", "chrome"));
     test2 = Optional
         .of(new com.cognifide.aet.communication.api.metadata.Test("testName", "proxy", "chrome"));
-    url = Optional.of(new Url("urlName","urlUrl","urlDomain"));
-    url2 = Optional.of(new Url("urlName2","urlUrl2","urlDomain2"));
+    url = Optional.of(new Url("urlName", "urlUrl", "urlDomain"));
+    url2 = Optional.of(new Url("urlName2", "urlUrl2", "urlDomain2"));
   }
 
   @Test
-  public void getUrls_expectZero() {
+  public void getUrlPackets_expectZero() {
     prepareZeroUrls();
-
-    ArrayList<MetadataRunDecorator<Url>> urlsResult = suiteRunIndexWrapper
-        .getUrls();
-    assertThat(urlsResult.size(), is(0));
+    List<UrlPacket> packets = suiteRunIndexWrapper.getUrlPackets();
+    Long urlsCount = packets.stream().map(UrlPacket::getUrls).mapToLong(List::size).sum();
+    assertThat(urlsCount, is(0L));
   }
 
   @Test
-  public void getUrls_expectThree() {
+  public void getUrlPackets_expectThree() {
     prepareThreeUrls();
 
-    ArrayList<MetadataRunDecorator<Url>> urlsResult = suiteRunIndexWrapper
-        .getUrls();
-    assertThat(urlsResult.size(), is(3));
+    List<UrlPacket> packets = suiteRunIndexWrapper.getUrlPackets();
+    Long urlsCount = packets.stream().map(UrlPacket::getUrls).mapToLong(List::size).sum();
+    assertThat(urlsCount, is(3L));
   }
 
   @Test
-  public void countUrls_expectZero(){
+  public void countUrls_expectZero() {
     prepareZeroUrls();
 
     assertThat(suiteRunIndexWrapper.countUrls(), is(0));
@@ -99,14 +97,14 @@ public void countUrls_expectThree() {
     assertThat(suiteRunIndexWrapper.countUrls(), is(3));
   }
 
-  private void prepareZeroUrls(){
+  private void prepareZeroUrls() {
     List<com.cognifide.aet.communication.api.metadata.Test> tests = new ArrayList<>();
     tests.add(test.get());
     tests.add(test2.get());
     when(suite.getTests()).thenReturn(tests);
   }
 
-  private void prepareThreeUrls(){
+  private void prepareThreeUrls() {
     Set<Url> firstUrlsSet = new HashSet<>();
     firstUrlsSet.add(url.get());
     test.get().setUrls(firstUrlsSet);