From 5f6e72d2ed099ea72b4f665d7aac2c34282762a3 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Wed, 18 Nov 2020 11:42:40 -0600 Subject: [PATCH] Issue #5605 - Adding more gzip consume all tests Signed-off-by: Joakim Erdfelt --- .../jetty/test/GzipWithSendErrorTest.java | 250 +++++++++++++++++- .../src/test/resources/zeros.gz.gz | Bin 0 -> 2773 bytes 2 files changed, 243 insertions(+), 7 deletions(-) create mode 100644 tests/test-integration/src/test/resources/zeros.gz.gz diff --git a/tests/test-integration/src/test/java/org/eclipse/jetty/test/GzipWithSendErrorTest.java b/tests/test-integration/src/test/java/org/eclipse/jetty/test/GzipWithSendErrorTest.java index 2a6c0c938bac..63e77b156e67 100644 --- a/tests/test-integration/src/test/java/org/eclipse/jetty/test/GzipWithSendErrorTest.java +++ b/tests/test-integration/src/test/java/org/eclipse/jetty/test/GzipWithSendErrorTest.java @@ -20,7 +20,16 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; @@ -28,35 +37,56 @@ import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.client.api.Response; +import org.eclipse.jetty.client.api.Result; import org.eclipse.jetty.client.util.BytesContentProvider; +import org.eclipse.jetty.client.util.DeferredContentProvider; import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.http.HttpMethod; +import org.eclipse.jetty.server.HttpChannel; +import org.eclipse.jetty.server.HttpConnection; +import org.eclipse.jetty.server.HttpInput; +import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.gzip.GzipHandler; import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.toolchain.test.MavenTestingUtils; import org.eclipse.jetty.util.IO; import org.eclipse.jetty.util.component.LifeCycle; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import static java.nio.charset.StandardCharsets.UTF_8; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class GzipWithSendErrorTest { private Server server; private HttpClient client; + private ServerConnector connector; + + private static void onComplete(Result result) + { + } @BeforeEach public void setup() throws Exception { server = new Server(); - ServerConnector connector = new ServerConnector(server); + connector = new ServerConnector(server); connector.setPort(0); server.addConnector(connector); @@ -100,7 +130,6 @@ public void testGzipNormalErrorNormal() throws Exception response = client.newRequest(serverURI.resolve("/submit")) .method(HttpMethod.POST) .header(HttpHeader.CONTENT_ENCODING, "gzip") - .header(HttpHeader.ACCEPT_ENCODING, "gzip") .content(new BytesContentProvider("text/plain", compressed("normal-A"))) .send(); @@ -110,7 +139,6 @@ public void testGzipNormalErrorNormal() throws Exception response = client.newRequest(serverURI.resolve("/fail")) .method(HttpMethod.POST) .header(HttpHeader.CONTENT_ENCODING, "gzip") - .header(HttpHeader.ACCEPT_ENCODING, "gzip") .content(new BytesContentProvider("text/plain", compressed("normal-B"))) .send(); @@ -120,7 +148,6 @@ public void testGzipNormalErrorNormal() throws Exception response = client.newRequest(serverURI.resolve("/submit")) .method(HttpMethod.POST) .header(HttpHeader.CONTENT_ENCODING, "gzip") - .header(HttpHeader.ACCEPT_ENCODING, "gzip") .content(new BytesContentProvider("text/plain", compressed("normal-C"))) .send(); @@ -139,6 +166,212 @@ private byte[] compressed(String content) throws IOException } } + /** + * Make request with compressed content. + *

+ * Request contains (roughly) 1 MB of request network data. + * Which unpacks to 1 GB of zeros. + *

+ *

+ * This test is to ensure that consumeAll only reads the network data, + * and doesn't process it through the interceptors. + *

+ */ + @Test + public void testGzipConsumeAllContentLengthBlocking() throws Exception + { + URI serverURI = server.getURI(); + + CountDownLatch serverRequestCompleteLatch = new CountDownLatch(1); + // count of bytes against network read + AtomicLong inputBytesIn = new AtomicLong(0L); + AtomicLong inputContentReceived = new AtomicLong(0L); + // count of bytes against API read + AtomicLong inputContentConsumed = new AtomicLong(0L); + + connector.addBean(new HttpChannel.Listener() + { + @Override + public void onComplete(Request request) + { + HttpConnection connection = (HttpConnection)request.getHttpChannel().getConnection(); + HttpInput httpInput = request.getHttpInput(); + inputContentConsumed.set(httpInput.getContentConsumed()); + inputContentReceived.set(httpInput.getContentReceived()); + inputBytesIn.set(connection.getBytesIn()); + serverRequestCompleteLatch.countDown(); + } + }); + + // This is a doubly-compressed (with gzip) test resource. + // There's no point putting into SCM the full 1MB file, when the + // 3KB version is adequate. + Path zerosCompressed = MavenTestingUtils.getTestResourcePathFile("zeros.gz.gz"); + byte[] compressedRequest; + try (InputStream in = Files.newInputStream(zerosCompressed); + GZIPInputStream gzipIn = new GZIPInputStream(in); + ByteArrayOutputStream out = new ByteArrayOutputStream()) + { + IO.copy(gzipIn, out); + compressedRequest = out.toByteArray(); + } + + int sizeActuallySent = compressedRequest.length / 2; + ByteBuffer start = ByteBuffer.wrap(compressedRequest, 0, sizeActuallySent); + DeferredContentProvider contentProvider = new DeferredContentProvider(start) + { + @Override + public long getLength() + { + return compressedRequest.length; + } + }; + AtomicReference clientResponseRef = new AtomicReference<>(); + CountDownLatch clientResponseSuccessLatch = new CountDownLatch(1); + CountDownLatch clientResultComplete = new CountDownLatch(1); + + client.newRequest(serverURI.resolve("/fail")) + .method(HttpMethod.POST) + .header(HttpHeader.CONTENT_TYPE, "application/octet-stream") + .header(HttpHeader.CONTENT_ENCODING, "gzip") + .content(contentProvider) + .onResponseSuccess((response) -> + { + clientResponseRef.set(response); + clientResponseSuccessLatch.countDown(); + }) + .send((result) -> clientResultComplete.countDown()); + + assertTrue(clientResponseSuccessLatch.await(5, TimeUnit.SECONDS), "Result not received"); + Response response = clientResponseRef.get(); + assertEquals(400, response.getStatus(), "Response status on /fail"); + + assertEquals("close", response.getHeaders().get(HttpHeader.CONNECTION), "Response Connection header"); + + // Await for server side to complete the request + assertTrue(serverRequestCompleteLatch.await(5, TimeUnit.SECONDS), "Request complete never occurred?"); + + // System.out.printf("Input Content Consumed: %,d%n", inputContentConsumed.get()); + // System.out.printf("Input Content Received: %,d%n", inputContentReceived.get()); + // System.out.printf("Input BytesIn Count: %,d%n", inputBytesIn.get()); + + // Servlet didn't read body content + assertThat("Request Input Content Consumed not have been used", inputContentConsumed.get(), is(0L)); + // Network reads + assertThat("Request Input Content Received should have seen content", inputContentReceived.get(), greaterThan(0L)); + assertThat("Request Input Content Received less then initial buffer", inputContentReceived.get(), lessThanOrEqualTo((long)sizeActuallySent)); + assertThat("Request Connection BytesIn should have some minimal data", inputBytesIn.get(), greaterThanOrEqualTo(1024L)); + assertThat("Request Connection BytesIn read should not have read all of the data", inputBytesIn.get(), lessThanOrEqualTo((long)sizeActuallySent)); + + // Now provide rest + contentProvider.offer(ByteBuffer.wrap(compressedRequest, sizeActuallySent, compressedRequest.length - sizeActuallySent)); + contentProvider.close(); + + assertTrue(clientResultComplete.await(5, TimeUnit.SECONDS)); + } + + /** + * Make request with compressed content. + *

+ * Request contains (roughly) 1 MB of request network data. + * Which unpacks to 1 GB of zeros. + *

+ *

+ * This test is to ensure that consumeAll only reads the network data, + * and doesn't process it through the interceptors. + *

+ */ + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testGzipConsumeAllChunkedBlockingOnLastBuffer(boolean read) throws Exception + { + URI serverURI = server.getURI(); + + CountDownLatch serverRequestCompleteLatch = new CountDownLatch(1); + // count of bytes against network read + AtomicLong inputBytesIn = new AtomicLong(0L); + AtomicLong inputContentReceived = new AtomicLong(0L); + // count of bytes against API read + AtomicLong inputContentConsumed = new AtomicLong(0L); + + connector.addBean(new HttpChannel.Listener() + { + @Override + public void onComplete(Request request) + { + HttpConnection connection = (HttpConnection)request.getHttpChannel().getConnection(); + HttpInput httpInput = request.getHttpInput(); + inputContentConsumed.set(httpInput.getContentConsumed()); + inputContentReceived.set(httpInput.getContentReceived()); + inputBytesIn.set(connection.getBytesIn()); + serverRequestCompleteLatch.countDown(); + } + }); + + // This is a doubly-compressed (with gzip) test resource. + // There's no point putting into SCM the full 1MB file, when the + // 3KB version is adequate. + Path zerosCompressed = MavenTestingUtils.getTestResourcePathFile("zeros.gz.gz"); + byte[] compressedRequest; + try (InputStream in = Files.newInputStream(zerosCompressed); + GZIPInputStream gzipIn = new GZIPInputStream(in); + ByteArrayOutputStream out = new ByteArrayOutputStream()) + { + IO.copy(gzipIn, out); + compressedRequest = out.toByteArray(); + } + + int sizeActuallySent = compressedRequest.length / 2; + ByteBuffer start = ByteBuffer.wrap(compressedRequest, 0, sizeActuallySent); + DeferredContentProvider contentProvider = new DeferredContentProvider(start); + AtomicReference clientResponseRef = new AtomicReference<>(); + CountDownLatch clientResponseSuccessLatch = new CountDownLatch(1); + CountDownLatch clientResultComplete = new CountDownLatch(1); + + URI uri = serverURI.resolve("/fail?read=" + read); + + client.newRequest(uri) + .method(HttpMethod.POST) + .header(HttpHeader.CONTENT_TYPE, "application/octet-stream") + .header(HttpHeader.CONTENT_ENCODING, "gzip") + .content(contentProvider) + .onResponseSuccess((response) -> + { + clientResponseRef.set(response); + clientResponseSuccessLatch.countDown(); + }) + .send((result) -> clientResultComplete.countDown()); + + assertTrue(clientResponseSuccessLatch.await(5, TimeUnit.SECONDS), "Result not received"); + Response response = clientResponseRef.get(); + assertEquals(400, response.getStatus(), "Response status on /fail"); + + assertEquals("close", response.getHeaders().get(HttpHeader.CONNECTION), "Response Connection header"); + + // Await for server side to complete the request + assertTrue(serverRequestCompleteLatch.await(5, TimeUnit.SECONDS), "Request complete never occurred?"); + + // System.out.printf("Input Content Consumed: %,d%n", inputContentConsumed.get()); + // System.out.printf("Input Content Received: %,d%n", inputContentReceived.get()); + // System.out.printf("Input BytesIn Count: %,d%n", inputBytesIn.get()); + + long readCount = read ? 1L : 0L; + + // Servlet read of body content + assertThat("Request Input Content Consumed not have been used", inputContentConsumed.get(), is(readCount)); + // Network reads + assertThat("Request Input Content Received should have seen content", inputContentReceived.get(), greaterThan(0L)); + assertThat("Request Input Content Received less then initial buffer", inputContentReceived.get(), lessThanOrEqualTo((long)sizeActuallySent)); + assertThat("Request Connection BytesIn should have some minimal data", inputBytesIn.get(), greaterThanOrEqualTo(1024L)); + assertThat("Request Connection BytesIn read should not have read all of the data", inputBytesIn.get(), lessThanOrEqualTo((long)sizeActuallySent)); + + // Now provide rest + contentProvider.offer(ByteBuffer.wrap(compressedRequest, sizeActuallySent, compressedRequest.length - sizeActuallySent)); + contentProvider.close(); + + assertTrue(clientResultComplete.await(5, TimeUnit.SECONDS)); + } + public static class PostServlet extends HttpServlet { @Override @@ -146,8 +379,6 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I { resp.setCharacterEncoding("utf-8"); resp.setContentType("text/plain"); - resp.setHeader("X-Servlet", req.getServletPath()); - String reqBody = IO.toString(req.getInputStream(), UTF_8); resp.getWriter().append(reqBody); } @@ -158,7 +389,12 @@ public static class FailServlet extends HttpServlet @Override protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException { - resp.setHeader("X-Servlet", req.getServletPath()); + boolean read = Boolean.parseBoolean(req.getParameter("read")); + if (read) + { + int val = req.getInputStream().read(); + assertNotEquals(-1, val); + } // intentionally do not read request body here. resp.sendError(400); } diff --git a/tests/test-integration/src/test/resources/zeros.gz.gz b/tests/test-integration/src/test/resources/zeros.gz.gz new file mode 100644 index 0000000000000000000000000000000000000000..3446901899d5f1ad1aa912b665e939d1166c3ac6 GIT binary patch literal 2773 zcmb2|=3wBNzbT%9`R!dpE^A-e;}8FDitae7rWwRq*uT+N;r5byN?~HBn{v-hXq|BC znDC8AZ7043p|8YMHlAiYqU5^j=+&9pJVtvy*j$}5pSfn{{q^<#yXEcweq4XJ|5JRu zc9*<;olpIb^Yj1z?T-J)00ni=met3dHema4QRjT}e`%=bgXeiMS3_l3&m3NNT9z3q zsIvR*?b1Jr4~i|rcP2x{8d7e*|1Gecq5tVV%ctuYp`w%au1gi%#&GtN%{L^Kk5{cb zB$hC%J$AY^8&si3+3oGGD^x~}1ck?FSdJzlP&ydRJ)^}2s5BTYFGnj1hS3^ww6!rP un+D&O*WPZ?J@9M!Q~CW}_AGzozdnus_iy+8fA0%_=xfF9;hXx5p8)_!b>wdV literal 0 HcmV?d00001