Align output to buffer size in rcfile writer

presto-rcfile/src/main/java/com/facebook/presto/rcfile/BufferedOutputStreamSliceOutput.java

@@ -36,7 +36,7 @@ public class BufferedOutputStreamSliceOutput
         extends SliceOutput
 {
     private static final int INSTANCE_SIZE = ClassLayout.parseClass(BufferedOutputStreamSliceOutput.class).instanceSize();
-    private static final int MINIMUM_CHUNK_SIZE = 4096;
+    private static final int CHUNK_SIZE = 4096;
 
     private final OutputStream outputStream;
 
@@ -52,15 +52,14 @@ public class BufferedOutputStreamSliceOutput
      */
     private int bufferPosition;
 
-    public BufferedOutputStreamSliceOutput(OutputStream outputStream, int bufferSize)
+    public BufferedOutputStreamSliceOutput(OutputStream outputStream)
     {
-        checkArgument(bufferSize >= MINIMUM_CHUNK_SIZE, "minimum buffer size of " + MINIMUM_CHUNK_SIZE + " required");
         if (outputStream == null) {
             throw new NullPointerException("outputStream is null");
         }
 
         this.outputStream = outputStream;
-        this.buffer = new byte[bufferSize];
+        this.buffer = new byte[CHUNK_SIZE];
         this.slice = Slices.wrappedBuffer(buffer);
     }
 
@@ -173,13 +172,28 @@ public void writeBytes(Slice source)
     @Override
     public void writeBytes(Slice source, int sourceIndex, int length)
     {
-        // Write huge chunks direct to OutputStream
-        if (length >= MINIMUM_CHUNK_SIZE) {
-            flushBufferToOutputStream();
-            writeToOutputStream(source, sourceIndex, length);
-            bufferOffset += length;
+        if (length >= CHUNK_SIZE) {
+            if (bufferPosition > 0) {
+                // fill up the current buffer
+                int flushLength = getFreeBufferLength();
+                slice.setBytes(bufferPosition, source, sourceIndex, flushLength);
+                bufferPosition = CHUNK_SIZE;
+                flushBufferToOutputStream();
+                sourceIndex += flushLength;
+                length -= flushLength;
+            }
+
+            // line up the chunk to chunk size and flush directly to OutputStream
+            while (length >= CHUNK_SIZE) {
+                writeToOutputStream(source, sourceIndex, CHUNK_SIZE);
+                sourceIndex += CHUNK_SIZE;
+                length -= CHUNK_SIZE;
+                bufferOffset += CHUNK_SIZE;
+            }
         }
-        else {
+
+        if (length > 0) {
+            // buffer the remaining data
             ensureWritableBytes(length);
             slice.setBytes(bufferPosition, source, sourceIndex, length);
             bufferPosition += length;
@@ -195,13 +209,28 @@ public void writeBytes(byte[] source)
     @Override
     public void writeBytes(byte[] source, int sourceIndex, int length)
     {
-        // Write huge chunks direct to OutputStream
-        if (length >= MINIMUM_CHUNK_SIZE) {
-            flushBufferToOutputStream();
-            writeToOutputStream(source, sourceIndex, length);
-            bufferOffset += length;
+        if (length >= CHUNK_SIZE) {
+            if (bufferPosition > 0) {
+                // fill up the current buffer
+                int flushLength = getFreeBufferLength();
+                slice.setBytes(bufferPosition, source, sourceIndex, flushLength);
+                bufferPosition = CHUNK_SIZE;
+                flushBufferToOutputStream();
+                sourceIndex += flushLength;
+                length -= flushLength;
+            }
+
+            // line up the chunk to chunk size and flush directly to OutputStream
+            while (length >= CHUNK_SIZE) {
+                writeToOutputStream(source, sourceIndex, CHUNK_SIZE);
+                sourceIndex += CHUNK_SIZE;
+                length -= CHUNK_SIZE;
+                bufferOffset += CHUNK_SIZE;
+            }
         }
-        else {
+
+        if (length > 0) {
+            // buffer the remaining data
             ensureWritableBytes(length);
             slice.setBytes(bufferPosition, source, sourceIndex, length);
             bufferPosition += length;
@@ -317,17 +346,22 @@ public String toString()
         return builder.toString();
     }
 
+    private int getFreeBufferLength()
+    {
+        return CHUNK_SIZE - bufferPosition;
+    }
+
     private void ensureWritableBytes(int minWritableBytes)
     {
-        if (bufferPosition + minWritableBytes > slice.length()) {
+        if (minWritableBytes > getFreeBufferLength()) {
             flushBufferToOutputStream();
         }
     }
 
     private int ensureBatchSize(int length)
     {
-        ensureWritableBytes(Math.min(MINIMUM_CHUNK_SIZE, length));
-        return Math.min(length, slice.length() - bufferPosition);
+        ensureWritableBytes(Math.min(CHUNK_SIZE, length));
+        return Math.min(length, CHUNK_SIZE - bufferPosition);
     }
 
     private void flushBufferToOutputStream()

presto-rcfile/src/main/java/com/facebook/presto/rcfile/RcFileCompressor.java

@@ -48,7 +48,7 @@ final class CompressedSliceOutput
          */
         public CompressedSliceOutput(OutputStream compressionStream, ChunkedSliceOutput bufferedOutput, Supplier<CompressedSliceOutput> resetFactory, Runnable onDestroy)
         {
-            super(compressionStream, 4096);
+            super(compressionStream);
             this.bufferedOutput = requireNonNull(bufferedOutput, "bufferedOutput is null");
             this.resetFactory = requireNonNull(resetFactory, "resetFactory is null");
             this.onDestroy = requireNonNull(onDestroy, "onDestroy is null");

presto-rcfile/src/test/java/com/facebook/presto/rcfile/TestBufferedOutputStreamSliceOutput.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.rcfile;
+
+import io.airlift.slice.Slice;
+import io.airlift.slice.Slices;
+import org.testng.annotations.Test;
+
+import java.io.ByteArrayOutputStream;
+
+import static io.airlift.testing.Assertions.assertLessThanOrEqual;
+import static org.testng.Assert.assertEquals;
+
+public class TestBufferedOutputStreamSliceOutput
+{
+    @Test
+    public void testWriteBytes()
+            throws Exception
+    {
+        // fill up some input bytes
+        int length = 65536;
+        byte[] inputArray = new byte[length];
+        for (int i = 0; i < length; i++) {
+            inputArray[i] = (byte) (i % 128);
+        }
+
+        // pick some offsets to make the inputs into different chunks
+        int[] offsets = {0, 100, 545, 1024, 2049, 2050, 2051, 2151, 10480, 20042, 20100, 40001, 65536};
+
+        // check byte array version
+        MockOutputStream byteOutputStream = new MockOutputStream(length);
+        BufferedOutputStreamSliceOutput output = new BufferedOutputStreamSliceOutput(byteOutputStream);
+        for (int i = 0; i < offsets.length - 1; i++) {
+            output.writeBytes(inputArray, offsets[i], offsets[i + 1] - offsets[i]);
+        }
+        // ignore the last flush size check
+        output.flush();
+        assertEquals(byteOutputStream.toByteArray(), inputArray);
+        byteOutputStream.close();
+
+        // check slice version
+        byteOutputStream = new MockOutputStream(length);
+        Slice inputSlice = Slices.wrappedBuffer(inputArray);
+        output = new BufferedOutputStreamSliceOutput(byteOutputStream);
+        for (int i = 0; i < offsets.length - 1; i++) {
+            output.writeBytes(inputSlice, offsets[i], offsets[i + 1] - offsets[i]);
+        }
+        // ignore the last flush size check
+        output.flush();
+        assertEquals(byteOutputStream.toByteArray(), inputArray);
+        byteOutputStream.close();
+    }
+
+    private class MockOutputStream
+            extends ByteArrayOutputStream
+    {
+        public MockOutputStream(int length)
+        {
+            super(length);
+        }
+
+        @Override
+        public void write(byte[] source, int sourceIndex, int length)
+        {
+            assertLessThanOrEqual(length, 4096);
+            super.write(source, sourceIndex, length);
+        }
+    }
+}