Skip to content

Commit

Permalink
CDPD-45931: ORC-484: Update PhysicalWriter to support encryption.
Browse files Browse the repository at this point in the history
Fixes apache#402

Signed-off-by: Owen O'Malley <omalley@apache.org>
Change-Id: I67ebedc34bf08811a369b2da2050c1664b787fb6
  • Loading branch information
omalley authored and Dmitriy Fingerman committed Nov 17, 2022
1 parent 82af845 commit 3c4c1d7
Show file tree
Hide file tree
Showing 14 changed files with 771 additions and 123 deletions.
35 changes: 24 additions & 11 deletions java/core/src/java/org/apache/orc/PhysicalWriter.java
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
Expand All @@ -22,6 +22,8 @@
import java.nio.ByteBuffer;

import org.apache.orc.impl.StreamName;
import org.apache.orc.impl.writer.StreamOptions;
import org.apache.orc.impl.writer.WriterEncryptionVariant;

/**
* This interface separates the physical layout of ORC files from the higher
Expand All @@ -39,7 +41,6 @@ interface OutputReceiver {
* Output the given buffer to the final destination
*
* @param buffer the buffer to output
* @throws IOException
*/
void output(ByteBuffer buffer) throws IOException;

Expand All @@ -48,24 +49,22 @@ interface OutputReceiver {
*/
void suppress();
}

/**
* Writes the header of the file, which consists of the magic "ORC" bytes.
* @throws IOException
*/
void writeHeader() throws IOException;

/**
* Create an OutputReceiver for the given name.
* @param name the name of the stream
* @throws IOException
*/
OutputReceiver createDataStream(StreamName name) throws IOException;

/**
* Write an index in the given stream name.
* @param name the name of the stream
* @param index the bloom filter to write
* @param codec the compression codec to use
*/
void writeIndex(StreamName name,
OrcProto.RowIndex.Builder index) throws IOException;
Expand All @@ -74,7 +73,6 @@ void writeIndex(StreamName name,
* Write a bloom filter index in the given stream name.
* @param name the name of the stream
* @param bloom the bloom filter to write
* @param codec the compression codec to use
*/
void writeBloomFilter(StreamName name,
OrcProto.BloomFilterIndex.Builder bloom) throws IOException;
Expand All @@ -89,6 +87,16 @@ void writeBloomFilter(StreamName name,
void finalizeStripe(OrcProto.StripeFooter.Builder footer,
OrcProto.StripeInformation.Builder dirEntry) throws IOException;

/**
* Write a stripe or file statistics to the file.
* @param name the name of the stream
* @param statistics the statistics to write
* @throws IOException
*/
void writeStatistics(StreamName name,
OrcProto.ColumnStatistics.Builder statistics
) throws IOException;

/**
* Writes out the file metadata.
* @param builder Metadata builder to finalize and write.
Expand Down Expand Up @@ -122,19 +130,24 @@ void finalizeStripe(OrcProto.StripeFooter.Builder footer,
* @param stripe Stripe data buffer.
* @param dirEntry File metadata entry for the stripe, to be updated with
* relevant data.
* @throws IOException
*/
void appendRawStripe(ByteBuffer stripe,
OrcProto.StripeInformation.Builder dirEntry
) throws IOException;

/** Gets a compression codec used by this writer. */
CompressionCodec getCompressionCodec();

/**
* Get the number of bytes for a file in a givem column.
* @param column column from which to get file size
* @param variant the encryption variant to check
* @return number of bytes for the given column
*/
long getFileBytes(int column);
long getFileBytes(int column, WriterEncryptionVariant variant);

/**
* Get the unencrypted stream options for this file. This class needs the
* stream options to write the indexes and footers.
*
* Additionally, the LLAP CacheWriter wants to disable the generic compression.
*/
StreamOptions getStreamOptions();
}
Loading

0 comments on commit 3c4c1d7

Please sign in to comment.