ORC-751: [C++] Implement Predicate Pushdown for C++ Reader

.asf.yaml

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# https://cwiki.apache.org/confluence/display/INFRA/git+-+.asf.yaml+features
+---
+github:
+  description: "Apache ORC - the smallest, fastest columnar storage for Hadoop workloads"
+  homepage: https://orc.apache.org/
+  enabled_merge_buttons:
+    merge: false
+    squash: true
+    rebase: true

.github/workflows/build_and_test.yml

@@ -1,4 +1,4 @@
-name: master
+name: Build and test
 
 on:
   push:
@@ -11,13 +11,14 @@ on:
 jobs:
   build:
     name: "Build with Java ${{ matrix.java }}"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         java:
           - 1.8
           - 11
+          - 15
     env:
       MAVEN_OPTS: -Xmx2g
       MAVEN_SKIP_RC: true
@@ -40,7 +41,5 @@ jobs:
         mkdir -p ~/.m2
         mkdir build
         cd build
-        cmake ..
+        cmake -DANALYZE_JAVA=ON ..
         make package test-out
-        cd ../java
-        mvn apache-rat:check

.github/workflows/publish_snapshot.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   publish-snapshot:
+    if: github.repository == 'apache/orc'
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@master
@@ -15,10 +16,11 @@ jobs:
       with:
         java-version: 8
 
-    - name: Release Maven package
-      uses: samuelmeuli/action-maven-publish@v1
-      with:
-        directory: java
-        server_id: apache.snapshots.https
-        nexus_username: ${{ secrets.NEXUS_USER }}
-        nexus_password: ${{ secrets.NEXUS_PW }}
+    - name: Publish snapshot
+      env:
+        ASF_USERNAME: ${{ secrets.NEXUS_USER }}
+        ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
+      run: |
+        cd java
+        echo "<settings><servers><server><id>apache.snapshots.https</id><username>$ASF_USERNAME</username><password>$ASF_PASSWORD</password></server></servers></settings>" > settings.xml
+        mvn --settings settings.xml -DskipTests deploy

README.md

@@ -23,9 +23,9 @@ Releases:
 * Downloads: <a href="http://orc.apache.org/downloads">Apache ORC downloads</a>
 
 The current build status:
-* Master branch <a href="https://travis-ci.org/apache/orc/branches">
-![master build status](https://travis-ci.org/apache/orc.svg?branch=master)</a>
-* <a href="https://travis-ci.org/apache/orc/pull_requests">Pull Requests</a>
+* Master branch <a href="https://travis-ci.com/apache/orc/branches">
+![master build status](https://travis-ci.com/apache/orc.svg?branch=master)</a>
+* <a href="https://travis-ci.com/github/apache/orc/pull_requests">Pull Requests</a>
 
 
 Bug tracking: <a href="http://orc.apache.org/bugs">Apache Jira</a>
@@ -44,7 +44,7 @@ The subdirectories are:
 
 ### Building
 
-* Install java 1.7 or higher
+* Install java 1.8 or higher
 * Install maven 3 or higher
 * Install cmake
 

c++/include/orc/Common.hh

@@ -69,6 +69,8 @@ namespace orc {
     ORC_JAVA_WRITER = 0,
     ORC_CPP_WRITER = 1,
     PRESTO_WRITER = 2,
+    SCRITCHLEY_GO = 3,
+    TRINO_WRITER = 4,
     UNKNOWN_WRITER = INT32_MAX
   };
 

c++/include/orc/Reader.hh

@@ -23,6 +23,7 @@
 #include "orc/Common.hh"
 #include "orc/orc-config.hh"
 #include "orc/Statistics.hh"
+#include "orc/sargs/SearchArgument.hh"
 #include "orc/Type.hh"
 #include "orc/Vector.hh"
 
@@ -191,6 +192,11 @@ namespace orc {
      */
     RowReaderOptions& setEnableLazyDecoding(bool enable);
 
+    /**
+     * Set search argument for predicate push down
+     */
+    RowReaderOptions& searchArgument(std::unique_ptr<SearchArgument> sargs);
+
     /**
      * Should enable encoding block mode
      */
@@ -245,6 +251,11 @@ namespace orc {
      * What scale should all Hive 0.11 decimals be normalized to?
      */
     int32_t getForcedScaleOnHive11Decimal() const;
+
+    /**
+     * Get search argument for predicate push down
+     */
+    std::shared_ptr<SearchArgument> getSearchArgument() const;
   };
 
 
@@ -538,6 +549,12 @@ namespace orc {
      */
     virtual void seekToRow(uint64_t rowNumber) = 0;
 
+    /**
+     * If PPD is enabled, returns true and store number of selected RGs and
+     * number of evaluated RGs into the stats pair; otherwise returns false.
+     */
+    virtual bool getPPDStats(std::pair<uint64_t, uint64_t>& stats) const = 0;
+
   };
 }
 

c++/src/ColumnReader.cc

@@ -534,6 +534,9 @@ namespace orc {
     std::unordered_map<uint64_t, PositionProvider>& positions) {
     ColumnReader::seekToRowGroup(positions);
     inputStream->seek(positions.at(columnId));
+    // clear buffer state after seek
+    bufferEnd = nullptr;
+    bufferPointer = nullptr;
   }
 
   class StringDictionaryColumnReader: public ColumnReader {
@@ -838,6 +841,9 @@ namespace orc {
     ColumnReader::seekToRowGroup(positions);
     blobStream->seek(positions.at(columnId));
     lengthRle->seek(positions.at(columnId));
+    // clear buffer state after seek
+    lastBuffer = nullptr;
+    lastBufferLength = 0;
   }
 
   class StructColumnReader: public ColumnReader {
@@ -1553,6 +1559,9 @@ namespace orc {
     ColumnReader::seekToRowGroup(positions);
     valueStream->seek(positions.at(columnId));
     scaleDecoder->seek(positions.at(columnId));
+    // clear buffer state after seek
+    buffer = nullptr;
+    bufferEnd = nullptr;
   }
 
   class Decimal128ColumnReader: public Decimal64ColumnReader {

c++/src/Options.hh

@@ -128,6 +128,7 @@ namespace orc {
     bool throwOnHive11DecimalOverflow;
     int32_t forcedScaleOnHive11Decimal;
     bool enableLazyDecoding;
+    std::shared_ptr<SearchArgument> sargs;
 
     RowReaderOptionsPrivate() {
       selection = ColumnSelection_NONE;
@@ -249,6 +250,15 @@ namespace orc {
     privateBits->enableLazyDecoding = enable;
     return *this;
   }
+
+  RowReaderOptions& RowReaderOptions::searchArgument(std::unique_ptr<SearchArgument> sargs) {
+    privateBits->sargs = std::move(sargs);
+    return *this;
+  }
+
+  std::shared_ptr<SearchArgument> RowReaderOptions::getSearchArgument() const {
+    return privateBits->sargs;
+  }
 }
 
 #endif

c++/src/OrcFile.cc

@@ -30,6 +30,8 @@
 #include <io.h>
 #define S_IRUSR _S_IREAD
 #define S_IWUSR _S_IWRITE
+#define stat _stat64
+#define fstat _fstat64
 #else
 #include <unistd.h>
 #define O_BINARY 0