Upgrade ORC version from 1.5.8 to 1.5.10

pom.xml

@@ -798,7 +798,7 @@
         <cudf.version>21.12.0-SNAPSHOT</cudf.version>
         <scala.binary.version>2.12</scala.binary.version>
         <scala.version>2.12.8</scala.version>
-        <orc.version>1.5.8</orc.version>
+        <orc.version>1.5.10</orc.version>
         <orc.classifier/>
         <!--
              If the shade package changes we need to also update jenkins/spark-premerge-build.sh
@@ -839,7 +839,7 @@
         <guava.cdh.version>30.0-jre</guava.cdh.version>
         <shim.module.name>${spark.version.classifier}</shim.module.name>
         <slf4j.version>1.7.30</slf4j.version>
-        <hive.storage.api.version>2.6.0</hive.storage.api.version>
+        <hive.storage.api.version>2.7.1</hive.storage.api.version>
         <protobuf.java.version>2.5.0</protobuf.java.version>
         <flatbuffers.java.version>1.11.0</flatbuffers.java.version>
         <scala.local-lib.path>org/scala-lang/scala-library/${scala.version}/scala-library-${scala.version}.jar</scala.local-lib.path>

sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOrcScanBase.scala

@@ -907,7 +907,9 @@ private case class GpuOrcFileFilterHandler(
         isCaseSensitive)
       val evolution = new SchemaEvolution(orcReader.getSchema, updatedReadSchema, readerOpts)
       val (sargApp, sargColumns) = getSearchApplier(evolution,
-        orcFileReaderOpts.getUseUTCTimestamp)
+        orcFileReaderOpts.getUseUTCTimestamp,
+        orcReader.writerUsedProlepticGregorian(), orcFileReaderOpts.getConvertToProlepticGregorian)
+
       val splitStripes = orcReader.getStripes.asScala.filter(s =>
         s.getOffset >= partFile.start && s.getOffset < partFile.start + partFile.length)
       val stripes = buildOutputStripes(splitStripes, evolution,
@@ -1220,11 +1222,14 @@ private case class GpuOrcFileFilterHandler(
      */
     private def getSearchApplier(
         evolution: SchemaEvolution,
-        useUTCTimestamp: Boolean): (SargApplier, Array[Boolean]) = {
+        useUTCTimestamp: Boolean,
+        writerUsedProlepticGregorian: Boolean,
+        convertToProlepticGregorian: Boolean): (SargApplier, Array[Boolean]) = {
       val searchArg = readerOpts.getSearchArgument
       if (searchArg != null && orcReader.getRowIndexStride != 0) {
         val sa = new SargApplier(searchArg, orcReader.getRowIndexStride, evolution,
-          orcReader.getWriterVersion, useUTCTimestamp)
+          orcReader.getWriterVersion, useUTCTimestamp,
+          writerUsedProlepticGregorian, convertToProlepticGregorian)
         // SargApplier.sargColumns is unfortunately not visible so we redundantly compute it here.
         val filterCols = RecordReaderImpl.mapSargColumnsToOrcInternalColIdx(searchArg.getLeaves,
           evolution)

tests/src/test/scala/com/nvidia/spark/rapids/OrcScanSuite.scala

@@ -17,8 +17,9 @@
 package com.nvidia.spark.rapids
 
 import org.apache.spark.SparkConf
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.functions.col
-import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructField, StructType}
+import org.apache.spark.sql.types.{DateType, IntegerType, LongType, StringType, StructField, StructType}
 
 class OrcScanSuite extends SparkQueryCompareTestSuite {
 
@@ -96,4 +97,41 @@ class OrcScanSuite extends SparkQueryCompareTestSuite {
     })
   }
 
+  /**
+   *
+   * The calendar of hybrid-Julian-calendar.orc file is hybrid Julian Gregorian
+   * This file has one date column one row, value is 1582-10-03
+   * When specify "orc.proleptic.gregorian" calendar to filter the file with c1 >= 1582-10-03,
+   * then no result returned. Because of 1582-10-03 in hybrid calender
+   * is actually 1582-09-23 in proleptic Gregorian calendar.
+   */
+  test("test hybrid Julian Gregorian calendar vs proleptic Gregorian calendar") {
+
+    withCpuSparkSession(spark => {
+      val df = frameFromOrcWithSchema("hybrid-Julian-calendar.orc",
+        StructType(Seq(StructField("c1", DateType))))(spark)
+      val ret = df.collect()
+      ret.length == 1
+      assert(ret(0).toString() == "[1582-10-03]")
+    })
+
+    def check(spark: SparkSession) = {
+      val df = frameFromOrcWithSchema("hybrid-Julian-calendar.orc",
+        StructType(Seq(StructField("c1", DateType))))(spark)
+      df.createOrReplaceTempView("df1")
+      // should not have data, if orc.proleptic.gregorian is specified
+      val ret = spark.sql("select c1 from df1 where c1 >= to_date('1582-10-03', 'yyyy-MM-dd') ")
+          .collect()
+      assert(ret.isEmpty)
+    }
+
+    val conf: SparkConf = new SparkConf()
+    // indicate convert to proleptic Gregorian calendar
+    conf.set("orc.proleptic.gregorian", "true")
+
+    // both cpu and gpu should return empty
+    withGpuSparkSession(check, conf)
+    withCpuSparkSession(check, conf)
+  }
+
 }