From 4802c0b8bca95ae6f67d67ed1234b740e4359b8f Mon Sep 17 00:00:00 2001 From: Niranjan Artal Date: Wed, 16 Mar 2022 15:13:41 -0700 Subject: [PATCH 1/6] Add RoundCeil and RoundFloor support in Spark-3.3 Signed-off-by: Niranjan Artal --- .../src/main/python/arithmetic_ops_test.py | 23 ++++++- .../rapids/shims/RapidsFloorCeilUtils.scala | 32 +++++++++ .../spark/rapids/shims/Spark33XShims.scala | 69 ++++++++++++++++++- .../rapids/shims/RapidsFloorCeilUtils.scala | 36 ++++++++++ .../spark/sql/rapids/mathExpressions.scala | 13 +--- 5 files changed, 159 insertions(+), 14 deletions(-) create mode 100644 sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala create mode 100644 sql-plugin/src/main/330+/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala diff --git a/integration_tests/src/main/python/arithmetic_ops_test.py b/integration_tests/src/main/python/arithmetic_ops_test.py index 833859c4926..97338013672 100644 --- a/integration_tests/src/main/python/arithmetic_ops_test.py +++ b/integration_tests/src/main/python/arithmetic_ops_test.py @@ -19,7 +19,7 @@ from marks import ignore_order, incompat, approximate_float, allow_non_gpu from pyspark.sql.types import * from pyspark.sql.types import IntegralType -from spark_session import with_cpu_session, with_gpu_session, with_spark_session, is_before_spark_311, is_before_spark_320, is_databricks91_or_later +from spark_session import with_cpu_session, with_gpu_session, with_spark_session, is_before_spark_311, is_before_spark_320, is_before_spark_330, is_databricks91_or_later import pyspark.sql.functions as f # No overflow gens here because we just focus on verifying the fallback to CPU when @@ -351,11 +351,32 @@ def test_floor(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark : unary_op_df(spark, data_gen).selectExpr('floor(a)')) +@pytest.mark.skipif(is_before_spark_330(), reason='scale parameter in Floor function is not supported before Spark 3.3.0') +@pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn) +def test_floor_scale_zero(data_gen): + assert_gpu_and_cpu_are_equal_collect( + lambda spark : unary_op_df(spark, data_gen).selectExpr('floor(a, 0)'), + conf={'spark.rapids.sql.castFloatToDecimal.enabled':'true'}) + @pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn) def test_ceil(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark : unary_op_df(spark, data_gen).selectExpr('ceil(a)')) +@pytest.mark.skipif(is_before_spark_330(), reason='scale parameter in Ceil function is not supported before Spark 3.3.0') +@pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn) +def test_ceil_scale_zero(data_gen): + assert_gpu_and_cpu_are_equal_collect( + lambda spark : unary_op_df(spark, data_gen).selectExpr('ceil(a, 0)'), + conf={'spark.rapids.sql.castFloatToDecimal.enabled':'true'}) + +@pytest.mark.skipif(is_before_spark_330(), reason='scale parameter in Ceil function is not supported before Spark 3.3.0') +@allow_non_gpu('ProjectExec') +@pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn) +def test_ceil_scale_nonzero(data_gen): + assert_gpu_and_cpu_are_equal_collect( + lambda spark : unary_op_df(spark, data_gen).selectExpr('ceil(a, -1)')) + @pytest.mark.parametrize('data_gen', [_decimal_gen_36_neg5, _decimal_gen_38_neg10], ids=idfn) def test_floor_ceil_overflow(data_gen): assert_gpu_and_cpu_error( diff --git a/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala b/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala new file mode 100644 index 00000000000..883c19bfe56 --- /dev/null +++ b/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.rapids.shims + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.rapids.GpuFloorCeil +import org.apache.spark.sql.types.{DataType, DecimalType, LongType} + +object RapidsFloorCeilUtils { + + def getDataType(child: Expression): DataType = { + child.dataType match { + case dt: DecimalType => + DecimalType.bounded(GpuFloorCeil.unboundedOutputPrecision(dt), 0) + case _ => LongType + } + } +} diff --git a/sql-plugin/src/main/330+/scala/com/nvidia/spark/rapids/shims/Spark33XShims.scala b/sql-plugin/src/main/330+/scala/com/nvidia/spark/rapids/shims/Spark33XShims.scala index 3e9982ea258..bd0525cdbb7 100644 --- a/sql-plugin/src/main/330+/scala/com/nvidia/spark/rapids/shims/Spark33XShims.scala +++ b/sql-plugin/src/main/330+/scala/com/nvidia/spark/rapids/shims/Spark33XShims.scala @@ -16,22 +16,23 @@ package com.nvidia.spark.rapids.shims +import ai.rapids.cudf.DType import com.nvidia.spark.rapids._ import org.apache.parquet.schema.MessageType import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Coalesce, DynamicPruningExpression, Expression, MetadataAttribute, TimeAdd} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Coalesce, DynamicPruningExpression, Expression, MetadataAttribute, RoundCeil, RoundFloor, TimeAdd} import org.apache.spark.sql.catalyst.json.rapids.shims.Spark33XFileOptionsShims import org.apache.spark.sql.execution.{BaseSubqueryExec, CoalesceExec, FileSourceScanExec, InSubqueryExec, ProjectExec, ReusedSubqueryExec, SparkPlan, SubqueryBroadcastExec} import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.datasources.{DataSourceUtils, FilePartition, FileScanRDD, HadoopFsRelation, PartitionedFile} import org.apache.spark.sql.execution.datasources.parquet.ParquetFilters import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.rapids.GpuFileSourceScanExec +import org.apache.spark.sql.rapids.{GpuCeil, GpuFileSourceScanExec, GpuFloor, GpuFloorCeil} import org.apache.spark.sql.rapids.shims.GpuTimeAdd -import org.apache.spark.sql.types.{CalendarIntervalType, DayTimeIntervalType, StructType} +import org.apache.spark.sql.types.{CalendarIntervalType, DayTimeIntervalType, DecimalType, StructType} import org.apache.spark.unsafe.types.CalendarInterval trait Spark33XShims extends Spark33XFileOptionsShims { @@ -127,6 +128,68 @@ trait Spark33XShims extends Spark33XFileOptionsShims { } } }), + GpuOverrides.expr[RoundCeil]( + "Computes the ceiling of the given expression to d decimal places", + ExprChecks.binaryProject( + TypeSig.gpuNumeric, TypeSig.cpuNumeric, + ("value", TypeSig.gpuNumeric + + TypeSig.psNote(TypeEnum.FLOAT, "result may round slightly differently") + + TypeSig.psNote(TypeEnum.DOUBLE, "result may round slightly differently"), + TypeSig.cpuNumeric), + ("scale", TypeSig.lit(TypeEnum.INT), TypeSig.lit(TypeEnum.INT))), + (ceil, conf, p, r) => new BinaryExprMeta[RoundCeil](ceil, conf, p, r) { + override def tagExprForGpu(): Unit = { + ceil.child.dataType match { + case dt: DecimalType => + val precision = GpuFloorCeil.unboundedOutputPrecision(dt) + if (precision > DType.DECIMAL128_MAX_PRECISION) { + willNotWorkOnGpu(s"output precision $precision would require overflow " + + s"checks, which are not supported yet") + } + case _ => // NOOP + } + GpuOverrides.extractLit(ceil.scale).foreach { scale => + if (scale.value != null && + scale.value.asInstanceOf[Integer] != 0) { + willNotWorkOnGpu("Scale other than 0 is not supported") + } + } + } + + override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = + GpuCeil(lhs) + }), + GpuOverrides.expr[RoundFloor]( + "Computes the floor of the given expression to d decimal places", + ExprChecks.binaryProject( + TypeSig.gpuNumeric, TypeSig.cpuNumeric, + ("value", TypeSig.gpuNumeric + + TypeSig.psNote(TypeEnum.FLOAT, "result may round slightly differently") + + TypeSig.psNote(TypeEnum.DOUBLE, "result may round slightly differently"), + TypeSig.cpuNumeric), + ("scale", TypeSig.lit(TypeEnum.INT), TypeSig.lit(TypeEnum.INT))), + (floor, conf, p, r) => new BinaryExprMeta[RoundFloor](floor, conf, p, r) { + override def tagExprForGpu(): Unit = { + floor.child.dataType match { + case dt: DecimalType => + val precision = GpuFloorCeil.unboundedOutputPrecision(dt) + if (precision > DType.DECIMAL128_MAX_PRECISION) { + willNotWorkOnGpu(s"output precision $precision would require overflow " + + s"checks, which are not supported yet") + } + case _ => // NOOP + } + GpuOverrides.extractLit(floor.scale).foreach { scale => + if (scale.value != null && + scale.value.asInstanceOf[Integer] != 0) { + willNotWorkOnGpu("Scale other than 0 is not supported") + } + } + } + + override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = + GpuFloor(lhs) + }), GpuOverrides.expr[TimeAdd]( "Adds interval to timestamp", ExprChecks.binaryProject(TypeSig.TIMESTAMP, TypeSig.TIMESTAMP, diff --git a/sql-plugin/src/main/330+/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala b/sql-plugin/src/main/330+/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala new file mode 100644 index 00000000000..ef6a56c5f1b --- /dev/null +++ b/sql-plugin/src/main/330+/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.rapids.shims + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.rapids.GpuFloorCeil +import org.apache.spark.sql.types.{DataType, DecimalType, LongType} + +object RapidsFloorCeilUtils { + + def getDataType(child: Expression): DataType = { + child.dataType match { + // For Ceil/Floor function we calculate the precision by calling unboundedOutputPrecision and + // for RoundCeil/RoundFloor we take the precision as it is. Here the actual precision is + // calculated by taking the max of these 2 to make sure we don't overflow while + // creating the DecimalType. + case dt: DecimalType => + val maxPrecision = math.max(GpuFloorCeil.unboundedOutputPrecision(dt), dt.precision) + DecimalType.bounded(maxPrecision, 0) + case _ => LongType + } + } +} diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/mathExpressions.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/mathExpressions.scala index c2d1c977bab..0177ea8db12 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/mathExpressions.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/mathExpressions.scala @@ -24,6 +24,7 @@ import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.RapidsPluginImplicits.ReallyAGpuExpression import org.apache.spark.sql.catalyst.expressions.{EmptyRow, Expression, ImplicitCastInputTypes} +import org.apache.spark.sql.rapids.shims.RapidsFloorCeilUtils import org.apache.spark.sql.types._ abstract class CudfUnaryMathExpression(name: String) extends GpuUnaryMathExpression(name) @@ -166,11 +167,7 @@ object GpuFloorCeil { } case class GpuCeil(child: Expression) extends CudfUnaryMathExpression("CEIL") { - override def dataType: DataType = child.dataType match { - case dt: DecimalType => - DecimalType.bounded(GpuFloorCeil.unboundedOutputPrecision(dt), 0) - case _ => LongType - } + override def dataType: DataType = RapidsFloorCeilUtils.getDataType(child) override def hasSideEffects: Boolean = true @@ -242,11 +239,7 @@ case class GpuExpm1(child: Expression) extends CudfUnaryMathExpression("EXPM1") } case class GpuFloor(child: Expression) extends CudfUnaryMathExpression("FLOOR") { - override def dataType: DataType = child.dataType match { - case dt: DecimalType => - DecimalType.bounded(GpuFloorCeil.unboundedOutputPrecision(dt), 0) - case _ => LongType - } + override def dataType: DataType = RapidsFloorCeilUtils.getDataType(child) override def hasSideEffects: Boolean = true From a79bd5286a32443ceb0095725fc786e39ffcbd0a Mon Sep 17 00:00:00 2001 From: Niranjan Artal Date: Wed, 16 Mar 2022 15:40:41 -0700 Subject: [PATCH 2/6] update copyright Signed-off-by: Niranjan Artal --- .../apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala | 2 +- .../scala/com/nvidia/spark/rapids/shims/Spark33XShims.scala | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala b/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala index 883c19bfe56..424657535ff 100644 --- a/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala +++ b/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/sql-plugin/src/main/330+/scala/com/nvidia/spark/rapids/shims/Spark33XShims.scala b/sql-plugin/src/main/330+/scala/com/nvidia/spark/rapids/shims/Spark33XShims.scala index 326c7ab5a02..130e9a3ce2a 100644 --- a/sql-plugin/src/main/330+/scala/com/nvidia/spark/rapids/shims/Spark33XShims.scala +++ b/sql-plugin/src/main/330+/scala/com/nvidia/spark/rapids/shims/Spark33XShims.scala @@ -17,7 +17,6 @@ package com.nvidia.spark.rapids.shims import ai.rapids.cudf.DType - import com.nvidia.spark.InMemoryTableScanMeta import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.GpuOverrides From 5afb004b4d16ccad5c2d6a8dae18486f4a56e287 Mon Sep 17 00:00:00 2001 From: Niranjan Artal Date: Thu, 17 Mar 2022 22:06:49 -0700 Subject: [PATCH 3/6] remove test Signed-off-by: Niranjan Artal --- integration_tests/src/main/python/arithmetic_ops_test.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/integration_tests/src/main/python/arithmetic_ops_test.py b/integration_tests/src/main/python/arithmetic_ops_test.py index 97338013672..0adf0386357 100644 --- a/integration_tests/src/main/python/arithmetic_ops_test.py +++ b/integration_tests/src/main/python/arithmetic_ops_test.py @@ -370,13 +370,6 @@ def test_ceil_scale_zero(data_gen): lambda spark : unary_op_df(spark, data_gen).selectExpr('ceil(a, 0)'), conf={'spark.rapids.sql.castFloatToDecimal.enabled':'true'}) -@pytest.mark.skipif(is_before_spark_330(), reason='scale parameter in Ceil function is not supported before Spark 3.3.0') -@allow_non_gpu('ProjectExec') -@pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn) -def test_ceil_scale_nonzero(data_gen): - assert_gpu_and_cpu_are_equal_collect( - lambda spark : unary_op_df(spark, data_gen).selectExpr('ceil(a, -1)')) - @pytest.mark.parametrize('data_gen', [_decimal_gen_36_neg5, _decimal_gen_38_neg10], ids=idfn) def test_floor_ceil_overflow(data_gen): assert_gpu_and_cpu_error( From c843e757ba375e008fd631567abd2ce1a7bb5833 Mon Sep 17 00:00:00 2001 From: Niranjan Artal Date: Sun, 20 Mar 2022 19:09:27 -0700 Subject: [PATCH 4/6] addressed review comments Signed-off-by: Niranjan Artal --- integration_tests/src/main/python/arithmetic_ops_test.py | 7 +++++++ .../spark/sql/rapids/shims/RapidsFloorCeilUtils.scala | 5 ++--- .../spark/sql/rapids/shims/RapidsFloorCeilUtils.scala | 5 ++--- .../org/apache/spark/sql/rapids/mathExpressions.scala | 4 ++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/integration_tests/src/main/python/arithmetic_ops_test.py b/integration_tests/src/main/python/arithmetic_ops_test.py index 0adf0386357..daefd2b0d29 100644 --- a/integration_tests/src/main/python/arithmetic_ops_test.py +++ b/integration_tests/src/main/python/arithmetic_ops_test.py @@ -358,6 +358,13 @@ def test_floor_scale_zero(data_gen): lambda spark : unary_op_df(spark, data_gen).selectExpr('floor(a, 0)'), conf={'spark.rapids.sql.castFloatToDecimal.enabled':'true'}) +@pytest.mark.skipif(is_before_spark_330(), reason='scale parameter in Floor function is not supported before Spark 3.3.0') +@allow_non_gpu('ProjectExec') +@pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn) +def test_floor_scale_nonzero(data_gen): + assert_gpu_and_cpu_are_equal_collect( + lambda spark : unary_op_df(spark, data_gen).selectExpr('ceil(a, -1)')) + @pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn) def test_ceil(data_gen): assert_gpu_and_cpu_are_equal_collect( diff --git a/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala b/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala index 424657535ff..4aa5523a04e 100644 --- a/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala +++ b/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala @@ -16,14 +16,13 @@ package org.apache.spark.sql.rapids.shims -import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.rapids.GpuFloorCeil import org.apache.spark.sql.types.{DataType, DecimalType, LongType} object RapidsFloorCeilUtils { - def getDataType(child: Expression): DataType = { - child.dataType match { + def outputDataType(dataType: DataType): DataType = { + dataType match { case dt: DecimalType => DecimalType.bounded(GpuFloorCeil.unboundedOutputPrecision(dt), 0) case _ => LongType diff --git a/sql-plugin/src/main/330+/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala b/sql-plugin/src/main/330+/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala index ef6a56c5f1b..029408711ed 100644 --- a/sql-plugin/src/main/330+/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala +++ b/sql-plugin/src/main/330+/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala @@ -15,14 +15,13 @@ */ package org.apache.spark.sql.rapids.shims -import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.rapids.GpuFloorCeil import org.apache.spark.sql.types.{DataType, DecimalType, LongType} object RapidsFloorCeilUtils { - def getDataType(child: Expression): DataType = { - child.dataType match { + def outputDataType(dataType: DataType): DataType = { + dataType match { // For Ceil/Floor function we calculate the precision by calling unboundedOutputPrecision and // for RoundCeil/RoundFloor we take the precision as it is. Here the actual precision is // calculated by taking the max of these 2 to make sure we don't overflow while diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/mathExpressions.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/mathExpressions.scala index 0177ea8db12..c444e72090e 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/mathExpressions.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/mathExpressions.scala @@ -167,7 +167,7 @@ object GpuFloorCeil { } case class GpuCeil(child: Expression) extends CudfUnaryMathExpression("CEIL") { - override def dataType: DataType = RapidsFloorCeilUtils.getDataType(child) + override def dataType: DataType = RapidsFloorCeilUtils.outputDataType(child.dataType) override def hasSideEffects: Boolean = true @@ -239,7 +239,7 @@ case class GpuExpm1(child: Expression) extends CudfUnaryMathExpression("EXPM1") } case class GpuFloor(child: Expression) extends CudfUnaryMathExpression("FLOOR") { - override def dataType: DataType = RapidsFloorCeilUtils.getDataType(child) + override def dataType: DataType = RapidsFloorCeilUtils.outputDataType(child.dataType) override def hasSideEffects: Boolean = true From b55486bb77995a7c3a70a6468637fac77c0a3011 Mon Sep 17 00:00:00 2001 From: Niranjan Artal Date: Mon, 21 Mar 2022 09:30:49 -0700 Subject: [PATCH 5/6] update test Signed-off-by: Niranjan Artal --- integration_tests/src/main/python/arithmetic_ops_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/src/main/python/arithmetic_ops_test.py b/integration_tests/src/main/python/arithmetic_ops_test.py index daefd2b0d29..c29edd97e2d 100644 --- a/integration_tests/src/main/python/arithmetic_ops_test.py +++ b/integration_tests/src/main/python/arithmetic_ops_test.py @@ -362,8 +362,8 @@ def test_floor_scale_zero(data_gen): @allow_non_gpu('ProjectExec') @pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn) def test_floor_scale_nonzero(data_gen): - assert_gpu_and_cpu_are_equal_collect( - lambda spark : unary_op_df(spark, data_gen).selectExpr('ceil(a, -1)')) + assert_gpu_fallback_collect( + lambda spark : unary_op_df(spark, data_gen).selectExpr('floor(a, -1)'), 'RoundFloor') @pytest.mark.parametrize('data_gen', double_n_long_gens + _arith_decimal_gens_no_neg_scale, ids=idfn) def test_ceil(data_gen): From bac1313ac83909d1c6bd52c3b07a835d48067da6 Mon Sep 17 00:00:00 2001 From: Niranjan Artal Date: Mon, 21 Mar 2022 12:21:43 -0700 Subject: [PATCH 6/6] move file to 311 shim as 301 is removed Signed-off-by: Niranjan Artal --- .../org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sql-plugin/src/main/{301until330-all => 311until330-all}/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala (100%) diff --git a/sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala b/sql-plugin/src/main/311until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala similarity index 100% rename from sql-plugin/src/main/301until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala rename to sql-plugin/src/main/311until330-all/scala/org/apache/spark/sql/rapids/shims/RapidsFloorCeilUtils.scala