From f0599ab5c9bd81a17656d35ded76adff6ef48856 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 8 Mar 2022 11:40:40 -0700 Subject: [PATCH] XFAIL some tests with Spark 3.3.0 with LEGACY timeParserPolicy --- integration_tests/src/main/python/json_test.py | 8 +++++--- integration_tests/src/main/python/spark_session.py | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/integration_tests/src/main/python/json_test.py b/integration_tests/src/main/python/json_test.py index 8d32add2801..0a4fb0a9c6b 100644 --- a/integration_tests/src/main/python/json_test.py +++ b/integration_tests/src/main/python/json_test.py @@ -19,7 +19,7 @@ from conftest import is_databricks_runtime from marks import approximate_float, allow_non_gpu, ignore_order -from spark_session import with_cpu_session, with_gpu_session, is_before_spark_330 +from spark_session import with_cpu_session, with_gpu_session, is_before_spark_330, is_spark_330_or_later json_supported_gens = [ # Spark does not escape '\r' or '\n' even though it uses it to mark end of record @@ -236,7 +236,8 @@ def test_basic_json_read(std_input_path, filename, schema, read_func, allow_non_ @pytest.mark.parametrize('read_func', [read_json_df, read_json_sql]) @pytest.mark.parametrize('ansi_enabled', ["true", "false"]) @pytest.mark.parametrize('time_parser_policy', [ - pytest.param('LEGACY', marks=pytest.mark.allow_non_gpu('FileSourceScanExec')), + pytest.param('LEGACY', marks=[pytest.mark.allow_non_gpu('FileSourceScanExec'), \ + pytest.mark.xfail(not is_before_spark_330(), reason="https://github.com/NVIDIA/spark-rapids/issues/4912")]), 'CORRECTED', 'EXCEPTION' ]) @@ -262,7 +263,8 @@ def test_json_read_valid_dates(std_input_path, filename, schema, read_func, ansi @pytest.mark.parametrize('read_func', [read_json_df, read_json_sql]) @pytest.mark.parametrize('ansi_enabled', ["true", "false"]) @pytest.mark.parametrize('time_parser_policy', [ - pytest.param('LEGACY', marks=pytest.mark.allow_non_gpu('FileSourceScanExec')), + pytest.param('LEGACY', marks=[pytest.mark.allow_non_gpu('FileSourceScanExec'), \ + pytest.mark.xfail(is_spark_330_or_later(), reason="https://github.com/NVIDIA/spark-rapids/issues/4912")]), 'CORRECTED', 'EXCEPTION' ]) diff --git a/integration_tests/src/main/python/spark_session.py b/integration_tests/src/main/python/spark_session.py index 099b7b9b9b5..dfbdd71c64f 100644 --- a/integration_tests/src/main/python/spark_session.py +++ b/integration_tests/src/main/python/spark_session.py @@ -117,6 +117,9 @@ def is_before_spark_320(): def is_before_spark_330(): return spark_version() < "3.3.0" +def is_spark_330_or_later(): + return spark_version() >= "3.3.0" + def is_databricks91_or_later(): spark = get_spark_i_know_what_i_am_doing() return spark.conf.get("spark.databricks.clusterUsageTags.sparkVersion", "") >= "9.1"