diff --git a/python/pyspark/testing/utils.py b/python/pyspark/testing/utils.py index 5da6f47174382..17f74960f8c71 100644 --- a/python/pyspark/testing/utils.py +++ b/python/pyspark/testing/utils.py @@ -758,16 +758,25 @@ def assertDataFrameEqual( has_pandas = False try: # If pandas dependencies are available, allow pandas or pandas-on-Spark DataFrame - import pyspark.pandas as ps import pandas as pd - from pyspark.testing.pandasutils import PandasOnSparkTestUtils has_pandas = True except ImportError: # no pandas, so we won't call pandasutils functions pass - if has_pandas: + has_arrow = False + try: + import pyarrow + + has_arrow = True + except ImportError: + pass + + if has_pandas and has_arrow: + import pyspark.pandas as ps + from pyspark.testing.pandasutils import PandasOnSparkTestUtils + if ( isinstance(actual, pd.DataFrame) or isinstance(expected, pd.DataFrame)