TST: Refactoring backend tests

ibis/backends/bigquery/tests/conftest.py

@@ -1,8 +1,15 @@
 import os
+from pathlib import Path
 
 import pytest
 
 import ibis
+import ibis.expr.types as ir
+from ibis.backends.tests.base import (
+    BackendTest,
+    RoundAwayFromZero,
+    UnorderedComparator,
+)
 
 PROJECT_ID = os.environ.get('GOOGLE_BIGQUERY_PROJECT_ID', 'ibis-gbq')
 DATASET_ID = 'testing'
@@ -49,6 +56,34 @@ def connect(project_id, dataset_id, application_name=None):
         pytest.skip(skip_message)
 
 
+class BigQueryTest(UnorderedComparator, BackendTest, RoundAwayFromZero):
+    supports_divide_by_zero = True
+    supports_floating_modulus = False
+    returned_timestamp_unit = 'us'
+
+    @staticmethod
+    def connect(data_directory: Path) -> ibis.client.Client:
+        project_id = os.environ.get('GOOGLE_BIGQUERY_PROJECT_ID')
+        if project_id is None:
+            pytest.skip(
+                'Environment variable GOOGLE_BIGQUERY_PROJECT_ID '
+                'not defined'
+            )
+        elif not project_id:
+            pytest.skip(
+                'Environment variable GOOGLE_BIGQUERY_PROJECT_ID is empty'
+            )
+        return connect(project_id, dataset_id='testing')
+
+    @property
+    def batting(self) -> ir.TableExpr:
+        return None
+
+    @property
+    def awards_players(self) -> ir.TableExpr:
+        return None
+
+
 @pytest.fixture(scope='session')
 def project_id():
     return PROJECT_ID

ibis/backends/clickhouse/tests/conftest.py

@@ -1,8 +1,16 @@
 import os
+from pathlib import Path
+from typing import Callable
 
 import pytest
 
 import ibis
+import ibis.expr.types as ir
+from ibis.backends.tests.base import (
+    BackendTest,
+    RoundHalfToEven,
+    UnorderedComparator,
+)
 
 CLICKHOUSE_HOST = os.environ.get('IBIS_TEST_CLICKHOUSE_HOST', 'localhost')
 CLICKHOUSE_PORT = int(os.environ.get('IBIS_TEST_CLICKHOUSE_PORT', 9000))
@@ -11,6 +19,56 @@
 IBIS_TEST_CLICKHOUSE_DB = os.environ.get('IBIS_TEST_DATA_DB', 'ibis_testing')
 
 
+class ClickhouseTest(UnorderedComparator, BackendTest, RoundHalfToEven):
+    check_dtype = False
+    supports_window_operations = False
+    returned_timestamp_unit = 's'
+    supported_to_timestamp_units = {'s'}
+    supports_floating_modulus = False
+
+    @staticmethod
+    def connect(data_directory: Path) -> ibis.client.Client:
+        host = os.environ.get('IBIS_TEST_CLICKHOUSE_HOST', 'localhost')
+        port = int(os.environ.get('IBIS_TEST_CLICKHOUSE_PORT', 9000))
+        user = os.environ.get('IBIS_TEST_CLICKHOUSE_USER', 'default')
+        password = os.environ.get('IBIS_TEST_CLICKHOUSE_PASSWORD', '')
+        database = os.environ.get(
+            'IBIS_TEST_CLICKHOUSE_DATABASE', 'ibis_testing'
+        )
+        return ibis.clickhouse.connect(
+            host=host,
+            port=port,
+            password=password,
+            database=database,
+            user=user,
+        )
+
+    @property
+    def functional_alltypes(self) -> ir.TableExpr:
+        t = super().functional_alltypes
+        return t.mutate(bool_col=t.bool_col == 1)
+
+    @staticmethod
+    def greatest(
+        f: Callable[..., ir.ValueExpr], *args: ir.ValueExpr
+    ) -> ir.ValueExpr:
+        if len(args) > 2:
+            raise NotImplementedError(
+                'Clickhouse does not support more than 2 arguments to greatest'
+            )
+        return f(*args)
+
+    @staticmethod
+    def least(
+        f: Callable[..., ir.ValueExpr], *args: ir.ValueExpr
+    ) -> ir.ValueExpr:
+        if len(args) > 2:
+            raise NotImplementedError(
+                'Clickhouse does not support more than 2 arguments to least'
+            )
+        return f(*args)
+
+
 @pytest.fixture(scope='module')
 def con():
     return ibis.clickhouse.connect(

ibis/backends/csv/tests/conftest.py

@@ -1,7 +1,65 @@
+from pathlib import Path
+
 import pandas as pd
 import pytest
 
+import ibis
+import ibis.expr.datatypes as dt
+import ibis.expr.types as ir
 from ibis.backends.csv import CSVClient
+from ibis.backends.pandas.tests.conftest import PandasTest
+
+
+class CsvTest(PandasTest):
+    check_names = False
+    supports_divide_by_zero = True
+    returned_timestamp_unit = 'ns'
+
+    @staticmethod
+    def connect(data_directory: Path) -> ibis.client.Client:
+        filename = data_directory / 'functional_alltypes.csv'
+        if not filename.exists():
+            pytest.skip('test data set {} not found'.format(filename))
+        return ibis.csv.connect(data_directory)
+
+    @property
+    def functional_alltypes(self) -> ir.TableExpr:
+        schema = ibis.schema(
+            [
+                ('bool_col', dt.boolean),
+                ('string_col', dt.string),
+                ('timestamp_col', dt.timestamp),
+            ]
+        )
+        return self.connection.table('functional_alltypes', schema=schema)
+
+    @property
+    def batting(self) -> ir.TableExpr:
+        schema = ibis.schema(
+            [
+                ('lgID', dt.string),
+                ('G', dt.float64),
+                ('AB', dt.float64),
+                ('R', dt.float64),
+                ('H', dt.float64),
+                ('X2B', dt.float64),
+                ('X3B', dt.float64),
+                ('HR', dt.float64),
+                ('RBI', dt.float64),
+                ('SB', dt.float64),
+                ('CS', dt.float64),
+                ('BB', dt.float64),
+                ('SO', dt.float64),
+            ]
+        )
+        return self.connection.table('batting', schema=schema)
+
+    @property
+    def awards_players(self) -> ir.TableExpr:
+        schema = ibis.schema(
+            [('lgID', dt.string), ('tie', dt.string), ('notes', dt.string)]
+        )
+        return self.connection.table('awards_players', schema=schema)
 
 
 @pytest.fixture

ibis/backends/dask/tests/conftest.py

@@ -2,9 +2,16 @@
 import pandas as pd
 import pytest
 
+from ibis.backends.pandas.tests.conftest import PandasTest
+
 from .. import connect
 
 
+class DaskTest(PandasTest):
+    # clone pandas directly until the rest of the dask backend is defined
+    pass
+
+
 @pytest.fixture
 def dataframe():
     return dd.from_pandas(

ibis/backends/hdf5/tests/conftest.py

@@ -1,6 +1,23 @@
+from pathlib import Path
+
 import pytest
 
+import ibis
 from ibis.backends.hdf5 import HDFClient
+from ibis.backends.pandas.tests.conftest import PandasTest
+
+
+class HDF5Test(PandasTest):
+    check_names = False
+    supports_divide_by_zero = True
+    returned_timestamp_unit = 'ns'
+
+    @staticmethod
+    def connect(data_directory: Path) -> ibis.client.Client:
+        filename = data_directory / 'functional_alltypes.h5'
+        if not filename.exists():
+            pytest.skip('test data set {} not found'.format(filename))
+        return ibis.hdf5.connect(data_directory)
 
 
 @pytest.fixture

ibis/backends/impala/tests/conftest.py

@@ -1,15 +1,61 @@
 import inspect
 import os
 import warnings
+from pathlib import Path
 
 import pytest
 
 import ibis
+import ibis.expr.types as ir
 import ibis.util as util
 from ibis import options
+from ibis.backends.tests.base import (
+    BackendTest,
+    RoundAwayFromZero,
+    UnorderedComparator,
+)
 from ibis.tests.expr.mocks import MockConnection
 
 
+class ImpalaTest(UnorderedComparator, BackendTest, RoundAwayFromZero):
+    supports_arrays = True
+    supports_arrays_outside_of_select = False
+    check_dtype = False
+    supports_divide_by_zero = True
+    returned_timestamp_unit = 's'
+
+    @staticmethod
+    def connect(data_directory: Path) -> ibis.client.Client:
+        from ibis.backends.impala.tests.conftest import IbisTestEnv
+
+        env = IbisTestEnv()
+        hdfs_client = ibis.impala.hdfs_connect(
+            host=env.nn_host,
+            port=env.webhdfs_port,
+            auth_mechanism=env.auth_mechanism,
+            verify=env.auth_mechanism not in ['GSSAPI', 'LDAP'],
+            user=env.webhdfs_user,
+        )
+        auth_mechanism = env.auth_mechanism
+        if auth_mechanism == 'GSSAPI' or auth_mechanism == 'LDAP':
+            print("Warning: ignoring invalid Certificate Authority errors")
+        return ibis.impala.connect(
+            host=env.impala_host,
+            port=env.impala_port,
+            auth_mechanism=env.auth_mechanism,
+            hdfs_client=hdfs_client,
+            database='ibis_testing',
+        )
+
+    @property
+    def batting(self) -> ir.TableExpr:
+        return None
+
+    @property
+    def awards_players(self) -> ir.TableExpr:
+        return None
+
+
 def isproperty(obj):
     return isinstance(obj, property)
 

ibis/backends/mysql/tests/conftest.py

@@ -0,0 +1,60 @@
+import os
+from pathlib import Path
+
+from pkg_resources import parse_version
+
+import ibis
+from ibis.backends.tests.base import BackendTest, RoundHalfToEven
+
+
+class MySQLTest(BackendTest, RoundHalfToEven):
+    # mysql has the same rounding behavior as postgres
+    check_dtype = False
+    supports_window_operations = False
+    returned_timestamp_unit = 's'
+
+    def __init__(self, data_directory: Path) -> None:
+        super().__init__(data_directory)
+        # mariadb supports window operations after version 10.2
+        # but the sqlalchemy version string looks like:
+        # 5.5.5.10.2.12.MariaDB.10.2.12+maria~jessie
+        # or 10.4.12.MariaDB.1:10.4.12+maria~bionic
+        # example of possible results:
+        # https://github.com/sqlalchemy/sqlalchemy/blob/rel_1_3/
+        # test/dialect/mysql/test_dialect.py#L244-L268
+        con = self.connection
+        if 'MariaDB' in str(con.version):
+            # we might move this parsing step to the mysql client
+            version_detail = con.con.dialect._parse_server_version(
+                str(con.version)
+            )
+            version = (
+                version_detail[:3]
+                if version_detail[3] == 'MariaDB'
+                else version_detail[3:6]
+            )
+            self.__class__.supports_window_operations = version >= (10, 2)
+        elif con.version >= parse_version('8.0'):
+            # mysql supports window operations after version 8
+            self.__class__.supports_window_operations = True
+
+    @staticmethod
+    def connect(data_directory: Path) -> ibis.client.Client:
+        user = os.environ.get('IBIS_TEST_MYSQL_USER', 'ibis')
+        password = os.environ.get('IBIS_TEST_MYSQL_PASSWORD', 'ibis')
+        host = os.environ.get('IBIS_TEST_MYSQL_HOST', 'localhost')
+        port = os.environ.get('IBIS_TEST_MYSQL_PORT', 3306)
+        database = os.environ.get('IBIS_TEST_MYSQL_DATABASE', 'ibis_testing')
+        return ibis.mysql.connect(
+            host=host,
+            port=port,
+            user=user,
+            password=password,
+            database=database,
+        )
+
+    @property
+    def functional_alltypes(self):
+        # BOOLEAN <-> TINYINT(1)
+        t = super().functional_alltypes
+        return t.mutate(bool_col=t.bool_col == 1)