diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ddf953d5..3bb830d9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Trigger downstream vector indices. - Fix vector_index function job. - Fix verbosity in component info +- Change default encoding to sqlvector ## [0.3.0](https://github.com/superduper-io/superduper/compare/0.3.0...0.2.0]) (2024-Jun-21) diff --git a/superduper/backends/ibis/data_backend.py b/superduper/backends/ibis/data_backend.py index 21c74a443..e2e7b2148 100644 --- a/superduper/backends/ibis/data_backend.py +++ b/superduper/backends/ibis/data_backend.py @@ -3,12 +3,13 @@ import typing as t from warnings import warn +import click import ibis import pandas from pandas.core.frame import DataFrame from sqlalchemy.exc import NoSuchTableError -from superduper import CFG +from superduper import CFG, logging from superduper.backends.base.data_backend import BaseDataBackend from superduper.backends.base.metadata import MetaDataStoreProxy from superduper.backends.ibis.db_helper import get_db_helper @@ -132,7 +133,10 @@ def insert(self, table_name, raw_documents): def drop_outputs(self): """Drop the outputs.""" - raise NotImplementedError + for table in self.conn.list_tables(): + logging.info(f"Dropping table: {table}") + if CFG.output_prefix in table: + self.conn.drop_table(table) def drop_table_or_collection(self, name: str): """Drop the table or collection. @@ -212,9 +216,13 @@ def drop(self, force: bool = False): :param force: Whether to force the drop. """ - raise NotImplementedError( - "Dropping tables needs to be done in each DB natively" - ) + if not force and not click.confirm("Are you sure you want to drop all tables?"): + logging.info("Aborting drop tables") + return + + for table in self.conn.list_tables(): + logging.info(f"Dropping table: {table}") + self.conn.drop_table(table) def get_table_or_collection(self, identifier): """Get a table or collection from the database. diff --git a/superduper/backends/sqlalchemy/metadata.py b/superduper/backends/sqlalchemy/metadata.py index b4cc96603..09e425822 100644 --- a/superduper/backends/sqlalchemy/metadata.py +++ b/superduper/backends/sqlalchemy/metadata.py @@ -142,9 +142,20 @@ def drop(self, force: bool = False): default=False, ): logging.warn('Aborting...') - self.job_table.drop(self.conn) - self.parent_child_association_table.drop(self.conn) - self.component_table.drop(self.conn) + try: + self.job_table.drop(self.conn) + except ProgrammingError as e: + logging.warn(f'Error dropping job table: {e}') + + try: + self.parent_child_association_table.drop(self.conn) + except ProgrammingError as e: + logging.warn(f'Error dropping parent-child association table: {e}') + + try: + self.component_table.drop(self.conn) + except ProgrammingError as e: + logging.warn(f'Error dropping component table {e}') @contextmanager def session_context(self): diff --git a/superduper/components/vector_index.py b/superduper/components/vector_index.py index e716d8f18..d20820127 100644 --- a/superduper/components/vector_index.py +++ b/superduper/components/vector_index.py @@ -365,7 +365,7 @@ def vector(shape, identifier: t.Optional[str] = None): @component() -def sqlvector(shape, bytes_encoding: str = 'Bytes'): +def sqlvector(shape, bytes_encoding: t.Optional[str] = None): """Create an encoder for a vector (list of ints/ floats) of a given shape. This is used for compatibility with SQL databases, as the default vector