Fix default encoding and add cleanup tools for ibis

CHANGELOG.md

@@ -44,6 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Trigger downstream vector indices.
 - Fix vector_index function job.
 - Fix verbosity in component info
+- Change default encoding to sqlvector
 
 ## [0.3.0](https://github.com/superduper-io/superduper/compare/0.3.0...0.2.0])    (2024-Jun-21)
 

superduper/backends/ibis/data_backend.py

@@ -3,12 +3,13 @@
 import typing as t
 from warnings import warn
 
+import click
 import ibis
 import pandas
 from pandas.core.frame import DataFrame
 from sqlalchemy.exc import NoSuchTableError
 
-from superduper import CFG
+from superduper import CFG, logging
 from superduper.backends.base.data_backend import BaseDataBackend
 from superduper.backends.base.metadata import MetaDataStoreProxy
 from superduper.backends.ibis.db_helper import get_db_helper
@@ -132,7 +133,10 @@ def insert(self, table_name, raw_documents):
 
     def drop_outputs(self):
         """Drop the outputs."""
-        raise NotImplementedError
+        for table in self.conn.list_tables():
+            logging.info(f"Dropping table: {table}")
+            if CFG.output_prefix in table:
+                self.conn.drop_table(table)
 
     def drop_table_or_collection(self, name: str):
         """Drop the table or collection.
@@ -212,9 +216,13 @@ def drop(self, force: bool = False):
 
         :param force: Whether to force the drop.
         """
-        raise NotImplementedError(
-            "Dropping tables needs to be done in each DB natively"
-        )
+        if not force and not click.confirm("Are you sure you want to drop all tables?"):
+            logging.info("Aborting drop tables")
+            return
+
+        for table in self.conn.list_tables():
+            logging.info(f"Dropping table: {table}")
+            self.conn.drop_table(table)
 
     def get_table_or_collection(self, identifier):
         """Get a table or collection from the database.

superduper/backends/sqlalchemy/metadata.py

@@ -142,9 +142,20 @@ def drop(self, force: bool = False):
                 default=False,
             ):
                 logging.warn('Aborting...')
-        self.job_table.drop(self.conn)
-        self.parent_child_association_table.drop(self.conn)
-        self.component_table.drop(self.conn)
+        try:
+            self.job_table.drop(self.conn)
+        except ProgrammingError as e:
+            logging.warn(f'Error dropping job table: {e}')
+
+        try:
+            self.parent_child_association_table.drop(self.conn)
+        except ProgrammingError as e:
+            logging.warn(f'Error dropping parent-child association table: {e}')
+
+        try:
+            self.component_table.drop(self.conn)
+        except ProgrammingError as e:
+            logging.warn(f'Error dropping component table {e}')
 
     @contextmanager
     def session_context(self):

superduper/components/vector_index.py

@@ -365,7 +365,7 @@ def vector(shape, identifier: t.Optional[str] = None):
 
 
 @component()
-def sqlvector(shape, bytes_encoding: str = 'Bytes'):
+def sqlvector(shape, bytes_encoding: t.Optional[str] = None):
     """Create an encoder for a vector (list of ints/ floats) of a given shape.
 
     This is used for compatibility with SQL databases, as the default vector