diff --git a/bcdata/bc2pg.py b/bcdata/bc2pg.py index 60460cd..f41289c 100644 --- a/bcdata/bc2pg.py +++ b/bcdata/bc2pg.py @@ -62,6 +62,7 @@ def bc2pg( # noqa: C901 timestamp=True, schema_only=False, append=False, + refresh=False, ): """Request table definition from bcdc and replicate in postgres""" if schema_only and append: @@ -228,22 +229,8 @@ def bc2pg( # noqa: C901 df = None # once load complete, note date/time of load completion in bcdata.log - if timestamp: - log.info("Logging download date to bcdata.log") - db.execute( - """CREATE SCHEMA IF NOT EXISTS bcdata; - CREATE TABLE IF NOT EXISTS bcdata.log ( - table_name text PRIMARY KEY, - latest_download timestamp WITH TIME ZONE - ); - """ - ) - db.execute( - """INSERT INTO bcdata.log (table_name, latest_download) - SELECT %s as table_name, NOW() as latest_download - ON CONFLICT (table_name) DO UPDATE SET latest_download = NOW(); - """, - (schema_name + "." + table_name,), - ) + # do not log refreshes here, they called by cli once loaded to target table + if timestamp and not refresh: + db.log(schema_name, table_name) return schema_name + "." + table_name diff --git a/bcdata/cli.py b/bcdata/cli.py index e6d3bd7..f57e4b3 100644 --- a/bcdata/cli.py +++ b/bcdata/cli.py @@ -429,6 +429,7 @@ def bc2pg( timestamp=timestamp, schema_only=schema_only, append=append, + refresh=refresh, ) # if refreshing, flush from temp bcdata schema to target schema @@ -437,6 +438,7 @@ def bc2pg( s, table = out_table.split(".") db.refresh(schema_target, table) out_table = schema_target + "." + table + db.log(schema_target, table) # do not notify of data load completion when no data load has occured if not schema_only: diff --git a/bcdata/database.py b/bcdata/database.py index 9c872d4..950bc4b 100644 --- a/bcdata/database.py +++ b/bcdata/database.py @@ -212,3 +212,21 @@ def get_columns(self, schema, table): metadata_obj = MetaData(schema=schema) table = Table(table, metadata_obj, schema=schema, autoload_with=self.engine) return list(table.columns.keys()) + + def log(self, schema_name, table_name): + log.info("Logging download date to bcdata.log") + self.execute( + """CREATE SCHEMA IF NOT EXISTS bcdata; + CREATE TABLE IF NOT EXISTS bcdata.log ( + table_name text PRIMARY KEY, + latest_download timestamp WITH TIME ZONE + ); + """ + ) + self.execute( + """INSERT INTO bcdata.log (table_name, latest_download) + SELECT %s as table_name, NOW() as latest_download + ON CONFLICT (table_name) DO UPDATE SET latest_download = NOW(); + """, + (schema_name + "." + table_name,), + )