From b41b188a4f05606b60ae0a864d24c73550a1a05f Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 15 Nov 2022 13:56:02 -0500 Subject: [PATCH 1/6] Rename MacroEvent to JinjaLog --- core/dbt/context/base.py | 6 +++--- core/dbt/events/proto_types.py | 4 ++-- core/dbt/events/types.proto | 4 ++-- core/dbt/events/types.py | 8 ++++---- tests/unit/test_events.py | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/core/dbt/context/base.py b/core/dbt/context/base.py index 68b5edb98c1..262ed45d3dc 100644 --- a/core/dbt/context/base.py +++ b/core/dbt/context/base.py @@ -16,7 +16,7 @@ disallow_secret_env_var, ) from dbt.events.functions import fire_event, get_invocation_id -from dbt.events.types import MacroEventInfo, MacroEventDebug +from dbt.events.types import JinjaLogInfo, JinjaLogDebug from dbt.version import __version__ as dbt_version # These modules are added to the context. Consider alternative @@ -557,9 +557,9 @@ def log(msg: str, info: bool = False) -> str: {% endmacro %}" """ if info: - fire_event(MacroEventInfo(msg=msg)) + fire_event(JinjaLogInfo(msg=msg)) else: - fire_event(MacroEventDebug(msg=msg)) + fire_event(JinjaLogDebug(msg=msg)) return "" @contextproperty diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 09d29b5563b..b8ee03ea99a 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -1334,7 +1334,7 @@ class SelectorReportInvalidSelector(betterproto.Message): @dataclass -class MacroEventInfo(betterproto.Message): +class JinjaLogInfo(betterproto.Message): """M011""" info: "EventInfo" = betterproto.message_field(1) @@ -1342,7 +1342,7 @@ class MacroEventInfo(betterproto.Message): @dataclass -class MacroEventDebug(betterproto.Message): +class JinjaLogDebug(betterproto.Message): """M012""" info: "EventInfo" = betterproto.message_field(1) diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index 8cafe71bd95..d980713b91f 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -1015,13 +1015,13 @@ message SelectorReportInvalidSelector { } // M011 -message MacroEventInfo { +message JinjaLogInfo { EventInfo info = 1; string msg = 2; } // M012 -message MacroEventDebug { +message JinjaLogDebug { EventInfo info = 1; string msg = 2; } diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index de738df9487..3861a366a7b 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -1626,7 +1626,7 @@ def message(self) -> str: @dataclass -class MacroEventInfo(InfoLevel, EventStringFunctor, pt.MacroEventInfo): +class JinjaLogInfo(InfoLevel, EventStringFunctor, pt.JinjaLogInfo): def code(self): return "M011" @@ -1636,7 +1636,7 @@ def message(self) -> str: @dataclass -class MacroEventDebug(DebugLevel, EventStringFunctor, pt.MacroEventDebug): +class JinjaLogDebug(DebugLevel, EventStringFunctor, pt.JinjaLogDebug): def code(self): return "M012" @@ -2932,8 +2932,8 @@ def message(self) -> str: RegistryProgressGETRequest(url="") RegistryProgressGETResponse(url="", resp_code=1234) SelectorReportInvalidSelector(valid_selectors="", spec_method="", raw_spec="") - MacroEventInfo(msg="") - MacroEventDebug(msg="") + JinjaLogInfo(msg="") + JinjaLogDebug(msg="") DepsNoPackagesFound() DepsStartPackageInstall(package_name="") DepsInstallInfo(version_name="") diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 050d5153c8d..8839b04b075 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -81,7 +81,7 @@ def test_formatting(self): event = AdapterEventDebug(name="dbt_tests", base_msg=[1,2,3], args=(3,)) assert isinstance(event.base_msg, str) - event = MacroEventDebug(msg=[1,2,3]) + event = JinjaLogDebug(msg=[1,2,3]) assert isinstance(event.msg, str) @@ -192,8 +192,8 @@ def MockNode(): SystemStdOutMsg(bmsg=b""), SystemStdErrMsg(bmsg=b""), SelectorReportInvalidSelector(valid_selectors="", spec_method="", raw_spec=""), - MacroEventInfo(msg=""), - MacroEventDebug(msg=""), + JinjaLogInfo(msg=""), + JinjaLogDebug(msg=""), NewConnection(conn_type="", conn_name=""), ConnectionReused(conn_name=""), ConnectionLeftOpen(conn_name=""), From aeb39092d9f953734766679d342dead9cc55b057 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 15 Nov 2022 14:02:38 -0500 Subject: [PATCH 2/6] Rename ConnectionClosed/2 --- core/dbt/adapters/base/connections.py | 6 +++--- core/dbt/events/proto_types.py | 4 ++-- core/dbt/events/types.proto | 4 ++-- core/dbt/events/types.py | 8 ++++---- tests/unit/test_events.py | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/dbt/adapters/base/connections.py b/core/dbt/adapters/base/connections.py index 5fd3769aa74..3c5db18c8a7 100644 --- a/core/dbt/adapters/base/connections.py +++ b/core/dbt/adapters/base/connections.py @@ -43,8 +43,8 @@ ConnectionReused, ConnectionLeftOpen, ConnectionLeftOpen2, + ConnectionClosedInCleanup, ConnectionClosed, - ConnectionClosed2, Rollback, RollbackFailed, ) @@ -308,7 +308,7 @@ def cleanup_all(self) -> None: if connection.state not in {"closed", "init"}: fire_event(ConnectionLeftOpen(conn_name=cast_to_str(connection.name))) else: - fire_event(ConnectionClosed(conn_name=cast_to_str(connection.name))) + fire_event(ConnectionClosedInCleanup(conn_name=cast_to_str(connection.name))) self.close(connection) # garbage collect these connections @@ -345,7 +345,7 @@ def _close_handle(cls, connection: Connection) -> None: """Perform the actual close operation.""" # On windows, sometimes connection handles don't have a close() attr. if hasattr(connection.handle, "close"): - fire_event(ConnectionClosed2(conn_name=cast_to_str(connection.name))) + fire_event(ConnectionClosed(conn_name=cast_to_str(connection.name))) connection.handle.close() else: fire_event(ConnectionLeftOpen2(conn_name=cast_to_str(connection.name))) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index b8ee03ea99a..2cb8f3bf21a 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -406,7 +406,7 @@ class ConnectionLeftOpen(betterproto.Message): @dataclass -class ConnectionClosed(betterproto.Message): +class ConnectionClosedInCleanup(betterproto.Message): """E008""" info: "EventInfo" = betterproto.message_field(1) @@ -423,7 +423,7 @@ class RollbackFailed(betterproto.Message): @dataclass -class ConnectionClosed2(betterproto.Message): +class ConnectionClosed(betterproto.Message): """E010""" info: "EventInfo" = betterproto.message_field(1) diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index d980713b91f..ae218ea6a96 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -314,7 +314,7 @@ message ConnectionLeftOpen { } // E008 -message ConnectionClosed { +message ConnectionClosedInCleanup { EventInfo info = 1; string conn_name = 2; } @@ -327,7 +327,7 @@ message RollbackFailed { } // E010 -message ConnectionClosed2 { +message ConnectionClosed { EventInfo info = 1; string conn_name = 2; } diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 3861a366a7b..b738d06be82 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -485,7 +485,7 @@ def message(self) -> str: @dataclass -class ConnectionClosed(DebugLevel, pt.ConnectionClosed): +class ConnectionClosedInCleanup(DebugLevel, pt.ConnectionClosedInCleanup): def code(self): return "E008" @@ -504,7 +504,7 @@ def message(self) -> str: # TODO: can we combine this with ConnectionClosed? @dataclass -class ConnectionClosed2(DebugLevel, pt.ConnectionClosed2): +class ConnectionClosed(DebugLevel, pt.ConnectionClosed): def code(self): return "E010" @@ -2784,9 +2784,9 @@ def message(self) -> str: NewConnection(conn_type="", conn_name="") ConnectionReused(conn_name="") ConnectionLeftOpen(conn_name="") - ConnectionClosed(conn_name="") + ConnectionClosedInCleanup(conn_name="") RollbackFailed(conn_name="") - ConnectionClosed2(conn_name="") + ConnectionClosed(conn_name="") ConnectionLeftOpen2(conn_name="") Rollback(conn_name="") CacheMiss(conn_name="", database="", schema="") diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 8839b04b075..371dd13300e 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -197,9 +197,9 @@ def MockNode(): NewConnection(conn_type="", conn_name=""), ConnectionReused(conn_name=""), ConnectionLeftOpen(conn_name=""), - ConnectionClosed(conn_name=""), + ConnectionClosedInCleanup(conn_name=""), RollbackFailed(conn_name=""), - ConnectionClosed2(conn_name=""), + ConnectionClosed(conn_name=""), ConnectionLeftOpen2(conn_name=""), Rollback(conn_name=""), CacheMiss(conn_name="", database="", schema=""), From 37a5d7c8b0da98bbc25c4304084c0c45f9b1e8b2 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 15 Nov 2022 14:09:20 -0500 Subject: [PATCH 3/6] Fix LogSeedResult --- core/dbt/events/proto_types.py | 11 ++++++----- core/dbt/events/types.proto | 11 ++++++----- core/dbt/events/types.py | 2 +- core/dbt/task/seed.py | 1 + 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 2cb8f3bf21a..eefd0f4b104 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -1602,11 +1602,12 @@ class LogSeedResult(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) node_info: "NodeInfo" = betterproto.message_field(2) status: str = betterproto.string_field(3) - index: int = betterproto.int32_field(4) - total: int = betterproto.int32_field(5) - execution_time: float = betterproto.float_field(6) - schema: str = betterproto.string_field(7) - relation: str = betterproto.string_field(8) + result_message: str = betterproto.string_field(4) + index: int = betterproto.int32_field(5) + total: int = betterproto.int32_field(6) + execution_time: float = betterproto.float_field(7) + schema: str = betterproto.string_field(8) + relation: str = betterproto.string_field(9) @dataclass diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index ae218ea6a96..394070b84d1 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -1227,11 +1227,12 @@ message LogSeedResult { EventInfo info = 1; NodeInfo node_info = 2; string status = 3; - int32 index = 4; - int32 total = 5; - float execution_time = 6; - string schema = 7; - string relation = 8; + string result_message = 4; + int32 index = 5; + int32 total = 6; + float execution_time = 7; + string schema = 8; + string relation = 9; } // Skipped Q017 diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index b738d06be82..a5c4e661b78 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -1999,7 +1999,7 @@ def message(self) -> str: status = red(self.status.upper()) else: info = "OK loaded" - status = green(self.status) + status = green(self.result_message) msg = f"{info} seed file {self.schema}.{self.relation}" return format_fancy_output_line( msg=msg, diff --git a/core/dbt/task/seed.py b/core/dbt/task/seed.py index 16b731e4f7d..5c922a5ba90 100644 --- a/core/dbt/task/seed.py +++ b/core/dbt/task/seed.py @@ -51,6 +51,7 @@ def print_result_line(self, result): LogSeedResult( info=info(level=level), status=result.status, + result_message=result.message, index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, From 075b97a69e8cc2e1543d74ff3b576910e35273a0 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 18 Nov 2022 13:25:53 -0500 Subject: [PATCH 4/6] Rename ConnectionLeftOpen events, fix test_events.py --- core/dbt/adapters/base/connections.py | 6 +- core/dbt/events/proto_types.py | 4 +- core/dbt/events/test_types.py | 15 - core/dbt/events/types.proto | 4 +- core/dbt/events/types.py | 359 +-------------------- tests/unit/test_events.py | 440 ++++++++++++++------------ 6 files changed, 256 insertions(+), 572 deletions(-) diff --git a/core/dbt/adapters/base/connections.py b/core/dbt/adapters/base/connections.py index 3c5db18c8a7..ea7150a1b80 100644 --- a/core/dbt/adapters/base/connections.py +++ b/core/dbt/adapters/base/connections.py @@ -41,8 +41,8 @@ from dbt.events.types import ( NewConnection, ConnectionReused, + ConnectionLeftOpenInCleanup, ConnectionLeftOpen, - ConnectionLeftOpen2, ConnectionClosedInCleanup, ConnectionClosed, Rollback, @@ -306,7 +306,7 @@ def cleanup_all(self) -> None: with self.lock: for connection in self.thread_connections.values(): if connection.state not in {"closed", "init"}: - fire_event(ConnectionLeftOpen(conn_name=cast_to_str(connection.name))) + fire_event(ConnectionLeftOpenInCleanup(conn_name=cast_to_str(connection.name))) else: fire_event(ConnectionClosedInCleanup(conn_name=cast_to_str(connection.name))) self.close(connection) @@ -348,7 +348,7 @@ def _close_handle(cls, connection: Connection) -> None: fire_event(ConnectionClosed(conn_name=cast_to_str(connection.name))) connection.handle.close() else: - fire_event(ConnectionLeftOpen2(conn_name=cast_to_str(connection.name))) + fire_event(ConnectionLeftOpen(conn_name=cast_to_str(connection.name))) @classmethod def _rollback(cls, connection: Connection) -> None: diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index eefd0f4b104..10fef0f06a2 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -398,7 +398,7 @@ class ConnectionReused(betterproto.Message): @dataclass -class ConnectionLeftOpen(betterproto.Message): +class ConnectionLeftOpenInCleanup(betterproto.Message): """E007""" info: "EventInfo" = betterproto.message_field(1) @@ -431,7 +431,7 @@ class ConnectionClosed(betterproto.Message): @dataclass -class ConnectionLeftOpen2(betterproto.Message): +class ConnectionLeftOpen(betterproto.Message): """E011""" info: "EventInfo" = betterproto.message_field(1) diff --git a/core/dbt/events/test_types.py b/core/dbt/events/test_types.py index 5f4a10cd7d7..cf7307125ca 100644 --- a/core/dbt/events/test_types.py +++ b/core/dbt/events/test_types.py @@ -61,18 +61,3 @@ def code(self): def message(self) -> str: return f"Unit Test: {self.msg}" - - -# since mypy doesn't run on every file we need to suggest to mypy that every -# class gets instantiated. But we don't actually want to run this code. -# making the conditional `if False` causes mypy to skip it as dead code so -# we need to skirt around that by computing something it doesn't check statically. -# -# TODO remove these lines once we run mypy everywhere. -if 1 == 0: - IntegrationTestInfo(msg="") - IntegrationTestDebug(msg="") - IntegrationTestWarn(msg="") - IntegrationTestError(msg="") - IntegrationTestException(msg="") - UnitTestInfo(msg="") diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index 394070b84d1..c18b94d5e1c 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -308,7 +308,7 @@ message ConnectionReused { } // E007 -message ConnectionLeftOpen { +message ConnectionLeftOpenInCleanup { EventInfo info = 1; string conn_name = 2; } @@ -333,7 +333,7 @@ message ConnectionClosed { } // E011 -message ConnectionLeftOpen2 { +message ConnectionLeftOpen { EventInfo info = 1; string conn_name = 2; } diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index a5c4e661b78..6444565718c 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -16,7 +16,7 @@ # The generated classes quote the included message classes, requiring the following line from dbt.events.proto_types import EventInfo, RunResultMsg, ListOfStrings # noqa -from dbt.events.proto_types import NodeInfo, ReferenceKeyMsg +from dbt.events.proto_types import NodeInfo, ReferenceKeyMsg # noqa from dbt.events import proto_types as pt from dbt.node_types import NodeType @@ -476,7 +476,7 @@ def message(self) -> str: @dataclass -class ConnectionLeftOpen(DebugLevel, pt.ConnectionLeftOpen): +class ConnectionLeftOpenInCleanup(DebugLevel, pt.ConnectionLeftOpenInCleanup): def code(self): return "E007" @@ -514,7 +514,7 @@ def message(self) -> str: # TODO: can we combine this with ConnectionLeftOpen? @dataclass -class ConnectionLeftOpen2(DebugLevel, pt.ConnectionLeftOpen2): +class ConnectionLeftOpen(DebugLevel, pt.ConnectionLeftOpen): def code(self): return "E011" @@ -2731,356 +2731,3 @@ def code(self): def message(self) -> str: # This is the message on the result object, cannot be formatted in event return self.msg - - -# since mypy doesn't run on every file we need to suggest to mypy that every -# class gets instantiated. But we don't actually want to run this code. -# making the conditional `if False` causes mypy to skip it as dead code so -# we need to skirt around that by computing something it doesn't check statically. -# -# TODO remove these lines once we run mypy everywhere. -if 1 == 0: - - # A - pre-project loading - MainReportVersion(version="") - MainReportArgs(args={}) - MainTrackingUserState(user_state="") - MergedFromState(num_merged=0, sample=[]) - MissingProfileTarget(profile_name="", target_name="") - InvalidVarsYAML() - DbtProjectError() - DbtProjectErrorException(exc="") - DbtProfileError() - DbtProfileErrorException(exc="") - ProfileListTitle() - ListSingleProfile(profile="") - NoDefinedProfiles() - ProfileHelpMessage() - StarterProjectPath(dir="") - ConfigFolderDirectory(dir="") - NoSampleProfileFound(adapter="") - ProfileWrittenWithSample(name="", path="") - ProfileWrittenWithTargetTemplateYAML(name="", path="") - ProfileWrittenWithProjectTemplateYAML(name="", path="") - SettingUpProfile() - InvalidProfileTemplateYAML() - ProjectNameAlreadyExists(name="") - ProjectCreated(project_name="") - - # D - Deprecations ====================== - PackageRedirectDeprecation(old_name="", new_name="") - PackageInstallPathDeprecation() - ConfigSourcePathDeprecation(deprecated_path="", exp_path="") - ConfigDataPathDeprecation(deprecated_path="", exp_path="") - AdapterDeprecationWarning(old_name="", new_name="") - MetricAttributesRenamed(metric_name="") - ExposureNameDeprecation(exposure="") - - # E - DB Adapter ====================== - AdapterEventDebug() - AdapterEventInfo() - AdapterEventWarning() - AdapterEventError() - NewConnection(conn_type="", conn_name="") - ConnectionReused(conn_name="") - ConnectionLeftOpen(conn_name="") - ConnectionClosedInCleanup(conn_name="") - RollbackFailed(conn_name="") - ConnectionClosed(conn_name="") - ConnectionLeftOpen2(conn_name="") - Rollback(conn_name="") - CacheMiss(conn_name="", database="", schema="") - ListRelations(database="", schema="") - ConnectionUsed(conn_type="", conn_name="") - SQLQuery(conn_name="", sql="") - SQLQueryStatus(status="", elapsed=0.1) - SQLCommit(conn_name="") - ColTypeChange( - orig_type="", new_type="", table=ReferenceKeyMsg(database="", schema="", identifier="") - ) - SchemaCreation(relation=ReferenceKeyMsg(database="", schema="", identifier="")) - SchemaDrop(relation=ReferenceKeyMsg(database="", schema="", identifier="")) - UncachedRelation( - dep_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ref_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ) - AddLink( - dep_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ref_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ) - AddRelation(relation=ReferenceKeyMsg(database="", schema="", identifier="")) - DropMissingRelation(relation=ReferenceKeyMsg(database="", schema="", identifier="")) - DropCascade( - dropped=ReferenceKeyMsg(database="", schema="", identifier=""), - consequences=[ReferenceKeyMsg(database="", schema="", identifier="")], - ) - DropRelation(dropped=ReferenceKeyMsg()) - UpdateReference( - old_key=ReferenceKeyMsg(database="", schema="", identifier=""), - new_key=ReferenceKeyMsg(database="", schema="", identifier=""), - cached_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ) - TemporaryRelation(key=ReferenceKeyMsg(database="", schema="", identifier="")) - RenameSchema( - old_key=ReferenceKeyMsg(database="", schema="", identifier=""), - new_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ) - DumpBeforeAddGraph(dump=dict()) - DumpAfterAddGraph(dump=dict()) - DumpBeforeRenameSchema(dump=dict()) - DumpAfterRenameSchema(dump=dict()) - AdapterImportError(exc="") - PluginLoadError(exc_info="") - NewConnectionOpening(connection_state="") - CodeExecution(conn_name="", code_content="") - CodeExecutionStatus(status="", elapsed=0.1) - CatalogGenerationError(exc="") - WriteCatalogFailure(num_exceptions=0) - CatalogWritten(path="") - CannotGenerateDocs() - BuildingCatalog() - DatabaseErrorRunningHook(hook_type="") - HooksRunning(num_hooks=0, hook_type="") - HookFinished(stat_line="", execution="", execution_time=0) - - # I - Project parsing ====================== - ParseCmdStart() - ParseCmdCompiling() - ParseCmdWritingManifest() - ParseCmdDone() - ManifestDependenciesLoaded() - ManifestLoaderCreated() - ManifestLoaded() - ManifestChecked() - ManifestFlatGraphBuilt() - ParseCmdPerfInfoPath(path="") - GenericTestFileParse(path="") - MacroFileParse(path="") - PartialParsingFullReparseBecauseOfError() - PartialParsingExceptionFile(file="") - PartialParsingFile(file_id="") - PartialParsingException(exc_info={}) - PartialParsingSkipParsing() - PartialParsingMacroChangeStartFullParse() - PartialParsingProjectEnvVarsChanged() - PartialParsingProfileEnvVarsChanged() - PartialParsingDeletedMetric(unique_id="") - ManifestWrongMetadataVersion(version="") - PartialParsingVersionMismatch(saved_version="", current_version="") - PartialParsingFailedBecauseConfigChange() - PartialParsingFailedBecauseProfileChange() - PartialParsingFailedBecauseNewProjectDependency() - PartialParsingFailedBecauseHashChanged() - PartialParsingNotEnabled() - ParsedFileLoadFailed(path="", exc="", exc_info="") - PartialParseSaveFileNotFound() - StaticParserCausedJinjaRendering(path="") - UsingExperimentalParser(path="") - SampleFullJinjaRendering(path="") - StaticParserFallbackJinjaRendering(path="") - StaticParsingMacroOverrideDetected(path="") - StaticParserSuccess(path="") - StaticParserFailure(path="") - ExperimentalParserSuccess(path="") - ExperimentalParserFailure(path="") - PartialParsingEnabled(deleted=0, added=0, changed=0) - PartialParsingAddedFile(file_id="") - PartialParsingDeletedFile(file_id="") - PartialParsingUpdatedFile(file_id="") - PartialParsingNodeMissingInSourceFile(file_id="") - PartialParsingMissingNodes(file_id="") - PartialParsingChildMapMissingUniqueID(unique_id="") - PartialParsingUpdateSchemaFile(file_id="") - PartialParsingDeletedSource(unique_id="") - PartialParsingDeletedExposure(unique_id="") - InvalidDisabledTargetInTestNode( - resource_type_title="", - unique_id="", - original_file_path="", - target_kind="", - target_name="", - target_package="", - ) - UnusedResourceConfigPath(unused_config_paths=[]) - SeedIncreased(package_name="", name="") - SeedExceedsLimitSamePath(package_name="", name="") - SeedExceedsLimitAndPathChanged(package_name="", name="") - SeedExceedsLimitChecksumChanged(package_name="", name="", checksum_name="") - UnusedTables(unused_tables=[]) - WrongResourceSchemaFile(patch_name="", resource_type="", file_path="", plural_resource_type="") - NoNodeForYamlKey(patch_name="", yaml_key="", file_path="") - MacroPatchNotFound(patch_name="") - NodeNotFoundOrDisabled( - original_file_path="", - unique_id="", - resource_type_title="", - target_name="", - target_kind="", - target_package="", - disabled="", - ) - - # M - Deps generation ====================== - - GitSparseCheckoutSubdirectory(subdir="") - GitProgressCheckoutRevision(revision="") - GitProgressUpdatingExistingDependency(dir="") - GitProgressPullingNewDependency(dir="") - GitNothingToDo(sha="") - GitProgressUpdatedCheckoutRange(start_sha="", end_sha="") - GitProgressCheckedOutAt(end_sha="") - RegistryProgressGETRequest(url="") - RegistryProgressGETResponse(url="", resp_code=1234) - SelectorReportInvalidSelector(valid_selectors="", spec_method="", raw_spec="") - JinjaLogInfo(msg="") - JinjaLogDebug(msg="") - DepsNoPackagesFound() - DepsStartPackageInstall(package_name="") - DepsInstallInfo(version_name="") - DepsUpdateAvailable(version_latest="") - DepsUpToDate() - DepsListSubdirectory(subdirectory="") - DepsNotifyUpdatesAvailable(packages=ListOfStrings()) - RetryExternalCall(attempt=0, max=0) - RecordRetryException(exc="") - RegistryIndexProgressGETRequest(url="") - RegistryIndexProgressGETResponse(url="", resp_code=1234) - RegistryResponseUnexpectedType(response=""), - RegistryResponseMissingTopKeys(response=""), - RegistryResponseMissingNestedKeys(response=""), - RegistryResponseExtraNestedKeys(response=""), - DepsSetDownloadDirectory(path="") - - # Q - Node execution ====================== - - RunningOperationCaughtError(exc="") - CompileComplete() - FreshnessCheckComplete() - SeedHeader(header="") - SeedHeaderSeparator(len_header=0) - SQLRunnerException(exc="") - LogTestResult( - name="", - index=0, - num_models=0, - execution_time=0, - num_failures=0, - ) - LogStartLine(description="", index=0, total=0, node_info=NodeInfo()) - LogModelResult( - description="", - status="", - index=0, - total=0, - execution_time=0, - ) - LogSnapshotResult( - status="", - description="", - cfg={}, - index=0, - total=0, - execution_time=0, - ) - LogSeedResult( - status="", - index=0, - total=0, - execution_time=0, - schema="", - relation="", - ) - LogFreshnessResult( - source_name="", - table_name="", - index=0, - total=0, - execution_time=0, - ) - LogCancelLine(conn_name="") - DefaultSelector(name="") - NodeStart(unique_id="") - NodeFinished(unique_id="") - QueryCancelationUnsupported(type="") - ConcurrencyLine(num_threads=0, target_name="") - CompilingNode(unique_id="") - WritingInjectedSQLForNode(unique_id="") - NodeCompiling(unique_id="") - NodeExecuting(unique_id="") - LogHookStartLine( - statement="", - index=0, - total=0, - ) - LogHookEndLine( - statement="", - status="", - index=0, - total=0, - execution_time=0, - ) - SkippingDetails( - resource_type="", - schema="", - node_name="", - index=0, - total=0, - ) - NothingToDo() - RunningOperationUncaughtError(exc="") - EndRunResult() - NoNodesSelected() - DepsUnpinned(revision="", git="") - NoNodesForSelectionCriteria(spec_raw="") - - # W - Node testing ====================== - - CatchableExceptionOnRun(exc="") - InternalExceptionOnRun(build_path="", exc="") - GenericExceptionOnRun(build_path="", unique_id="", exc="") - NodeConnectionReleaseError(node_name="", exc="") - FoundStats(stat_line="") - - # Z - misc ====================== - - MainKeyboardInterrupt() - MainEncounteredError(exc="") - MainStackTrace(stack_trace="") - SystemErrorRetrievingModTime(path="") - SystemCouldNotWrite(path="", reason="", exc="") - SystemExecutingCmd(cmd=[""]) - SystemStdOutMsg(bmsg=b"") - SystemStdErrMsg(bmsg=b"") - SystemReportReturnCode(returncode=0) - TimingInfoCollected() - LogDebugStackTrace() - CheckCleanPath(path="") - ConfirmCleanPath(path="") - ProtectedCleanPath(path="") - FinishedCleanPaths() - OpenCommand(open_cmd="", profiles_dir="") - EmptyLine() - ServingDocsPort(address="", port=0) - ServingDocsAccessInfo(port="") - ServingDocsExitInfo() - RunResultWarning(resource_type="", node_name="", path="") - RunResultFailure(resource_type="", node_name="", path="") - StatsLine(stats={}) - RunResultError(msg="") - RunResultErrorNoMessage(status="") - SQLCompiledPath(path="") - CheckNodeTestFailure(relation_name="") - FirstRunResultError(msg="") - AfterFirstRunResultError(msg="") - EndOfRunSummary(num_errors=0, num_warnings=0, keyboard_interrupt=False) - LogSkipBecauseError(schema="", relation="", index=0, total=0) - EnsureGitInstalled() - DepsCreatingLocalSymlink() - DepsSymlinkNotAvailable() - DisableTracking() - SendingEvent(kwargs="") - SendEventFailure() - FlushEvents() - FlushEventsFailure() - TrackingInitializeFailure() - EventBufferFull() diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 371dd13300e..fd99a7cf416 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -1,7 +1,7 @@ # flake8: noqa from dbt.events.test_types import UnitTestInfo from dbt.events import AdapterLogger -from dbt.events.functions import event_to_json, LOG_VERSION, reset_event_history +from dbt.events.functions import event_to_json, LOG_VERSION, reset_event_history, event_to_dict from dbt.events.types import * from dbt.events.test_types import * @@ -29,10 +29,8 @@ def get_all_subclasses(cls): all_subclasses = [] for subclass in cls.__subclasses__(): - # If the test breaks because of abcs this list might have to be updated. - if subclass in [TestLevel, DebugLevel, WarnLevel, InfoLevel, ErrorLevel, DynamicLevel]: - continue - all_subclasses.append(subclass) + if subclass not in [TestLevel, DebugLevel, WarnLevel, InfoLevel, ErrorLevel, DynamicLevel]: + all_subclasses.append(subclass) all_subclasses.extend(get_all_subclasses(subclass)) return set(all_subclasses) @@ -93,15 +91,15 @@ def test_event_codes(self): all_concrete = get_all_subclasses(BaseEvent) all_codes = set() - for event in all_concrete: - if not inspect.isabstract(event): - # must be in the form 1 capital letter, 3 digits - assert re.match("^[A-Z][0-9]{3}", event.info.code) - # cannot have been used already - assert ( - event.info.code not in all_codes - ), f"{event.code} is assigned more than once. Check types.py for duplicates." - all_codes.add(event.info.code) + for event_cls in all_concrete: + code = event_cls.code(event_cls) + # must be in the form 1 capital letter, 3 digits + assert re.match("^[A-Z][0-9]{3}", code) + # cannot have been used already + assert ( + code not in all_codes + ), f"{code} is assigned more than once. Check types.py for duplicates." + all_codes.add(code) class TestEventBuffer: @@ -164,56 +162,62 @@ def MockNode(): sample_values = [ - MainReportVersion(version="", log_version=LOG_VERSION), - MainKeyboardInterrupt(), - MainEncounteredError(exc=""), - MainStackTrace(stack_trace=""), + # A - pre-project loading + MainReportVersion(version=""), + MainReportArgs(args={}), MainTrackingUserState(user_state=""), - ParseCmdStart(), - ParseCmdCompiling(), - ParseCmdWritingManifest(), - ParseCmdDone(), - ManifestDependenciesLoaded(), - ManifestLoaderCreated(), - ManifestLoaded(), - ManifestChecked(), - ManifestFlatGraphBuilt(), - ParseCmdPerfInfoPath(path=""), - GitSparseCheckoutSubdirectory(subdir=""), - GitProgressCheckoutRevision(revision=""), - GitProgressUpdatingExistingDependency(dir=""), - GitProgressPullingNewDependency(dir=""), - GitNothingToDo(sha=""), - GitProgressUpdatedCheckoutRange(start_sha="", end_sha=""), - GitProgressCheckedOutAt(end_sha=""), - SystemErrorRetrievingModTime(path=""), - SystemCouldNotWrite(path="", reason="", exc=""), - SystemExecutingCmd(cmd=[""]), - SystemStdOutMsg(bmsg=b""), - SystemStdErrMsg(bmsg=b""), - SelectorReportInvalidSelector(valid_selectors="", spec_method="", raw_spec=""), - JinjaLogInfo(msg=""), - JinjaLogDebug(msg=""), + MergedFromState(num_merged=0, sample=[]), + MissingProfileTarget(profile_name="", target_name=""), + InvalidVarsYAML(), + DbtProjectError(), + DbtProjectErrorException(exc=""), + DbtProfileError(), + DbtProfileErrorException(exc=""), + ProfileListTitle(), + ListSingleProfile(profile=""), + NoDefinedProfiles(), + ProfileHelpMessage(), + StarterProjectPath(dir=""), + ConfigFolderDirectory(dir=""), + NoSampleProfileFound(adapter=""), + ProfileWrittenWithSample(name="", path=""), + ProfileWrittenWithTargetTemplateYAML(name="", path=""), + ProfileWrittenWithProjectTemplateYAML(name="", path=""), + SettingUpProfile(), + InvalidProfileTemplateYAML(), + ProjectNameAlreadyExists(name=""), + ProjectCreated(project_name=""), + + # D - Deprecations ====================== + PackageRedirectDeprecation(old_name="", new_name=""), + PackageInstallPathDeprecation(), + ConfigSourcePathDeprecation(deprecated_path="", exp_path=""), + ConfigDataPathDeprecation(deprecated_path="", exp_path=""), + AdapterDeprecationWarning(old_name="", new_name=""), + MetricAttributesRenamed(metric_name=""), + ExposureNameDeprecation(exposure=""), + + # E - DB Adapter ====================== + AdapterEventDebug(), + AdapterEventInfo(), + AdapterEventWarning(), + AdapterEventError(), NewConnection(conn_type="", conn_name=""), ConnectionReused(conn_name=""), - ConnectionLeftOpen(conn_name=""), + ConnectionLeftOpenInCleanup(conn_name=""), ConnectionClosedInCleanup(conn_name=""), RollbackFailed(conn_name=""), ConnectionClosed(conn_name=""), - ConnectionLeftOpen2(conn_name=""), + ConnectionLeftOpen(conn_name=""), Rollback(conn_name=""), CacheMiss(conn_name="", database="", schema=""), - ListRelations(database="", schema="", relations=[]), + ListRelations(database="", schema=""), ConnectionUsed(conn_type="", conn_name=""), SQLQuery(conn_name="", sql=""), SQLQueryStatus(status="", elapsed=0.1), - CodeExecution(conn_name="", code_content=""), - CodeExecutionStatus(status="", elapsed=0.1), SQLCommit(conn_name=""), ColTypeChange( - orig_type="", - new_type="", - table=ReferenceKeyMsg(database="", schema="", identifier=""), + orig_type="", new_type="", table=ReferenceKeyMsg(database="", schema="", identifier="") ), SchemaCreation(relation=ReferenceKeyMsg(database="", schema="", identifier="")), SchemaDrop(relation=ReferenceKeyMsg(database="", schema="", identifier="")), @@ -231,6 +235,7 @@ def MockNode(): dropped=ReferenceKeyMsg(database="", schema="", identifier=""), consequences=[ReferenceKeyMsg(database="", schema="", identifier="")], ), + DropRelation(dropped=ReferenceKeyMsg()), UpdateReference( old_key=ReferenceKeyMsg(database="", schema="", identifier=""), new_key=ReferenceKeyMsg(database="", schema="", identifier=""), @@ -246,29 +251,49 @@ def MockNode(): DumpBeforeRenameSchema(dump=dict()), DumpAfterRenameSchema(dump=dict()), AdapterImportError(exc=""), - PluginLoadError(), - SystemReportReturnCode(returncode=0), + PluginLoadError(exc_info=""), NewConnectionOpening(connection_state=""), - TimingInfoCollected(), - MergedFromState(num_merged=0, sample=[]), - MissingProfileTarget(profile_name="", target_name=""), - InvalidVarsYAML(), + CodeExecution(conn_name="", code_content=""), + CodeExecutionStatus(status="", elapsed=0.1), + CatalogGenerationError(exc=""), + WriteCatalogFailure(num_exceptions=0), + CatalogWritten(path=""), + CannotGenerateDocs(), + BuildingCatalog(), + DatabaseErrorRunningHook(hook_type=""), + HooksRunning(num_hooks=0, hook_type=""), + HookFinished(stat_line="", execution="", execution_time=0), + + # I - Project parsing ====================== + ParseCmdStart(), + ParseCmdCompiling(), + ParseCmdWritingManifest(), + ParseCmdDone(), + ManifestDependenciesLoaded(), + ManifestLoaderCreated(), + ManifestLoaded(), + ManifestChecked(), + ManifestFlatGraphBuilt(), + ParseCmdPerfInfoPath(path=""), GenericTestFileParse(path=""), MacroFileParse(path=""), PartialParsingFullReparseBecauseOfError(), - PartialParsingFile(file_id=""), PartialParsingExceptionFile(file=""), + PartialParsingFile(file_id=""), PartialParsingException(exc_info={}), PartialParsingSkipParsing(), PartialParsingMacroChangeStartFullParse(), + PartialParsingProjectEnvVarsChanged(), + PartialParsingProfileEnvVarsChanged(), + PartialParsingDeletedMetric(unique_id=""), ManifestWrongMetadataVersion(version=""), PartialParsingVersionMismatch(saved_version="", current_version=""), PartialParsingFailedBecauseConfigChange(), PartialParsingFailedBecauseProfileChange(), PartialParsingFailedBecauseNewProjectDependency(), PartialParsingFailedBecauseHashChanged(), - PartialParsingDeletedMetric(unique_id=""), - ParsedFileLoadFailed(path="", exc=""), + PartialParsingNotEnabled(), + ParsedFileLoadFailed(path="", exc="", exc_info=""), PartialParseSaveFileNotFound(), StaticParserCausedJinjaRendering(path=""), UsingExperimentalParser(path=""), @@ -297,50 +322,172 @@ def MockNode(): target_name="", target_package="", ), + UnusedResourceConfigPath(unused_config_paths=[]), + SeedIncreased(package_name="", name=""), + SeedExceedsLimitSamePath(package_name="", name=""), + SeedExceedsLimitAndPathChanged(package_name="", name=""), + SeedExceedsLimitChecksumChanged(package_name="", name="", checksum_name=""), + UnusedTables(unused_tables=[]), + WrongResourceSchemaFile(patch_name="", resource_type="", file_path="", plural_resource_type=""), + NoNodeForYamlKey(patch_name="", yaml_key="", file_path=""), + MacroPatchNotFound(patch_name=""), + NodeNotFoundOrDisabled( + original_file_path="", + unique_id="", + resource_type_title="", + target_name="", + target_kind="", + target_package="", + disabled="", + ), + GeneralMacroWarning(), + + # M - Deps generation ====================== + + GitSparseCheckoutSubdirectory(subdir=""), + GitProgressCheckoutRevision(revision=""), + GitProgressUpdatingExistingDependency(dir=""), + GitProgressPullingNewDependency(dir=""), + GitNothingToDo(sha=""), + GitProgressUpdatedCheckoutRange(start_sha="", end_sha=""), + GitProgressCheckedOutAt(end_sha=""), + RegistryProgressGETRequest(url=""), + RegistryProgressGETResponse(url="", resp_code=1234), + SelectorReportInvalidSelector(valid_selectors="", spec_method="", raw_spec=""), + JinjaLogInfo(msg=""), + JinjaLogDebug(msg=""), + DepsNoPackagesFound(), + DepsStartPackageInstall(package_name=""), + DepsInstallInfo(version_name=""), + DepsUpdateAvailable(version_latest=""), + DepsUpToDate(), + DepsListSubdirectory(subdirectory=""), + DepsNotifyUpdatesAvailable(packages=ListOfStrings()), + RetryExternalCall(attempt=0, max=0), + RecordRetryException(exc=""), + RegistryIndexProgressGETRequest(url=""), + RegistryIndexProgressGETResponse(url="", resp_code=1234), + RegistryResponseUnexpectedType(response=""), + RegistryResponseMissingTopKeys(response=""), + RegistryResponseMissingNestedKeys(response=""), + RegistryResponseExtraNestedKeys(response=""), + DepsSetDownloadDirectory(path=""), + + # Q - Node execution ====================== + RunningOperationCaughtError(exc=""), + CompileComplete(), + FreshnessCheckComplete(), + SeedHeader(header=""), + SeedHeaderSeparator(len_header=0), + SQLRunnerException(exc=""), + LogTestResult( + name="", + index=0, + num_models=0, + execution_time=0, + num_failures=0, + ), + LogStartLine(description="", index=0, total=0, node_info=NodeInfo()), + LogModelResult( + description="", + status="", + index=0, + total=0, + execution_time=0, + ), + LogSnapshotResult( + status="", + description="", + cfg={}, + index=0, + total=0, + execution_time=0, + ), + LogSeedResult( + status="", + index=0, + total=0, + execution_time=0, + schema="", + relation="", + ), + LogFreshnessResult( + source_name="", + table_name="", + index=0, + total=0, + execution_time=0, + ), + LogCancelLine(conn_name=""), + DefaultSelector(name=""), + NodeStart(unique_id=""), + NodeFinished(unique_id=""), + QueryCancelationUnsupported(type=""), + ConcurrencyLine(num_threads=0, target_name=""), + CompilingNode(unique_id=""), + WritingInjectedSQLForNode(unique_id=""), + NodeCompiling(unique_id=""), + NodeExecuting(unique_id=""), + LogHookStartLine( + statement="", + index=0, + total=0, + ), + LogHookEndLine( + statement="", + status="", + index=0, + total=0, + execution_time=0, + ), + SkippingDetails( + resource_type="", + schema="", + node_name="", + index=0, + total=0, + ), + NothingToDo(), RunningOperationUncaughtError(exc=""), - DbtProjectError(), - DbtProjectErrorException(exc=""), - DbtProfileError(), - DbtProfileErrorException(exc=""), - ProfileListTitle(), - ListSingleProfile(profile=""), - NoDefinedProfiles(), - ProfileHelpMessage(), + EndRunResult(), + NoNodesSelected(), + DepsUnpinned(revision="", git=""), + NoNodesForSelectionCriteria(spec_raw=""), + + # W - Node testing ====================== + CatchableExceptionOnRun(exc=""), InternalExceptionOnRun(build_path="", exc=""), GenericExceptionOnRun(build_path="", unique_id="", exc=""), NodeConnectionReleaseError(node_name="", exc=""), + FoundStats(stat_line=""), + + # Z - misc ====================== + + MainKeyboardInterrupt(), + MainEncounteredError(exc=""), + MainStackTrace(stack_trace=""), + SystemErrorRetrievingModTime(path=""), + SystemCouldNotWrite(path="", reason="", exc=""), + SystemExecutingCmd(cmd=[""]), + SystemStdOutMsg(bmsg=b""), + SystemStdErrMsg(bmsg=b""), + SystemReportReturnCode(returncode=0), + TimingInfoCollected(), + LogDebugStackTrace(), CheckCleanPath(path=""), ConfirmCleanPath(path=""), ProtectedCleanPath(path=""), FinishedCleanPaths(), OpenCommand(open_cmd="", profiles_dir=""), - DepsNoPackagesFound(), - DepsStartPackageInstall(package_name=""), - DepsInstallInfo(version_name=""), - DepsUpdateAvailable(version_latest=""), - DepsListSubdirectory(subdirectory=""), - DepsNotifyUpdatesAvailable(packages=ListOfStrings()), - DepsNotifyUpdatesAvailable(packages=ListOfStrings(['dbt-utils'])), - DatabaseErrorRunningHook(hook_type=""), EmptyLine(), - HooksRunning(num_hooks=0, hook_type=""), - HookFinished(stat_line="", execution="", execution_time=0), - WriteCatalogFailure(num_exceptions=0), - CatalogWritten(path=""), - CannotGenerateDocs(), - BuildingCatalog(), - CompileComplete(), - FreshnessCheckComplete(), ServingDocsPort(address="", port=0), ServingDocsAccessInfo(port=""), ServingDocsExitInfo(), - SeedHeader(header=""), - SeedHeaderSeparator(len_header=0), RunResultWarning(resource_type="", node_name="", path=""), RunResultFailure(resource_type="", node_name="", path=""), - StatsLine(stats={"pass": 0, "warn": 0, "error": 0, "skip": 0, "total": 0}), + StatsLine(stats={"error": 0, "skip": 0, "pass": 0, "warn": 0,"total": 0}), RunResultError(msg=""), RunResultErrorNoMessage(status=""), SQLCompiledPath(path=""), @@ -348,90 +495,27 @@ def MockNode(): FirstRunResultError(msg=""), AfterFirstRunResultError(msg=""), EndOfRunSummary(num_errors=0, num_warnings=0, keyboard_interrupt=False), - LogStartLine(description="", index=0, total=0, node_info=NodeInfo()), - LogHookStartLine(statement="", index=0, total=0, node_info=NodeInfo()), - LogHookEndLine( - statement="", status="", index=0, total=0, execution_time=0, node_info=NodeInfo() - ), - SkippingDetails( - resource_type="", schema="", node_name="", index=0, total=0, node_info=NodeInfo() - ), - LogTestResult( - name="", index=0, num_models=0, execution_time=0, num_failures=0, node_info=NodeInfo() - ), LogSkipBecauseError(schema="", relation="", index=0, total=0), - LogModelResult( - description="", status="", index=0, total=0, execution_time=0, node_info=NodeInfo() - ), - LogSnapshotResult( - status="", description="", cfg={}, index=0, total=0, execution_time=0, node_info=NodeInfo() - ), - LogSeedResult( - status="", index=0, total=0, execution_time=0, schema="", relation="", node_info=NodeInfo() - ), - LogFreshnessResult( - source_name="", table_name="", index=0, total=0, execution_time=0, node_info=NodeInfo() - ), - LogCancelLine(conn_name=""), - DefaultSelector(name=""), - NodeStart(unique_id="", node_info=NodeInfo()), - NodeCompiling(unique_id="", node_info=NodeInfo()), - NodeExecuting(unique_id="", node_info=NodeInfo()), - NodeFinished(unique_id="", node_info=NodeInfo(), run_result=RunResultMsg()), - QueryCancelationUnsupported(type=""), - ConcurrencyLine(num_threads=0, target_name=""), - StarterProjectPath(dir=""), - ConfigFolderDirectory(dir=""), - NoSampleProfileFound(adapter=""), - ProfileWrittenWithSample(name="", path=""), - ProfileWrittenWithTargetTemplateYAML(name="", path=""), - ProfileWrittenWithProjectTemplateYAML(name="", path=""), - SettingUpProfile(), - InvalidProfileTemplateYAML(), - ProjectNameAlreadyExists(name=""), - ProjectCreated(project_name="", docs_url="", slack_url=""), - DepsSetDownloadDirectory(path=""), EnsureGitInstalled(), DepsCreatingLocalSymlink(), DepsSymlinkNotAvailable(), - FoundStats(stat_line=""), - CompilingNode(unique_id=""), - WritingInjectedSQLForNode(unique_id=""), DisableTracking(), SendingEvent(kwargs=""), SendEventFailure(), FlushEvents(), FlushEventsFailure(), TrackingInitializeFailure(), - RetryExternalCall(attempt=0, max=0), - PartialParsingProfileEnvVarsChanged(), - AdapterEventDebug(name="", base_msg="", args=()), - AdapterEventInfo(name="", base_msg="", args=()), - AdapterEventWarning(name="", base_msg="", args=()), - AdapterEventError(name="", base_msg="", args=()), - LogDebugStackTrace(), - MainReportArgs(args={}), - RegistryProgressGETRequest(url=""), - RegistryIndexProgressGETRequest(url=""), - RegistryIndexProgressGETResponse(url="", resp_code=1), - RegistryResponseUnexpectedType(response=""), - RegistryResponseMissingTopKeys(response=""), - RegistryResponseMissingNestedKeys(response=""), - RegistryResponseExtraNestedKeys(response=""), - DepsUpToDate(), - PartialParsingNotEnabled(), - SQLRunnerException(exc=""), - DropRelation(dropped=ReferenceKeyMsg(database="", schema="", identifier="")), - PartialParsingProjectEnvVarsChanged(), - RegistryProgressGETResponse(url="", resp_code=1), - IntegrationTestDebug(msg=""), - IntegrationTestInfo(msg=""), - IntegrationTestWarn(msg=""), - IntegrationTestError(msg=""), - IntegrationTestException(msg=""), EventBufferFull(), - RecordRetryException(exc=""), - UnitTestInfo(msg=""), + RunResultWarningMessage(), + + # T - tests ====================== + IntegrationTestInfo(), + IntegrationTestDebug(), + IntegrationTestWarn(), + IntegrationTestError(), + IntegrationTestException(), + UnitTestInfo(), + ] @@ -441,13 +525,8 @@ class TestEventJSONSerialization: # event types that take `Any` are not possible to test in this way since some will serialize # just fine and others won't. def test_all_serializable(self): - no_test = [DummyCacheEvent] - all_non_abstract_events = set( - filter( - lambda x: not inspect.isabstract(x) and x not in no_test, - get_all_subclasses(BaseEvent), - ) + get_all_subclasses(BaseEvent), ) all_event_values_list = list(map(lambda x: x.__class__, sample_values)) diff = all_non_abstract_events.difference(set(all_event_values_list)) @@ -461,7 +540,7 @@ def test_all_serializable(self): # if we have everything we need to test, try to serialize everything for event in sample_values: - event_dict = event.to_dict() + event_dict = event_to_dict(event) try: event_json = event_to_json(event) except Exception as e: @@ -469,30 +548,3 @@ def test_all_serializable(self): T = TypeVar("T") - - -@dataclass -class Counter(Generic[T], SerializableType): - dummy_val: T - count: int = 0 - - def next(self) -> T: - self.count = self.count + 1 - return self.dummy_val - - # mashumaro serializer - def _serialize() -> Dict[str, int]: - return {"count": count} - - -@dataclass -class DummyCacheEvent(InfoLevel, Cache, SerializableType): - code = "X999" - counter: Counter - - def message(self) -> str: - return f"state: {self.counter.next()}" - - # mashumaro serializer - def _serialize() -> str: - return "DummyCacheEvent" From d1e2d03564955baeda919887ca475464be50a476 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 18 Nov 2022 13:35:29 -0500 Subject: [PATCH 5/6] Update events README.md, add "category" to EventInfo --- core/dbt/events/README.md | 7 ++++--- core/dbt/events/proto_types.py | 1 + core/dbt/events/types.proto | 1 + tests/unit/test_proto_events.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/dbt/events/README.md b/core/dbt/events/README.md index cadc59ab126..52edd7d35d4 100644 --- a/core/dbt/events/README.md +++ b/core/dbt/events/README.md @@ -8,9 +8,10 @@ The event module provides types that represent what is happening in dbt in `even When events are processed via `fire_event`, nearly everything is logged. Whether or not the user has enabled the debug flag, all debug messages are still logged to the file. However, some events are particularly time consuming to construct because they return a huge amount of data. Today, the only messages in this category are cache events and are only logged if the `--log-cache-events` flag is on. This is important because these messages should not be created unless they are going to be logged, because they cause a noticable performance degredation. These events use a "fire_event_if" functions. # Adding a New Event -New events need to have a proto message definition created in core/dbt/events/types.proto. Every message must include EventInfo as the first field, named "info" and numbered 1. To update the proto_types.py file, in the core/dbt/events directory: ```protoc --python_betterproto_out . types.proto``` - -A matching class needs to be created in the core/dbt/events/types.py file, which will have two superclasses, the "Level" mixin and the generated class from proto_types.py. These classes will also generally have two methods, a "code" method that returns the event code, and a "message" method that is used to construct the "msg" from the event fields. In addition the "Level" mixin will provide a "level_tag" method to set the level (which can also be overridden using the "info" convenience function from functions.py) +* Add a new message in types.proto with an EventInfo field first +* run the protoc compiler to update proto_types.py: ```protoc --python_betterproto_out . types.proto``` +* Add a wrapping class in core/dbt/event/types.py with a Level superclass and the superclass from proto_types.py, plus code and message methods +* Add the class to tests/unit/test_events.py Note that no attributes can exist in these event classes except for fields defined in the protobuf definitions, because the betterproto metaclass will throw an error. Betterproto provides a to_dict() method to convert the generated classes to a dictionary and from that to json. However some attributes will successfully convert to dictionaries but not to serialized protobufs, so we need to test both output formats. diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 10fef0f06a2..bdd3df5ae5e 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -23,6 +23,7 @@ class EventInfo(betterproto.Message): extra: Dict[str, str] = betterproto.map_field( 9, betterproto.TYPE_STRING, betterproto.TYPE_STRING ) + category: str = betterproto.string_field(10) @dataclass diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index c18b94d5e1c..da3e5240405 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -15,6 +15,7 @@ message EventInfo { string thread = 7; google.protobuf.Timestamp ts = 8; map extra = 9; + string category = 10; } // TimingInfo diff --git a/tests/unit/test_proto_events.py b/tests/unit/test_proto_events.py index 31837ed0271..d5b070c41e2 100644 --- a/tests/unit/test_proto_events.py +++ b/tests/unit/test_proto_events.py @@ -13,7 +13,7 @@ from dbt.version import installed -info_keys = {"name", "code", "msg", "level", "invocation_id", "pid", "thread", "ts", "extra"} +info_keys = {"name", "code", "msg", "level", "invocation_id", "pid", "thread", "ts", "extra", "category"} def test_events(): From 7584c88292c5fdf98c5eda701c394732427d45f1 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 18 Nov 2022 15:58:57 -0500 Subject: [PATCH 6/6] Rename GeneralMacroWarning to JinjaLogWarning --- core/dbt/events/proto_types.py | 2 +- core/dbt/events/types.proto | 2 +- core/dbt/events/types.py | 2 +- core/dbt/exceptions.py | 4 ++-- tests/unit/test_events.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index bdd3df5ae5e..c4d195e8777 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -1243,7 +1243,7 @@ class NodeNotFoundOrDisabled(betterproto.Message): @dataclass -class GeneralMacroWarning(betterproto.Message): +class JinjaLogWarning(betterproto.Message): """I061""" info: "EventInfo" = betterproto.message_field(1) diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index da3e5240405..2666a3565e0 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -944,7 +944,7 @@ message NodeNotFoundOrDisabled { } // I061 -message GeneralMacroWarning { +message JinjaLogWarning { EventInfo info = 1; string msg = 2; } diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 6444565718c..55439e0ec15 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -1519,7 +1519,7 @@ def message(self) -> str: @dataclass -class GeneralMacroWarning(WarnLevel, pt.GeneralMacroWarning): +class JinjaLogWarning(WarnLevel, pt.JinjaLogWarning): def code(self): return "I061" diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index f0eeb4f6d4f..05f3debafe6 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -3,7 +3,7 @@ from typing import NoReturn, Optional, Mapping, Any from dbt.events.helpers import env_secrets, scrub_secrets -from dbt.events.types import GeneralMacroWarning +from dbt.events.types import JinjaLogWarning from dbt.node_types import NodeType import dbt.dataclass_schema @@ -996,7 +996,7 @@ def raise_duplicate_alias( def warn(msg, node=None): - dbt.events.functions.warn_or_error(GeneralMacroWarning(msg=msg), node=node) + dbt.events.functions.warn_or_error(JinjaLogWarning(msg=msg), node=node) return "" diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index fd99a7cf416..eb4a87f495e 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -340,7 +340,7 @@ def MockNode(): target_package="", disabled="", ), - GeneralMacroWarning(), + JinjaLogWarning(), # M - Deps generation ======================