From 1c4ead35724c9a733b1fee89df3c5f18b32d4128 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 2 Jul 2018 15:53:15 -0400 Subject: [PATCH 1/5] remove run_error from tracking code --- dbt/node_runners.py | 1 - dbt/tracking.py | 40 +++++++++++++++++++--------------------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 3de7edd5abb..35870a7d51d 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -32,7 +32,6 @@ def track_model_run(index, num_nodes, run_model_result): "execution_time": run_model_result.execution_time, "run_status": run_model_result.status, "run_skipped": run_model_result.skip, - "run_error": run_model_result.error, "model_materialization": dbt.utils.get_materialization(run_model_result.node), # noqa "model_id": dbt.utils.get_hash(run_model_result.node), "hashed_contents": dbt.utils.get_hashed_contents(run_model_result.node), # noqa diff --git a/dbt/tracking.py b/dbt/tracking.py index 9d1da5f7f31..f9aa734ccdc 100644 --- a/dbt/tracking.py +++ b/dbt/tracking.py @@ -9,8 +9,6 @@ import uuid import yaml import os -import json -import logging import dbt.clients.system @@ -24,7 +22,7 @@ INVOCATION_SPEC = 'iglu:com.dbt/invocation/jsonschema/1-0-0' PLATFORM_SPEC = 'iglu:com.dbt/platform/jsonschema/1-0-0' -RUN_MODEL_SPEC = 'iglu:com.dbt/run_model/jsonschema/1-0-0' +RUN_MODEL_SPEC = 'iglu:com.dbt/run_model/jsonschema/2-0-0' INVOCATION_ENV_SPEC = 'iglu:com.dbt/invocation_env/jsonschema/1-0-0' PACKAGE_INSTALL_SPEC = 'iglu:com.dbt/package_install/jsonschema/1-0-0' @@ -78,7 +76,7 @@ def get_cookie(self): user = yaml.safe_load(fh) if user is None: user = self.set_cookie() - except yaml.reader.ReaderError as e: + except yaml.reader.ReaderError: user = self.set_cookie() return user @@ -89,15 +87,15 @@ def get_run_type(args): def get_invocation_context(user, project, args): return { - "project_id": None if project is None else project.hashed_name(), - "user_id": user.id, - "invocation_id": user.invocation_id, + "project_id": None if project is None else project.hashed_name(), + "user_id": user.id, + "invocation_id": user.invocation_id, - "command": args.which, - "options": None, - "version": dbt_version.installed, + "command": args.which, + "options": None, + "version": dbt_version.installed, - "run_type": get_run_type(args), + "run_type": get_run_type(args), } @@ -171,7 +169,7 @@ def track(user, *args, **kwargs): logger.debug("Sending event: {}".format(kwargs)) try: tracker.track_struct_event(*args, **kwargs) - except Exception as e: + except Exception: logger.debug( "An error was encountered while trying to send an event" ) @@ -179,9 +177,9 @@ def track(user, *args, **kwargs): def track_invocation_start(project=None, args=None): context = [ - get_invocation_start_context(active_user, project, args), - get_platform_context(), - get_dbt_env_context() + get_invocation_start_context(active_user, project, args), + get_platform_context(), + get_dbt_env_context() ] track( @@ -195,7 +193,7 @@ def track_invocation_start(project=None, args=None): def track_model_run(options): context = [SelfDescribingJson(RUN_MODEL_SPEC, options)] - model_id = options['model_id'] + track( active_user, category="dbt", @@ -241,11 +239,11 @@ def track_invalid_invocation( user = active_user invocation_context = get_invocation_invalid_context( - user, - project, - args, - result_type, - result + user, + project, + args, + result_type, + result ) context = [ From 0f37c9811edc11b5c13424ff775513ceedb5b4d4 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 3 Jul 2018 15:54:03 -0400 Subject: [PATCH 2/5] remove result, as well --- dbt/main.py | 15 ++++++--------- dbt/tracking.py | 24 ++++++++++-------------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/dbt/main.py b/dbt/main.py index cb7d6add016..3c6e0c56ebf 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -147,17 +147,17 @@ def run_from_task(task, proj, parsed_args): try: result = task.run() dbt.tracking.track_invocation_end( - project=proj, args=parsed_args, result_type="ok", result=None + project=proj, args=parsed_args, result_type="ok" ) except (dbt.exceptions.NotImplementedException, dbt.exceptions.FailedToConnectException) as e: logger.info('ERROR: {}'.format(e)) dbt.tracking.track_invocation_end( - project=proj, args=parsed_args, result_type="error", result=str(e) + project=proj, args=parsed_args, result_type="error" ) except Exception as e: dbt.tracking.track_invocation_end( - project=proj, args=parsed_args, result_type="error", result=str(e) + project=proj, args=parsed_args, result_type="error" ) raise @@ -196,8 +196,7 @@ def invoke_dbt(parsed): dbt.tracking.track_invalid_invocation( project=proj, args=parsed, - result_type="invalid_profile", - result=str(e)) + result_type="invalid_profile") return None except project.DbtProfileError as e: @@ -207,8 +206,7 @@ def invoke_dbt(parsed): dbt.tracking.track_invalid_invocation( project=proj, args=parsed, - result_type="invalid_profile", - result=str(e)) + result_type="invalid_profile") return None @@ -228,8 +226,7 @@ def invoke_dbt(parsed): dbt.tracking.track_invalid_invocation( project=proj, args=parsed, - result_type="invalid_target", - result="target not found") + result_type="invalid_target") return None diff --git a/dbt/tracking.py b/dbt/tracking.py index f9aa734ccdc..dc6ea183dd7 100644 --- a/dbt/tracking.py +++ b/dbt/tracking.py @@ -20,7 +20,7 @@ COOKIE_PATH = os.path.join(os.path.expanduser('~'), '.dbt/.user.yml') -INVOCATION_SPEC = 'iglu:com.dbt/invocation/jsonschema/1-0-0' +INVOCATION_SPEC = 'iglu:com.dbt/invocation/jsonschema/2-0-0' PLATFORM_SPEC = 'iglu:com.dbt/platform/jsonschema/1-0-0' RUN_MODEL_SPEC = 'iglu:com.dbt/run_model/jsonschema/2-0-0' INVOCATION_ENV_SPEC = 'iglu:com.dbt/invocation_env/jsonschema/1-0-0' @@ -104,34 +104,31 @@ def get_invocation_start_context(user, project, args): start_data = { "progress": "start", - "result_type": None, - "result": None + "result_type": None } data.update(start_data) return SelfDescribingJson(INVOCATION_SPEC, data) -def get_invocation_end_context(user, project, args, result_type, result): +def get_invocation_end_context(user, project, args, result_type): data = get_invocation_context(user, project, args) start_data = { "progress": "end", - "result_type": result_type, - "result": result, + "result_type": result_type } data.update(start_data) return SelfDescribingJson(INVOCATION_SPEC, data) -def get_invocation_invalid_context(user, project, args, result_type, result): +def get_invocation_invalid_context(user, project, args, result_type): data = get_invocation_context(user, project, args) start_data = { "progress": "invalid", - "result_type": result_type, - "result": result, + "result_type": result_type } data.update(start_data) @@ -216,11 +213,11 @@ def track_package_install(options): def track_invocation_end( - project=None, args=None, result_type=None, result=None + project=None, args=None, result_type=None ): user = active_user context = [ - get_invocation_end_context(user, project, args, result_type, result), + get_invocation_end_context(user, project, args, result_type), get_platform_context(), get_dbt_env_context() ] @@ -234,7 +231,7 @@ def track_invocation_end( def track_invalid_invocation( - project=None, args=None, result_type=None, result=None + project=None, args=None, result_type=None ): user = active_user @@ -242,8 +239,7 @@ def track_invalid_invocation( user, project, args, - result_type, - result + result_type ) context = [ From 5b2ba18bfea9856a8d623eb183afed5bb5fa2c65 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 5 Jul 2018 14:06:12 -0400 Subject: [PATCH 3/5] revert to 1.0.0 schemas, null out fields instead --- dbt/node_runners.py | 1 + dbt/tracking.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 35870a7d51d..b4006401ef7 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -32,6 +32,7 @@ def track_model_run(index, num_nodes, run_model_result): "execution_time": run_model_result.execution_time, "run_status": run_model_result.status, "run_skipped": run_model_result.skip, + "run_error": None, "model_materialization": dbt.utils.get_materialization(run_model_result.node), # noqa "model_id": dbt.utils.get_hash(run_model_result.node), "hashed_contents": dbt.utils.get_hashed_contents(run_model_result.node), # noqa diff --git a/dbt/tracking.py b/dbt/tracking.py index dc6ea183dd7..6b764f1c89a 100644 --- a/dbt/tracking.py +++ b/dbt/tracking.py @@ -20,9 +20,9 @@ COOKIE_PATH = os.path.join(os.path.expanduser('~'), '.dbt/.user.yml') -INVOCATION_SPEC = 'iglu:com.dbt/invocation/jsonschema/2-0-0' +INVOCATION_SPEC = 'iglu:com.dbt/invocation/jsonschema/1-0-0' PLATFORM_SPEC = 'iglu:com.dbt/platform/jsonschema/1-0-0' -RUN_MODEL_SPEC = 'iglu:com.dbt/run_model/jsonschema/2-0-0' +RUN_MODEL_SPEC = 'iglu:com.dbt/run_model/jsonschema/1-0-0' INVOCATION_ENV_SPEC = 'iglu:com.dbt/invocation_env/jsonschema/1-0-0' PACKAGE_INSTALL_SPEC = 'iglu:com.dbt/package_install/jsonschema/1-0-0' @@ -104,7 +104,8 @@ def get_invocation_start_context(user, project, args): start_data = { "progress": "start", - "result_type": None + "result_type": None, + "result": None } data.update(start_data) @@ -116,7 +117,8 @@ def get_invocation_end_context(user, project, args, result_type): start_data = { "progress": "end", - "result_type": result_type + "result_type": result_type, + "result": None } data.update(start_data) @@ -128,7 +130,8 @@ def get_invocation_invalid_context(user, project, args, result_type): start_data = { "progress": "invalid", - "result_type": result_type + "result_type": result_type, + "result": None } data.update(start_data) From 285f1da8476a0eb58e0ea6aa5cb84ff4c9425fe3 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 6 Jul 2018 09:25:22 -0400 Subject: [PATCH 4/5] fix invocations by stringifying version --- dbt/tracking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/tracking.py b/dbt/tracking.py index 6b764f1c89a..de6bcd5e98c 100644 --- a/dbt/tracking.py +++ b/dbt/tracking.py @@ -93,7 +93,7 @@ def get_invocation_context(user, project, args): "command": args.which, "options": None, - "version": dbt_version.installed, + "version": str(dbt_version.installed), "run_type": get_run_type(args), } From 2ff8a25192a4be678fc6fa66e8f623215a806401 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 6 Jul 2018 09:50:33 -0400 Subject: [PATCH 5/5] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 178846b1275..524a3ec7a0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Write JSON manifest file to disk during compilation ([#761](https://github.com/fishtown-analytics/dbt/pull/761)) - Add forward and backward graph edges to the JSON manifest file ([#762](https://github.com/fishtown-analytics/dbt/pull/762)) - Add a 'dbt docs generate' command to generate a JSON catalog file ([#774](https://github.com/fishtown-analytics/dbt/pull/774)) +- Stop tracking `run_error` in tracking code ([#817](https://github.com/fishtown-analytics/dbt/pull/817)) ## dbt 0.10.1 (May 18, 2018)