From d1d0d4ff8656723b259c5e9a83e833bb8290aafb Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 3 Aug 2023 09:30:55 -0500 Subject: [PATCH 1/8] add show test for json data --- core/dbt/clients/agate_helper.py | 12 ++++++++++++ tests/functional/show/fixtures.py | 14 +++++++++++--- tests/functional/show/test_show.py | 15 +++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/core/dbt/clients/agate_helper.py b/core/dbt/clients/agate_helper.py index 27f69aa9cae..a4ff48b602e 100644 --- a/core/dbt/clients/agate_helper.py +++ b/core/dbt/clients/agate_helper.py @@ -19,9 +19,21 @@ class Number(agate.data_types.Number): def cast(self, d): if type(d) == bool: raise agate.exceptions.CastError("Do not cast True to 1 or False to 0.") + # preserve integers as native Python int + elif type(d) == int: + return d else: return super().cast(d) + def jsonify(self, d): + if d is None: + return d + # do not cast integers to floats + elif type(d) == int: + return d + + return float(d) + class ISODateTime(agate.data_types.DateTime): def cast(self, d): diff --git a/tests/functional/show/fixtures.py b/tests/functional/show/fixtures.py index 85bfcd26c29..5159397ff34 100644 --- a/tests/functional/show/fixtures.py +++ b/tests/functional/show/fixtures.py @@ -2,6 +2,14 @@ select * from {{ ref('sample_seed') }} """ +models__sample_number_model = """ +select + cast(1.0 as int) as float_to_int_field, + 3.0 as float_field, + 4.3 as float_with_dec_field, + 5 as int_field +""" + models__second_model = """ select sample_num as col_one, @@ -75,10 +83,10 @@ seeds__sample_seed = """sample_num,sample_bool 1,true -2,false +2.0,false 3,true -4,false +4.3,false 5,true -6,false +6.5,false 7,true """ diff --git a/tests/functional/show/test_show.py b/tests/functional/show/test_show.py index 4429050191c..a8db41d26a0 100644 --- a/tests/functional/show/test_show.py +++ b/tests/functional/show/test_show.py @@ -6,6 +6,7 @@ models__second_ephemeral_model, seeds__sample_seed, models__sample_model, + models__sample_number_model, models__second_model, models__ephemeral_model, schema_yml, @@ -19,6 +20,7 @@ class TestShow: def models(self): return { "sample_model.sql": models__sample_model, + "sample_number_model.sql": models__sample_number_model, "second_model.sql": models__second_model, "ephemeral_model.sql": models__ephemeral_model, "sql_header.sql": models__sql_header, @@ -62,6 +64,19 @@ def test_select_single_model_json(self, project): assert "sample_num" in log_output assert "sample_bool" in log_output + def test_numeric_values(self, project): + run_dbt(["build"]) + (results, log_output) = run_dbt_and_capture( + ["show", "--select", "sample_number_model", "--output", "json"] + ) + assert "Previewing node 'sample_number_model'" not in log_output + assert "1.0" not in log_output + assert "1" in log_output + assert "3.0" in log_output + assert "4.3" in log_output + assert "5" in log_output + assert "5.0" not in log_output + def test_inline_pass(self, project): run_dbt(["build"]) (results, log_output) = run_dbt_and_capture( From 01192021ed30331e7bbc0b4b9f7375e659ffb672 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 3 Aug 2023 09:35:12 -0500 Subject: [PATCH 2/8] oh changie my changie --- .changes/unreleased/Fixes-20230803-093502.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20230803-093502.yaml diff --git a/.changes/unreleased/Fixes-20230803-093502.yaml b/.changes/unreleased/Fixes-20230803-093502.yaml new file mode 100644 index 00000000000..358d538d8f0 --- /dev/null +++ b/.changes/unreleased/Fixes-20230803-093502.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: update agage.Number to handle ints +time: 2023-08-03T09:35:02.163968-05:00 +custom: + Author: dave-connors-3 + Issue: "8153" From d8581d71301bcf15f6e07f4407425e905e0d74a1 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 3 Aug 2023 10:01:18 -0500 Subject: [PATCH 3/8] revert unecessary cahnge to fixture --- tests/functional/show/fixtures.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/show/fixtures.py b/tests/functional/show/fixtures.py index 5159397ff34..578c6b5f096 100644 --- a/tests/functional/show/fixtures.py +++ b/tests/functional/show/fixtures.py @@ -83,10 +83,10 @@ seeds__sample_seed = """sample_num,sample_bool 1,true -2.0,false +2,false 3,true -4.3,false +4,false 5,true -6.5,false +6,false 7,true """ From 6890757b68b09b2507ebd34716b43f47806a852e Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 3 Aug 2023 10:20:15 -0500 Subject: [PATCH 4/8] keep decimal class for precision methods, but return __int__ value --- core/dbt/clients/agate_helper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/dbt/clients/agate_helper.py b/core/dbt/clients/agate_helper.py index a4ff48b602e..a7e5eff12e8 100644 --- a/core/dbt/clients/agate_helper.py +++ b/core/dbt/clients/agate_helper.py @@ -1,4 +1,5 @@ from codecs import BOM_UTF8 +from decimal import Decimal import agate import datetime @@ -21,7 +22,7 @@ def cast(self, d): raise agate.exceptions.CastError("Do not cast True to 1 or False to 0.") # preserve integers as native Python int elif type(d) == int: - return d + return Decimal(d).__int__() else: return super().cast(d) @@ -30,7 +31,7 @@ def jsonify(self, d): return d # do not cast integers to floats elif type(d) == int: - return d + return Decimal(d).__int__() return float(d) From 628024a2fd86d9112cb9bb95a280014c35398280 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 17 Aug 2023 15:20:27 -0500 Subject: [PATCH 5/8] jerco updates --- core/dbt/clients/agate_helper.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/core/dbt/clients/agate_helper.py b/core/dbt/clients/agate_helper.py index a7e5eff12e8..273d9c51923 100644 --- a/core/dbt/clients/agate_helper.py +++ b/core/dbt/clients/agate_helper.py @@ -1,5 +1,4 @@ from codecs import BOM_UTF8 -from decimal import Decimal import agate import datetime @@ -10,31 +9,29 @@ from dbt.exceptions import DbtRuntimeError - BOM = BOM_UTF8.decode("utf-8") # '\ufeff' +class Integer(agate.data_types.DataType): + def cast(self, d): + if type(d) == int: + return d + else: + raise agate.exceptions.CastError('Can not parse value "%s" as Integer.' % d) + + def jsonify(self, d): + return d + + class Number(agate.data_types.Number): # undo the change in https://github.com/wireservice/agate/pull/733 # i.e. do not cast True and False to numeric 1 and 0 def cast(self, d): if type(d) == bool: raise agate.exceptions.CastError("Do not cast True to 1 or False to 0.") - # preserve integers as native Python int - elif type(d) == int: - return Decimal(d).__int__() else: return super().cast(d) - def jsonify(self, d): - if d is None: - return d - # do not cast integers to floats - elif type(d) == int: - return Decimal(d).__int__() - - return float(d) - class ISODateTime(agate.data_types.DateTime): def cast(self, d): @@ -61,6 +58,7 @@ def build_type_tester( ) -> agate.TypeTester: types = [ + Integer(null_values=("null", "")), Number(null_values=("null", "")), agate.data_types.Date(null_values=("null", ""), date_format="%Y-%m-%d"), agate.data_types.DateTime(null_values=("null", ""), datetime_format="%Y-%m-%d %H:%M:%S"), From c24932235e0c70b847949e026a1c7bf22e7771ac Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 17 Aug 2023 15:52:34 -0500 Subject: [PATCH 6/8] update integer type --- tests/unit/test_agate_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_agate_helper.py b/tests/unit/test_agate_helper.py index a2d9fcdb0e2..6455461f2eb 100644 --- a/tests/unit/test_agate_helper.py +++ b/tests/unit/test_agate_helper.py @@ -191,7 +191,7 @@ def test_nocast_bool_01(self): self.assertEqual(len(tbl), len(result_set)) assert isinstance(tbl.column_types[0], agate.data_types.Boolean) - assert isinstance(tbl.column_types[1], agate.data_types.Number) + assert isinstance(tbl.column_types[1], agate_helper.Integer) expected = [ [True, Decimal(1)], From 5e3e0f6b783badc1a7af81dd28467f8c6512e266 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Fri, 18 Aug 2023 08:52:57 -0500 Subject: [PATCH 7/8] update other tests --- tests/unit/test_agate_helper.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_agate_helper.py b/tests/unit/test_agate_helper.py index 6455461f2eb..87658633158 100644 --- a/tests/unit/test_agate_helper.py +++ b/tests/unit/test_agate_helper.py @@ -121,37 +121,37 @@ def test_datetime_formats(self): self.assertEqual(tbl[0][0], expected) def test_merge_allnull(self): - t1 = agate.Table([(1, "a", None), (2, "b", None)], ("a", "b", "c")) - t2 = agate.Table([(3, "c", None), (4, "d", None)], ("a", "b", "c")) + t1 = agate_helper.table_from_rows([(1, "a", None), (2, "b", None)], ("a", "b", "c")) + t2 = agate_helper.table_from_rows([(3, "c", None), (4, "d", None)], ("a", "b", "c")) result = agate_helper.merge_tables([t1, t2]) self.assertEqual(result.column_names, ("a", "b", "c")) - assert isinstance(result.column_types[0], agate.data_types.Number) + assert isinstance(result.column_types[0], agate_helper.Integer) assert isinstance(result.column_types[1], agate.data_types.Text) assert isinstance(result.column_types[2], agate.data_types.Number) self.assertEqual(len(result), 4) def test_merge_mixed(self): - t1 = agate.Table([(1, "a", None), (2, "b", None)], ("a", "b", "c")) - t2 = agate.Table([(3, "c", "dog"), (4, "d", "cat")], ("a", "b", "c")) - t3 = agate.Table([(3, "c", None), (4, "d", None)], ("a", "b", "c")) + t1 = agate_helper.table_from_rows([(1, "a", None), (2, "b", None)], ("a", "b", "c")) + t2 = agate_helper.table_from_rows([(3, "c", "dog"), (4, "d", "cat")], ("a", "b", "c")) + t3 = agate_helper.table_from_rows([(3, "c", None), (4, "d", None)], ("a", "b", "c")) result = agate_helper.merge_tables([t1, t2]) self.assertEqual(result.column_names, ("a", "b", "c")) - assert isinstance(result.column_types[0], agate.data_types.Number) + assert isinstance(result.column_types[0], agate_helper.Integer) assert isinstance(result.column_types[1], agate.data_types.Text) assert isinstance(result.column_types[2], agate.data_types.Text) self.assertEqual(len(result), 4) result = agate_helper.merge_tables([t2, t3]) self.assertEqual(result.column_names, ("a", "b", "c")) - assert isinstance(result.column_types[0], agate.data_types.Number) + assert isinstance(result.column_types[0], agate_helper.Integer) assert isinstance(result.column_types[1], agate.data_types.Text) assert isinstance(result.column_types[2], agate.data_types.Text) self.assertEqual(len(result), 4) result = agate_helper.merge_tables([t1, t2, t3]) self.assertEqual(result.column_names, ("a", "b", "c")) - assert isinstance(result.column_types[0], agate.data_types.Number) + assert isinstance(result.column_types[0], agate_helper.Integer) assert isinstance(result.column_types[1], agate.data_types.Text) assert isinstance(result.column_types[2], agate.data_types.Text) self.assertEqual(len(result), 6) From 16446288e10cfc1a3c6e21746a42edfd35fc05d1 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 18 Aug 2023 10:34:54 -0500 Subject: [PATCH 8/8] Update .changes/unreleased/Fixes-20230803-093502.yaml --- .changes/unreleased/Fixes-20230803-093502.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Fixes-20230803-093502.yaml b/.changes/unreleased/Fixes-20230803-093502.yaml index 358d538d8f0..ffa0b3aba1c 100644 --- a/.changes/unreleased/Fixes-20230803-093502.yaml +++ b/.changes/unreleased/Fixes-20230803-093502.yaml @@ -1,5 +1,5 @@ kind: Fixes -body: update agage.Number to handle ints +body: Add explicit support for integers for the show command time: 2023-08-03T09:35:02.163968-05:00 custom: Author: dave-connors-3