From e09f8bf97e07cd99ae4bbd8597e980a63f19dfa0 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 8 May 2018 16:35:00 -0600 Subject: [PATCH 01/43] First pass, just dump generated catalog data to stdout --- dbt/adapters/default/impl.py | 11 ++++++++ dbt/adapters/postgres/impl.py | 53 +++++++++++++++++++++++++++++++++++ dbt/main.py | 11 ++++++++ dbt/task/generate.py | 34 ++++++++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 dbt/task/generate.py diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index a804e8e04d6..61bf98394ea 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -270,6 +270,17 @@ def get_columns_in_table(cls, profile, project_cfg, schema_name, def _table_columns_to_dict(cls, columns): return {col.name: col for col in columns} + @classmethod + def get_catalog_for_schemas(cls, profile, schemas=None): + """Get the catalog information for a given profile and list of + schemas. If no schemas are given, return all schemas. + Returns a list of dictionaries, each one representing a single column. + + TODO: document the minimum guaranteed information callers can expect + (what keys are in each dict, basically) + """ + raise NotImplementedError() + @classmethod def expand_target_column_types(cls, profile, project_cfg, temp_table, diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index 17797dc9fe3..19461d3f129 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -177,6 +177,59 @@ def check_schema_exists(cls, profile, project, schema, model_name=None): return results[0] > 0 + @classmethod + def get_catalog_for_schemas(cls, profile, schemas=None): + """Get the catalog information for a given profile and list of + schemas. If no schemas are given, return all schemas. + Returns a list of dictionaries, each one representing a single column. + + TODO: document the minimum guaranteed information callers can expect + (what keys are in each dict, basically) + """ + # from https://github.com/fishtown-analytics/dbt/issues/763#issuecomment-387490363 + sql = """ + with tables as ( + select + table_schema, + table_name, + table_type + + from information_schema.tables + + ), + + columns as ( + + select + table_schema, + table_name, + null as table_comment, + + column_name, + ordinal_position as column_index, + data_type as column_type, + null as column_comment + + + from information_schema.columns + + ) + + select * + from tables + join columns using (table_schema, table_name) + + where table_schema != 'information_schema' + and table_schema not like 'pg_%' + """.strip() + # TODO: make sure I can just ignore status. + _, results = cls.execute(profile, sql, fetch=True) + # ok, have an agate.Table now consisting of my results. Filter out + # schemas if necessary (this should probably happen in SQL, right?). + if schemas is not None: + results = results.where(lambda r: r['table_schema'] in schemas) + return [dict(zip(results.column_names, row)) for row in results] + @classmethod def convert_text_type(cls, agate_table, col_idx): return "text" diff --git a/dbt/main.py b/dbt/main.py index 29388240bab..c3d1406e355 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -17,6 +17,7 @@ import dbt.task.seed as seed_task import dbt.task.test as test_task import dbt.task.archive as archive_task +import dbt.task.generate as generate_task import dbt.tracking import dbt.config as config @@ -354,6 +355,7 @@ def parse_args(args): compile_sub = subs.add_parser('compile', parents=[base_subparser]) compile_sub.set_defaults(cls=compile_task.CompileTask, which='compile') + for sub in [run_sub, compile_sub]: sub.add_argument( '--models', @@ -414,6 +416,15 @@ def parse_args(args): ) seed_sub.set_defaults(cls=seed_task.SeedTask, which='seed') + catalog_sub = subs.add_parser('catalog', parents=[base_subparser]) + catalog_subs = catalog_sub.add_subparsers() + # it might look like catalog_sub is the correct parents entry, but that + # will cause weird errors about 'conflicting option strings'. + generate_sub = catalog_subs.add_parser('generate', + parents=[base_subparser]) + generate_sub.set_defaults(cls=generate_task.GenerateTask, + which='generate') + sub = subs.add_parser('test', parents=[base_subparser]) sub.add_argument( '--data', diff --git a/dbt/task/generate.py b/dbt/task/generate.py new file mode 100644 index 00000000000..072981ebb59 --- /dev/null +++ b/dbt/task/generate.py @@ -0,0 +1,34 @@ + +from pprint import pformat + +from dbt.logger import GLOBAL_LOGGER as logger +from dbt.adapters.factory import get_adapter +import dbt.ui.printer + +from dbt.task.base_task import BaseTask + + +# derive from BaseTask as I don't really want any result interpretation. +class GenerateTask(BaseTask): + def run(self): + profile = self.project.run_environment() + adapter = get_adapter(profile) + + # To get a list of schemas, it looks like we'd need to have the + # compiled project and use node_runners.BaseRunner.get_model_schemas. + # But I think we don't really want to compile, here, right? Or maybe + # we do and I need to add all of that? But then we probably need to + # go through the whole BaseRunner.safe_run path which makes things + # more complex - need to figure out how to handle all the + # is_ephemeral_model stuff, etc. + # TODO: talk to connor/drew about this question. + try: + results = adapter.get_catalog_for_schemas(profile, schemas=None) + finally: + adapter.cleanup_connections() + + # dump results to stdout for the moment. + dbt.ui.printer.print_timestamped_line('results={}'.format(pformat(results))) + dbt.ui.printer.print_timestamped_line('Done.') + + return results From 89ad5d6de1ba06697a59c971d83cd04f3a8f100e Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 9 May 2018 11:04:21 -0600 Subject: [PATCH 02/43] Actually write structured data, write it to disk --- dbt/adapters/default/impl.py | 3 +- dbt/compat.py | 2 + dbt/task/generate.py | 102 ++++++++++++++++++++++++++++++++--- 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 61bf98394ea..80e5f7bc01b 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -276,8 +276,7 @@ def get_catalog_for_schemas(cls, profile, schemas=None): schemas. If no schemas are given, return all schemas. Returns a list of dictionaries, each one representing a single column. - TODO: document the minimum guaranteed information callers can expect - (what keys are in each dict, basically) + Required keys in the output: table_schema, table_name, column_index """ raise NotImplementedError() diff --git a/dbt/compat.py b/dbt/compat.py index 26cba2577d3..ee313b5ec79 100644 --- a/dbt/compat.py +++ b/dbt/compat.py @@ -10,8 +10,10 @@ if WHICH_PYTHON == 2: basestring = basestring + bigint = long else: basestring = str + bigint = int def to_unicode(s): diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 072981ebb59..9f43cb5a9cc 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -1,13 +1,94 @@ - -from pprint import pformat +import json +import os from dbt.logger import GLOBAL_LOGGER as logger from dbt.adapters.factory import get_adapter +from dbt.clients.system import write_file +from dbt.compat import bigint import dbt.ui.printer from dbt.task.base_task import BaseTask +def get_stripped_prefix(source, prefix): + """Go through source, extracting every key/value pair where the key starts + with the given prefix. + """ + cut = len(prefix) + return { + k[cut:]: v for k, v in source.items() + if k.startswith(prefix) + } + + +def unflatten(columns): + """Given a list of column dictionaries following this layout: + + [{ + 'column_comment': None, + 'column_index': Decimal('1'), + 'column_name': 'id', + 'column_type': 'integer', + 'table_comment': None, + 'table_name': 'test_table, + 'table_schema': 'test_schema', + 'table_type': 'BASE TABLE' + }] + + unflatten will convert them into a dict with this nested structure: + + { + 'test_schema': { + 'test_table': { + 'metadata': { + 'comment': None, + 'name': 'table', + 'type': 'BASE_TABLE' + } + 'columns': [ + { + 'type': 'integer', + 'comment': None, + 'index': 1, + } + ] + } + } + } + + TODO: Verify that we want 'name' but not 'schema' inside the metadata + (seems to me we would want none or both since both are represented?) + + Note: the docstring for DefaultAdapter.get_catalog_for_schemas discusses + what keys are guaranteed to exist. This method makes use of those keys. + + Keys prefixed with 'column_' end up in per-column data and keys prefixed + with 'table_' end up in table metadata. Keys without either prefix are + ignored. + """ + structured = {} + for entry in columns: + schema_name = entry['table_schema'] + table_name = entry['table_name'] + + if schema_name not in structured: + structured[schema_name] = {} + schema = structured[schema_name] + + if table_name not in schema: + metadata = get_stripped_prefix(entry, 'table_') + metadata.pop('schema') + schema[table_name] = {'metadata': metadata, 'columns': []} + table = schema[table_name] + + column = get_stripped_prefix(entry, 'column_') + # TODO: is this ok? Is it safe to assume that index is always + # representable with long/int? + column['index'] = bigint(column['index']) + table['columns'].append(column) + return structured + + # derive from BaseTask as I don't really want any result interpretation. class GenerateTask(BaseTask): def run(self): @@ -23,12 +104,21 @@ def run(self): # is_ephemeral_model stuff, etc. # TODO: talk to connor/drew about this question. try: - results = adapter.get_catalog_for_schemas(profile, schemas=None) + columns = adapter.get_catalog_for_schemas(profile, schemas=None) + # TODO: should we uncomment this line to shut up a cranky message + # when run with debug? Or make a special name that we pass through + # from get_catalog_for_schemas and then pass here? + # adapter.release_connection(profile, name='master') finally: adapter.cleanup_connections() - # dump results to stdout for the moment. - dbt.ui.printer.print_timestamped_line('results={}'.format(pformat(results))) - dbt.ui.printer.print_timestamped_line('Done.') + results = unflatten(columns) + + path = os.path.join(self.project['target-path'], 'catalog.json') + write_file(path, json.dumps(results)) + + dbt.ui.printer.print_timestamped_line( + 'Catalog written to {}'.format(os.path.abspath(path)) + ) return results From 2ae45ab1ee06570c4f920aa676115d90db0a86a6 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 9 May 2018 11:21:04 -0600 Subject: [PATCH 03/43] Fix release_connection issues in dbt catalog generate --- dbt/adapters/default/impl.py | 9 ++------- dbt/task/generate.py | 13 ++++--------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 80e5f7bc01b..480cb44a897 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -390,14 +390,9 @@ def get_default_schema(cls, profile, project_cfg): return profile.get('schema') @classmethod - def get_connection(cls, profile, name=None, recache_if_missing=True): + def get_connection(cls, profile, name='master', recache_if_missing=True): global connections_in_use - if name is None: - # if a name isn't specified, we'll re-use a single handle - # named 'master' - name = 'master' - if connections_in_use.get(name): return connections_in_use.get(name) @@ -481,7 +476,7 @@ def acquire_connection(cls, profile, name): lock.release() @classmethod - def release_connection(cls, profile, name): + def release_connection(cls, profile, name='master'): global connections_in_use, connections_available, lock if connections_in_use.get(name) is None: diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 9f43cb5a9cc..544b081838b 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -44,6 +44,7 @@ def unflatten(columns): 'comment': None, 'name': 'table', 'type': 'BASE_TABLE' + 'schema': 'test_schema', } 'columns': [ { @@ -56,9 +57,6 @@ def unflatten(columns): } } - TODO: Verify that we want 'name' but not 'schema' inside the metadata - (seems to me we would want none or both since both are represented?) - Note: the docstring for DefaultAdapter.get_catalog_for_schemas discusses what keys are guaranteed to exist. This method makes use of those keys. @@ -82,8 +80,8 @@ def unflatten(columns): table = schema[table_name] column = get_stripped_prefix(entry, 'column_') - # TODO: is this ok? Is it safe to assume that index is always - # representable with long/int? + # the index should really never be that big so it's ok to end up + # serializing this to JSON (2^53 is the max safe value there) column['index'] = bigint(column['index']) table['columns'].append(column) return structured @@ -105,10 +103,7 @@ def run(self): # TODO: talk to connor/drew about this question. try: columns = adapter.get_catalog_for_schemas(profile, schemas=None) - # TODO: should we uncomment this line to shut up a cranky message - # when run with debug? Or make a special name that we pass through - # from get_catalog_for_schemas and then pass here? - # adapter.release_connection(profile, name='master') + adapter.release_connection(profile) finally: adapter.cleanup_connections() From 8cb30919c5f6048f0b05ef44098ff913dc161d1e Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 9 May 2018 11:35:32 -0600 Subject: [PATCH 04/43] clean up TODOs --- dbt/adapters/postgres/impl.py | 10 ++-------- dbt/main.py | 1 - dbt/task/generate.py | 10 ++-------- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index 19461d3f129..031c1a23fc3 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -179,14 +179,8 @@ def check_schema_exists(cls, profile, project, schema, model_name=None): @classmethod def get_catalog_for_schemas(cls, profile, schemas=None): - """Get the catalog information for a given profile and list of - schemas. If no schemas are given, return all schemas. - Returns a list of dictionaries, each one representing a single column. - - TODO: document the minimum guaranteed information callers can expect - (what keys are in each dict, basically) - """ - # from https://github.com/fishtown-analytics/dbt/issues/763#issuecomment-387490363 + # TODO: we want to move this into separate files that users can modify + # so they can generate their own. sql = """ with tables as ( select diff --git a/dbt/main.py b/dbt/main.py index c3d1406e355..2ce78d59383 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -355,7 +355,6 @@ def parse_args(args): compile_sub = subs.add_parser('compile', parents=[base_subparser]) compile_sub.set_defaults(cls=compile_task.CompileTask, which='compile') - for sub in [run_sub, compile_sub]: sub.add_argument( '--models', diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 544b081838b..62efc9f9e2a 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -93,14 +93,8 @@ def run(self): profile = self.project.run_environment() adapter = get_adapter(profile) - # To get a list of schemas, it looks like we'd need to have the - # compiled project and use node_runners.BaseRunner.get_model_schemas. - # But I think we don't really want to compile, here, right? Or maybe - # we do and I need to add all of that? But then we probably need to - # go through the whole BaseRunner.safe_run path which makes things - # more complex - need to figure out how to handle all the - # is_ephemeral_model stuff, etc. - # TODO: talk to connor/drew about this question. + # To get a list of schemas, we'd need to generate the parsed manifest + # for now just pass None. try: columns = adapter.get_catalog_for_schemas(profile, schemas=None) adapter.release_connection(profile) From da107f332a254d52e7189f8ddd47c9ab1d6cce9b Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 10 May 2018 08:04:59 -0600 Subject: [PATCH 05/43] Fix bugs, add unit tests --- dbt/task/generate.py | 36 +++--- test/unit/test_catalog_generate.py | 193 +++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 18 deletions(-) create mode 100644 test/unit/test_catalog_generate.py diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 62efc9f9e2a..0caceab8fd6 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -30,32 +30,33 @@ def unflatten(columns): 'column_name': 'id', 'column_type': 'integer', 'table_comment': None, - 'table_name': 'test_table, + 'table_name': 'test_table', 'table_schema': 'test_schema', 'table_type': 'BASE TABLE' }] unflatten will convert them into a dict with this nested structure: - { - 'test_schema': { - 'test_table': { - 'metadata': { - 'comment': None, - 'name': 'table', - 'type': 'BASE_TABLE' - 'schema': 'test_schema', - } - 'columns': [ - { - 'type': 'integer', + { + 'test_schema': { + 'test_table': { + 'metadata': { 'comment': None, - 'index': 1, - } - ] + 'name': 'test_table', + 'type': 'BASE TABLE', + 'schema': 'test_schema', + }, + 'columns': [ + { + 'type': 'integer', + 'comment': None, + 'index': bigint(1), + 'name': 'id' + } + ] + } } } - } Note: the docstring for DefaultAdapter.get_catalog_for_schemas discusses what keys are guaranteed to exist. This method makes use of those keys. @@ -75,7 +76,6 @@ def unflatten(columns): if table_name not in schema: metadata = get_stripped_prefix(entry, 'table_') - metadata.pop('schema') schema[table_name] = {'metadata': metadata, 'columns': []} table = schema[table_name] diff --git a/test/unit/test_catalog_generate.py b/test/unit/test_catalog_generate.py new file mode 100644 index 00000000000..b183bd69397 --- /dev/null +++ b/test/unit/test_catalog_generate.py @@ -0,0 +1,193 @@ +from decimal import Decimal +import unittest +import os + +import dbt.flags +from dbt.compat import bigint +from dbt.task import generate + + +class GenerateTest(unittest.TestCase): + def setUp(self): + dbt.flags.STRICT_MODE = True + self.maxDiff = None + + def test__unflatten_empty(self): + columns = [] + expected = {} + result = generate.unflatten(columns) + self.assertEqual(result, expected) + + def test__unflatten_one_column(self): + columns = [{ + 'column_comment': None, + 'column_index': Decimal('1'), + 'column_name': 'id', + 'column_type': 'integer', + 'table_comment': None, + 'table_name': 'test_table', + 'table_schema': 'test_schema', + 'table_type': 'BASE TABLE' + }] + + expected = { + 'test_schema': { + 'test_table': { + 'metadata': { + 'comment': None, + 'name': 'test_table', + 'type': 'BASE TABLE', + 'schema': 'test_schema', + }, + 'columns': [ + { + 'type': 'integer', + 'comment': None, + 'index': bigint(1), + 'name': 'id' + } + ] + } + } + } + result = generate.unflatten(columns) + self.assertEqual(result, expected) + + def test__unflatten_multiple_schemas(self): + columns = [ + { + 'column_comment': None, + 'column_index': Decimal('1'), + 'column_name': 'id', + 'column_type': 'integer', + 'table_comment': None, + 'table_name': 'test_table', + 'table_schema': 'test_schema', + 'table_type': 'BASE TABLE' + }, + { + 'column_comment': None, + 'column_index': Decimal('2'), + 'column_name': 'name', + 'column_type': 'text', + 'table_comment': None, + 'table_name': 'test_table', + 'table_schema': 'test_schema', + 'table_type': 'BASE TABLE' + }, + { + 'column_comment': None, + 'column_index': Decimal('1'), + 'column_name': 'id', + 'column_type': 'integer', + 'table_comment': None, + 'table_name': 'other_test_table', + 'table_schema': 'test_schema', + 'table_type': 'BASE TABLE', + }, + { + 'column_comment': None, + 'column_index': Decimal('2'), + 'column_name': 'email', + 'column_type': 'character varying', + 'table_comment': None, + 'table_name': 'other_test_table', + 'table_schema': 'test_schema', + 'table_type': 'BASE TABLE', + }, + { + 'column_comment': None, + 'column_index': Decimal('1'), + 'column_name': 'id', + 'column_type': 'integer', + 'table_comment': None, + 'table_name': 'test_table', + 'table_schema': 'other_test_schema', + 'table_type': 'BASE TABLE' + }, + { + 'column_comment': None, + 'column_index': Decimal('2'), + 'column_name': 'name', + 'column_type': 'text', + 'table_comment': None, + 'table_name': 'test_table', + 'table_schema': 'other_test_schema', + 'table_type': 'BASE TABLE' + }, + ] + + expected = { + 'test_schema': { + 'test_table': { + 'metadata': { + 'comment': None, + 'name': 'test_table', + 'type': 'BASE TABLE', + 'schema': 'test_schema', + }, + 'columns': [ + { + 'type': 'integer', + 'comment': None, + 'index': bigint(1), + 'name': 'id' + }, + { + 'type': 'text', + 'comment': None, + 'index': Decimal('2'), + 'name': 'name', + } + ], + }, + 'other_test_table': { + 'metadata': { + 'comment': None, + 'name': 'other_test_table', + 'type': 'BASE TABLE', + 'schema': 'test_schema', + }, + 'columns': [ + { + 'type': 'integer', + 'comment': None, + 'index': bigint(1), + 'name': 'id' + }, + { + 'type': 'character varying', + 'comment': None, + 'index': Decimal('2'), + 'name': 'email', + } + ] + } + }, + 'other_test_schema': { + 'test_table': { + 'metadata': { + 'comment': None, + 'name': 'test_table', + 'type': 'BASE TABLE', + 'schema': 'other_test_schema', + }, + 'columns': [ + { + 'type': 'integer', + 'comment': None, + 'index': bigint(1), + 'name': 'id' + }, + { + 'type': 'text', + 'comment': None, + 'index': Decimal('2'), + 'name': 'name', + } + ], + }, + } + } + result = generate.unflatten(columns) + self.assertEqual(result, expected) From b29fe7feda5447d00f48f058f22e032378f9d1bb Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 10 May 2018 08:14:30 -0600 Subject: [PATCH 06/43] ok, I guess accepting None is pretty important here --- dbt/adapters/default/impl.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 480cb44a897..f97e96be419 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -390,9 +390,14 @@ def get_default_schema(cls, profile, project_cfg): return profile.get('schema') @classmethod - def get_connection(cls, profile, name='master', recache_if_missing=True): + def get_connection(cls, profile, name=None, recache_if_missing=True): global connections_in_use + if name is None: + # if a name isn't specified, we'll re-use a single handle + # named 'master' + name = 'master' + if connections_in_use.get(name): return connections_in_use.get(name) From 1e9565732cced0ee6aa6f748724de185b454a183 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 10 May 2018 10:08:44 -0600 Subject: [PATCH 07/43] Finally get a (bad) integration test going --- .../models/.gitkeep | 0 .../models/disabled_one.sql | 3 + .../models/disabled_two.sql | 3 + .../models/empty.sql | 3 + .../models/view_summary.sql | 18 + .../029_catalog_generate_tests/seed.sql | 587 ++++++++++++++++++ .../test_catalog_generate.py | 73 +++ 7 files changed, 687 insertions(+) create mode 100644 test/integration/029_catalog_generate_tests/models/.gitkeep create mode 100644 test/integration/029_catalog_generate_tests/models/disabled_one.sql create mode 100644 test/integration/029_catalog_generate_tests/models/disabled_two.sql create mode 100644 test/integration/029_catalog_generate_tests/models/empty.sql create mode 100644 test/integration/029_catalog_generate_tests/models/view_summary.sql create mode 100644 test/integration/029_catalog_generate_tests/seed.sql create mode 100644 test/integration/029_catalog_generate_tests/test_catalog_generate.py diff --git a/test/integration/029_catalog_generate_tests/models/.gitkeep b/test/integration/029_catalog_generate_tests/models/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/029_catalog_generate_tests/models/disabled_one.sql b/test/integration/029_catalog_generate_tests/models/disabled_one.sql new file mode 100644 index 00000000000..6f373105aac --- /dev/null +++ b/test/integration/029_catalog_generate_tests/models/disabled_one.sql @@ -0,0 +1,3 @@ +{{config(enabled=False)}} + +select 1 diff --git a/test/integration/029_catalog_generate_tests/models/disabled_two.sql b/test/integration/029_catalog_generate_tests/models/disabled_two.sql new file mode 100644 index 00000000000..b20d379edea --- /dev/null +++ b/test/integration/029_catalog_generate_tests/models/disabled_two.sql @@ -0,0 +1,3 @@ +{{config(enabled=False)}} + +select * from {{ref('disabled_one')}} diff --git a/test/integration/029_catalog_generate_tests/models/empty.sql b/test/integration/029_catalog_generate_tests/models/empty.sql new file mode 100644 index 00000000000..b28b04f6431 --- /dev/null +++ b/test/integration/029_catalog_generate_tests/models/empty.sql @@ -0,0 +1,3 @@ + + + diff --git a/test/integration/029_catalog_generate_tests/models/view_summary.sql b/test/integration/029_catalog_generate_tests/models/view_summary.sql new file mode 100644 index 00000000000..4ff38b12de7 --- /dev/null +++ b/test/integration/029_catalog_generate_tests/models/view_summary.sql @@ -0,0 +1,18 @@ + +{{ + config( + materialized='view' + ) +}} + + +with t as ( + + select * from {{ ref('view_model') }} + +) + +select date_trunc('year', updated_at) as year, + count(*) +from t +group by 1 diff --git a/test/integration/029_catalog_generate_tests/seed.sql b/test/integration/029_catalog_generate_tests/seed.sql new file mode 100644 index 00000000000..5bb6f00beaa --- /dev/null +++ b/test/integration/029_catalog_generate_tests/seed.sql @@ -0,0 +1,587 @@ +create table {schema}.seed ( + id INTEGER, + first_name VARCHAR(11), + email VARCHAR(31), + ip_address VARCHAR(15), + updated_at TIMESTAMP WITHOUT TIME ZONE +); + + +INSERT INTO {schema}.seed + ("id","first_name","email","ip_address","updated_at") +VALUES + (1,'Larry','lking0@miitbeian.gov.cn','69.135.206.194','2008-09-12 19:08:31'), + (2,'Larry','lperkins1@toplist.cz','64.210.133.162','1978-05-09 04:15:14'), + (3,'Anna','amontgomery2@miitbeian.gov.cn','168.104.64.114','2011-10-16 04:07:57'), + (4,'Sandra','sgeorge3@livejournal.com','229.235.252.98','1973-07-19 10:52:43'), + (5,'Fred','fwoods4@google.cn','78.229.170.124','2012-09-30 16:38:29'), + (6,'Stephen','shanson5@livejournal.com','182.227.157.105','1995-11-07 21:40:50'), + (7,'William','wmartinez6@upenn.edu','135.139.249.50','1982-09-05 03:11:59'), + (8,'Jessica','jlong7@hao123.com','203.62.178.210','1991-10-16 11:03:15'), + (9,'Douglas','dwhite8@tamu.edu','178.187.247.1','1979-10-01 09:49:48'), + (10,'Lisa','lcoleman9@nydailynews.com','168.234.128.249','2011-05-26 07:45:49'), + (11,'Ralph','rfieldsa@home.pl','55.152.163.149','1972-11-18 19:06:11'), + (12,'Louise','lnicholsb@samsung.com','141.116.153.154','2014-11-25 20:56:14'), + (13,'Clarence','cduncanc@sfgate.com','81.171.31.133','2011-11-17 07:02:36'), + (14,'Daniel','dfranklind@omniture.com','8.204.211.37','1980-09-13 00:09:04'), + (15,'Katherine','klanee@auda.org.au','176.96.134.59','1997-08-22 19:36:56'), + (16,'Billy','bwardf@wikia.com','214.108.78.85','2003-10-19 02:14:47'), + (17,'Annie','agarzag@ocn.ne.jp','190.108.42.70','1988-10-28 15:12:35'), + (18,'Shirley','scolemanh@fastcompany.com','109.251.164.84','1988-08-24 10:50:57'), + (19,'Roger','rfrazieri@scribd.com','38.145.218.108','1985-12-31 15:17:15'), + (20,'Lillian','lstanleyj@goodreads.com','47.57.236.17','1970-06-08 02:09:05'), + (21,'Aaron','arodriguezk@nps.gov','205.245.118.221','1985-10-11 23:07:49'), + (22,'Patrick','pparkerl@techcrunch.com','19.8.100.182','2006-03-29 12:53:56'), + (23,'Phillip','pmorenom@intel.com','41.38.254.103','2011-11-07 15:35:43'), + (24,'Henry','hgarcian@newsvine.com','1.191.216.252','2008-08-28 08:30:44'), + (25,'Irene','iturnero@opera.com','50.17.60.190','1994-04-01 07:15:02'), + (26,'Andrew','adunnp@pen.io','123.52.253.176','2000-11-01 06:03:25'), + (27,'David','dgutierrezq@wp.com','238.23.203.42','1988-01-25 07:29:18'), + (28,'Henry','hsanchezr@cyberchimps.com','248.102.2.185','1983-01-01 13:36:37'), + (29,'Evelyn','epetersons@gizmodo.com','32.80.46.119','1979-07-16 17:24:12'), + (30,'Tammy','tmitchellt@purevolume.com','249.246.167.88','2001-04-03 10:00:23'), + (31,'Jacqueline','jlittleu@domainmarket.com','127.181.97.47','1986-02-11 21:35:50'), + (32,'Earl','eortizv@opera.com','166.47.248.240','1996-07-06 08:16:27'), + (33,'Juan','jgordonw@sciencedirect.com','71.77.2.200','1987-01-31 03:46:44'), + (34,'Diane','dhowellx@nyu.edu','140.94.133.12','1994-06-11 02:30:05'), + (35,'Randy','rkennedyy@microsoft.com','73.255.34.196','2005-05-26 20:28:39'), + (36,'Janice','jriveraz@time.com','22.214.227.32','1990-02-09 04:16:52'), + (37,'Laura','lperry10@diigo.com','159.148.145.73','2015-03-17 05:59:25'), + (38,'Gary','gray11@statcounter.com','40.193.124.56','1970-01-27 10:04:51'), + (39,'Jesse','jmcdonald12@typepad.com','31.7.86.103','2009-03-14 08:14:29'), + (40,'Sandra','sgonzalez13@goodreads.com','223.80.168.239','1993-05-21 14:08:54'), + (41,'Scott','smoore14@archive.org','38.238.46.83','1980-08-30 11:16:56'), + (42,'Phillip','pevans15@cisco.com','158.234.59.34','2011-12-15 23:26:31'), + (43,'Steven','sriley16@google.ca','90.247.57.68','2011-10-29 19:03:28'), + (44,'Deborah','dbrown17@hexun.com','179.125.143.240','1995-04-10 14:36:07'), + (45,'Lori','lross18@ow.ly','64.80.162.180','1980-12-27 16:49:15'), + (46,'Sean','sjackson19@tumblr.com','240.116.183.69','1988-06-12 21:24:45'), + (47,'Terry','tbarnes1a@163.com','118.38.213.137','1997-09-22 16:43:19'), + (48,'Dorothy','dross1b@ebay.com','116.81.76.49','2005-02-28 13:33:24'), + (49,'Samuel','swashington1c@house.gov','38.191.253.40','1989-01-19 21:15:48'), + (50,'Ralph','rcarter1d@tinyurl.com','104.84.60.174','2007-08-11 10:21:49'), + (51,'Wayne','whudson1e@princeton.edu','90.61.24.102','1983-07-03 16:58:12'), + (52,'Rose','rjames1f@plala.or.jp','240.83.81.10','1995-06-08 11:46:23'), + (53,'Louise','lcox1g@theglobeandmail.com','105.11.82.145','2016-09-19 14:45:51'), + (54,'Kenneth','kjohnson1h@independent.co.uk','139.5.45.94','1976-08-17 11:26:19'), + (55,'Donna','dbrown1i@amazon.co.uk','19.45.169.45','2006-05-27 16:51:40'), + (56,'Johnny','jvasquez1j@trellian.com','118.202.238.23','1975-11-17 08:42:32'), + (57,'Patrick','pramirez1k@tamu.edu','231.25.153.198','1997-08-06 11:51:09'), + (58,'Helen','hlarson1l@prweb.com','8.40.21.39','1993-08-04 19:53:40'), + (59,'Patricia','pspencer1m@gmpg.org','212.198.40.15','1977-08-03 16:37:27'), + (60,'Joseph','jspencer1n@marriott.com','13.15.63.238','2005-07-23 20:22:06'), + (61,'Phillip','pschmidt1o@blogtalkradio.com','177.98.201.190','1976-05-19 21:47:44'), + (62,'Joan','jwebb1p@google.ru','105.229.170.71','1972-09-07 17:53:47'), + (63,'Phyllis','pkennedy1q@imgur.com','35.145.8.244','2000-01-01 22:33:37'), + (64,'Katherine','khunter1r@smh.com.au','248.168.205.32','1991-01-09 06:40:24'), + (65,'Laura','lvasquez1s@wiley.com','128.129.115.152','1997-10-23 12:04:56'), + (66,'Juan','jdunn1t@state.gov','44.228.124.51','2004-11-10 05:07:35'), + (67,'Judith','jholmes1u@wiley.com','40.227.179.115','1977-08-02 17:01:45'), + (68,'Beverly','bbaker1v@wufoo.com','208.34.84.59','2016-03-06 20:07:23'), + (69,'Lawrence','lcarr1w@flickr.com','59.158.212.223','1988-09-13 06:07:21'), + (70,'Gloria','gwilliams1x@mtv.com','245.231.88.33','1995-03-18 22:32:46'), + (71,'Steven','ssims1y@cbslocal.com','104.50.58.255','2001-08-05 21:26:20'), + (72,'Betty','bmills1z@arstechnica.com','103.177.214.220','1981-12-14 21:26:54'), + (73,'Mildred','mfuller20@prnewswire.com','151.158.8.130','2000-04-19 10:13:55'), + (74,'Donald','dday21@icq.com','9.178.102.255','1972-12-03 00:58:24'), + (75,'Eric','ethomas22@addtoany.com','85.2.241.227','1992-11-01 05:59:30'), + (76,'Joyce','jarmstrong23@sitemeter.com','169.224.20.36','1985-10-24 06:50:01'), + (77,'Maria','mmartinez24@amazonaws.com','143.189.167.135','2005-10-05 05:17:42'), + (78,'Harry','hburton25@youtube.com','156.47.176.237','1978-03-26 05:53:33'), + (79,'Kevin','klawrence26@hao123.com','79.136.183.83','1994-10-12 04:38:52'), + (80,'David','dhall27@prweb.com','133.149.172.153','1976-12-15 16:24:24'), + (81,'Kathy','kperry28@twitter.com','229.242.72.228','1979-03-04 02:58:56'), + (82,'Adam','aprice29@elegantthemes.com','13.145.21.10','1982-11-07 11:46:59'), + (83,'Brandon','bgriffin2a@va.gov','73.249.128.212','2013-10-30 05:30:36'), + (84,'Henry','hnguyen2b@discovery.com','211.36.214.242','1985-01-09 06:37:27'), + (85,'Eric','esanchez2c@edublogs.org','191.166.188.251','2004-05-01 23:21:42'), + (86,'Jason','jlee2d@jimdo.com','193.92.16.182','1973-01-08 09:05:39'), + (87,'Diana','drichards2e@istockphoto.com','19.130.175.245','1994-10-05 22:50:49'), + (88,'Andrea','awelch2f@abc.net.au','94.155.233.96','2002-04-26 08:41:44'), + (89,'Louis','lwagner2g@miitbeian.gov.cn','26.217.34.111','2003-08-25 07:56:39'), + (90,'Jane','jsims2h@seesaa.net','43.4.220.135','1987-03-20 20:39:04'), + (91,'Larry','lgrant2i@si.edu','97.126.79.34','2000-09-07 20:26:19'), + (92,'Louis','ldean2j@prnewswire.com','37.148.40.127','2011-09-16 20:12:14'), + (93,'Jennifer','jcampbell2k@xing.com','38.106.254.142','1988-07-15 05:06:49'), + (94,'Wayne','wcunningham2l@google.com.hk','223.28.26.187','2009-12-15 06:16:54'), + (95,'Lori','lstevens2m@icq.com','181.250.181.58','1984-10-28 03:29:19'), + (96,'Judy','jsimpson2n@marriott.com','180.121.239.219','1986-02-07 15:18:10'), + (97,'Phillip','phoward2o@usa.gov','255.247.0.175','2002-12-26 08:44:45'), + (98,'Gloria','gwalker2p@usa.gov','156.140.7.128','1997-10-04 07:58:58'), + (99,'Paul','pjohnson2q@umn.edu','183.59.198.197','1991-11-14 12:33:55'), + (100,'Frank','fgreene2r@blogspot.com','150.143.68.121','2010-06-12 23:55:39'), + (101,'Deborah','dknight2s@reverbnation.com','222.131.211.191','1970-07-08 08:54:23'), + (102,'Sandra','sblack2t@tripadvisor.com','254.183.128.254','2000-04-12 02:39:36'), + (103,'Edward','eburns2u@dailymotion.com','253.89.118.18','1993-10-10 10:54:01'), + (104,'Anthony','ayoung2v@ustream.tv','118.4.193.176','1978-08-26 17:07:29'), + (105,'Donald','dlawrence2w@wp.com','139.200.159.227','2007-07-21 20:56:20'), + (106,'Matthew','mfreeman2x@google.fr','205.26.239.92','2014-12-05 17:05:39'), + (107,'Sean','ssanders2y@trellian.com','143.89.82.108','1993-07-14 21:45:02'), + (108,'Sharon','srobinson2z@soundcloud.com','66.234.247.54','1977-04-06 19:07:03'), + (109,'Jennifer','jwatson30@t-online.de','196.102.127.7','1998-03-07 05:12:23'), + (110,'Clarence','cbrooks31@si.edu','218.93.234.73','2002-11-06 17:22:25'), + (111,'Jose','jflores32@goo.gl','185.105.244.231','1995-01-05 06:32:21'), + (112,'George','glee33@adobe.com','173.82.249.196','2015-01-04 02:47:46'), + (113,'Larry','lhill34@linkedin.com','66.5.206.195','2010-11-02 10:21:17'), + (114,'Marie','mmeyer35@mysql.com','151.152.88.107','1990-05-22 20:52:51'), + (115,'Clarence','cwebb36@skype.com','130.198.55.217','1972-10-27 07:38:54'), + (116,'Sarah','scarter37@answers.com','80.89.18.153','1971-08-24 19:29:30'), + (117,'Henry','hhughes38@webeden.co.uk','152.60.114.174','1973-01-27 09:00:42'), + (118,'Teresa','thenry39@hao123.com','32.187.239.106','2015-11-06 01:48:44'), + (119,'Billy','bgutierrez3a@sun.com','52.37.70.134','2002-03-19 03:20:19'), + (120,'Anthony','agibson3b@github.io','154.251.232.213','1991-04-19 01:08:15'), + (121,'Sandra','sromero3c@wikia.com','44.124.171.2','1998-09-06 20:30:34'), + (122,'Paula','pandrews3d@blogs.com','153.142.118.226','2003-06-24 16:31:24'), + (123,'Terry','tbaker3e@csmonitor.com','99.120.45.219','1970-12-09 23:57:21'), + (124,'Lois','lwilson3f@reuters.com','147.44.171.83','1971-01-09 22:28:51'), + (125,'Sara','smorgan3g@nature.com','197.67.192.230','1992-01-28 20:33:24'), + (126,'Charles','ctorres3h@china.com.cn','156.115.216.2','1993-10-02 19:36:34'), + (127,'Richard','ralexander3i@marriott.com','248.235.180.59','1999-02-03 18:40:55'), + (128,'Christina','charper3j@cocolog-nifty.com','152.114.116.129','1978-09-13 00:37:32'), + (129,'Steve','sadams3k@economist.com','112.248.91.98','2004-03-21 09:07:43'), + (130,'Katherine','krobertson3l@ow.ly','37.220.107.28','1977-03-18 19:28:50'), + (131,'Donna','dgibson3m@state.gov','222.218.76.221','1999-02-01 06:46:16'), + (132,'Christina','cwest3n@mlb.com','152.114.6.160','1979-12-24 15:30:35'), + (133,'Sandra','swillis3o@meetup.com','180.71.49.34','1984-09-27 08:05:54'), + (134,'Clarence','cedwards3p@smugmug.com','10.64.180.186','1979-04-16 16:52:10'), + (135,'Ruby','rjames3q@wp.com','98.61.54.20','2007-01-13 14:25:52'), + (136,'Sarah','smontgomery3r@tripod.com','91.45.164.172','2009-07-25 04:34:30'), + (137,'Sarah','soliver3s@eventbrite.com','30.106.39.146','2012-05-09 22:12:33'), + (138,'Deborah','dwheeler3t@biblegateway.com','59.105.213.173','1999-11-09 08:08:44'), + (139,'Deborah','dray3u@i2i.jp','11.108.186.217','2014-02-04 03:15:19'), + (140,'Paul','parmstrong3v@alexa.com','6.250.59.43','2009-12-21 10:08:53'), + (141,'Aaron','abishop3w@opera.com','207.145.249.62','1996-04-25 23:20:23'), + (142,'Henry','hsanders3x@google.ru','140.215.203.171','2012-01-29 11:52:32'), + (143,'Anne','aanderson3y@1688.com','74.150.102.118','1982-04-03 13:46:17'), + (144,'Victor','vmurphy3z@hugedomains.com','222.155.99.152','1987-11-03 19:58:41'), + (145,'Evelyn','ereid40@pbs.org','249.122.33.117','1977-12-14 17:09:57'), + (146,'Brian','bgonzalez41@wikia.com','246.254.235.141','1991-02-24 00:45:58'), + (147,'Sandra','sgray42@squarespace.com','150.73.28.159','1972-07-28 17:26:32'), + (148,'Alice','ajones43@a8.net','78.253.12.177','2002-12-05 16:57:46'), + (149,'Jessica','jhanson44@mapquest.com','87.229.30.160','1994-01-30 11:40:04'), + (150,'Louise','lbailey45@reuters.com','191.219.31.101','2011-09-07 21:11:45'), + (151,'Christopher','cgonzalez46@printfriendly.com','83.137.213.239','1984-10-24 14:58:04'), + (152,'Gregory','gcollins47@yandex.ru','28.176.10.115','1998-07-25 17:17:10'), + (153,'Jane','jperkins48@usnews.com','46.53.164.159','1979-08-19 15:25:00'), + (154,'Phyllis','plong49@yahoo.co.jp','208.140.88.2','1985-07-06 02:16:36'), + (155,'Adam','acarter4a@scribd.com','78.48.148.204','2005-07-20 03:31:09'), + (156,'Frank','fweaver4b@angelfire.com','199.180.255.224','2011-03-04 23:07:54'), + (157,'Ronald','rmurphy4c@cloudflare.com','73.42.97.231','1991-01-11 10:39:41'), + (158,'Richard','rmorris4d@e-recht24.de','91.9.97.223','2009-01-17 21:05:15'), + (159,'Rose','rfoster4e@woothemes.com','203.169.53.16','1991-04-21 02:09:38'), + (160,'George','ggarrett4f@uiuc.edu','186.61.5.167','1989-11-11 11:29:42'), + (161,'Victor','vhamilton4g@biblegateway.com','121.229.138.38','2012-06-22 18:01:23'), + (162,'Mark','mbennett4h@businessinsider.com','209.184.29.203','1980-04-16 15:26:34'), + (163,'Martin','mwells4i@ifeng.com','97.223.55.105','2010-05-26 14:08:18'), + (164,'Diana','dstone4j@google.ru','90.155.52.47','2013-02-11 00:14:54'), + (165,'Walter','wferguson4k@blogger.com','30.63.212.44','1986-02-20 17:46:46'), + (166,'Denise','dcoleman4l@vistaprint.com','10.209.153.77','1992-05-13 20:14:14'), + (167,'Philip','pknight4m@xing.com','15.28.135.167','2000-09-11 18:41:13'), + (168,'Russell','rcarr4n@youtube.com','113.55.165.50','2008-07-10 17:49:27'), + (169,'Donna','dburke4o@dion.ne.jp','70.0.105.111','1992-02-10 17:24:58'), + (170,'Anne','along4p@squidoo.com','36.154.58.107','2012-08-19 23:35:31'), + (171,'Clarence','cbanks4q@webeden.co.uk','94.57.53.114','1972-03-11 21:46:44'), + (172,'Betty','bbowman4r@cyberchimps.com','178.115.209.69','2013-01-13 21:34:51'), + (173,'Andrew','ahudson4s@nytimes.com','84.32.252.144','1998-09-15 14:20:04'), + (174,'Keith','kgordon4t@cam.ac.uk','189.237.211.102','2009-01-22 05:34:38'), + (175,'Patrick','pwheeler4u@mysql.com','47.22.117.226','1984-09-05 22:33:15'), + (176,'Jesse','jfoster4v@mapquest.com','229.95.131.46','1990-01-20 12:19:15'), + (177,'Arthur','afisher4w@jugem.jp','107.255.244.98','1983-10-13 11:08:46'), + (178,'Nicole','nryan4x@wsj.com','243.211.33.221','1974-05-30 23:19:14'), + (179,'Bruce','bjohnson4y@sfgate.com','17.41.200.101','1992-09-23 02:02:19'), + (180,'Terry','tcox4z@reference.com','20.189.120.106','1982-02-13 12:43:14'), + (181,'Ashley','astanley50@kickstarter.com','86.3.56.98','1976-05-09 01:27:16'), + (182,'Michael','mrivera51@about.me','72.118.249.0','1971-11-11 17:28:37'), + (183,'Steven','sgonzalez52@mozilla.org','169.112.247.47','2002-08-24 14:59:25'), + (184,'Kathleen','kfuller53@bloglovin.com','80.93.59.30','2002-03-11 13:41:29'), + (185,'Nicole','nhenderson54@usda.gov','39.253.60.30','1995-04-24 05:55:07'), + (186,'Ralph','rharper55@purevolume.com','167.147.142.189','1980-02-10 18:35:45'), + (187,'Heather','hcunningham56@photobucket.com','96.222.196.229','2007-06-15 05:37:50'), + (188,'Nancy','nlittle57@cbc.ca','241.53.255.175','2007-07-12 23:42:48'), + (189,'Juan','jramirez58@pinterest.com','190.128.84.27','1978-11-07 23:37:37'), + (190,'Beverly','bfowler59@chronoengine.com','54.144.230.49','1979-03-31 23:27:28'), + (191,'Shirley','sstevens5a@prlog.org','200.97.231.248','2011-12-06 07:08:50'), + (192,'Annie','areyes5b@squidoo.com','223.32.182.101','2011-05-28 02:42:09'), + (193,'Jack','jkelley5c@tiny.cc','47.34.118.150','1981-12-05 17:31:40'), + (194,'Keith','krobinson5d@1und1.de','170.210.209.31','1999-03-09 11:05:43'), + (195,'Joseph','jmiller5e@google.com.au','136.74.212.139','1984-10-08 13:18:20'), + (196,'Annie','aday5f@blogspot.com','71.99.186.69','1986-02-18 12:27:34'), + (197,'Nancy','nperez5g@liveinternet.ru','28.160.6.107','1983-10-20 17:51:20'), + (198,'Tammy','tward5h@ucoz.ru','141.43.164.70','1980-03-31 04:45:29'), + (199,'Doris','dryan5i@ted.com','239.117.202.188','1985-07-03 03:17:53'), + (200,'Rose','rmendoza5j@photobucket.com','150.200.206.79','1973-04-21 21:36:40'), + (201,'Cynthia','cbutler5k@hubpages.com','80.153.174.161','2001-01-20 01:42:26'), + (202,'Samuel','soliver5l@people.com.cn','86.127.246.140','1970-09-02 02:19:00'), + (203,'Carl','csanchez5m@mysql.com','50.149.237.107','1993-12-01 07:02:09'), + (204,'Kathryn','kowens5n@geocities.jp','145.166.205.201','2004-07-06 18:39:33'), + (205,'Nicholas','nnichols5o@parallels.com','190.240.66.170','2014-11-11 18:52:19'), + (206,'Keith','kwillis5p@youtube.com','181.43.206.100','1998-06-13 06:30:51'), + (207,'Justin','jwebb5q@intel.com','211.54.245.74','2000-11-04 16:58:26'), + (208,'Gary','ghicks5r@wikipedia.org','196.154.213.104','1992-12-01 19:48:28'), + (209,'Martin','mpowell5s@flickr.com','153.67.12.241','1983-06-30 06:24:32'), + (210,'Brenda','bkelley5t@xinhuanet.com','113.100.5.172','2005-01-08 20:50:22'), + (211,'Edward','eray5u@a8.net','205.187.246.65','2011-09-26 08:04:44'), + (212,'Steven','slawson5v@senate.gov','238.150.250.36','1978-11-22 02:48:09'), + (213,'Robert','rthompson5w@furl.net','70.7.89.236','2001-09-12 08:52:07'), + (214,'Jack','jporter5x@diigo.com','220.172.29.99','1976-07-26 14:29:21'), + (215,'Lisa','ljenkins5y@oakley.com','150.151.170.180','2010-03-20 19:21:16'), + (216,'Theresa','tbell5z@mayoclinic.com','247.25.53.173','2001-03-11 05:36:40'), + (217,'Jimmy','jstephens60@weather.com','145.101.93.235','1983-04-12 09:35:30'), + (218,'Louis','lhunt61@amazon.co.jp','78.137.6.253','1997-08-29 19:34:34'), + (219,'Lawrence','lgilbert62@ted.com','243.132.8.78','2015-04-08 22:06:56'), + (220,'David','dgardner63@4shared.com','204.40.46.136','1971-07-09 03:29:11'), + (221,'Charles','ckennedy64@gmpg.org','211.83.233.2','2011-02-26 11:55:04'), + (222,'Lillian','lbanks65@msu.edu','124.233.12.80','2010-05-16 20:29:02'), + (223,'Ernest','enguyen66@baidu.com','82.45.128.148','1996-07-04 10:07:04'), + (224,'Ryan','rrussell67@cloudflare.com','202.53.240.223','1983-08-05 12:36:29'), + (225,'Donald','ddavis68@ustream.tv','47.39.218.137','1989-05-27 02:30:56'), + (226,'Joe','jscott69@blogspot.com','140.23.131.75','1973-03-16 12:21:31'), + (227,'Anne','amarshall6a@google.ca','113.162.200.197','1988-12-09 03:38:29'), + (228,'Willie','wturner6b@constantcontact.com','85.83.182.249','1991-10-06 01:51:10'), + (229,'Nicole','nwilson6c@sogou.com','30.223.51.135','1977-05-29 19:54:56'), + (230,'Janet','jwheeler6d@stumbleupon.com','153.194.27.144','2011-03-13 12:48:47'), + (231,'Lois','lcarr6e@statcounter.com','0.41.36.53','1993-02-06 04:52:01'), + (232,'Shirley','scruz6f@tmall.com','37.156.39.223','2007-02-18 17:47:01'), + (233,'Patrick','pford6g@reverbnation.com','36.198.200.89','1977-03-06 15:47:24'), + (234,'Lisa','lhudson6h@usatoday.com','134.213.58.137','2014-10-28 01:56:56'), + (235,'Pamela','pmartinez6i@opensource.org','5.151.127.202','1987-11-30 16:44:47'), + (236,'Larry','lperez6j@infoseek.co.jp','235.122.96.148','1979-01-18 06:33:45'), + (237,'Pamela','pramirez6k@census.gov','138.233.34.163','2012-01-29 10:35:20'), + (238,'Daniel','dcarr6l@php.net','146.21.152.242','1984-11-17 08:22:59'), + (239,'Patrick','psmith6m@indiegogo.com','136.222.199.36','2001-05-30 22:16:44'), + (240,'Raymond','rhenderson6n@hc360.com','116.31.112.38','2000-01-05 20:35:41'), + (241,'Teresa','treynolds6o@miitbeian.gov.cn','198.126.205.220','1996-11-08 01:27:31'), + (242,'Johnny','jmason6p@flickr.com','192.8.232.114','2013-05-14 05:35:50'), + (243,'Angela','akelly6q@guardian.co.uk','234.116.60.197','1977-08-20 02:05:17'), + (244,'Douglas','dcole6r@cmu.edu','128.135.212.69','2016-10-26 17:40:36'), + (245,'Frances','fcampbell6s@twitpic.com','94.22.243.235','1987-04-26 07:07:13'), + (246,'Donna','dgreen6t@chron.com','227.116.46.107','2011-07-25 12:59:54'), + (247,'Benjamin','bfranklin6u@redcross.org','89.141.142.89','1974-05-03 20:28:18'), + (248,'Randy','rpalmer6v@rambler.ru','70.173.63.178','2011-12-20 17:40:18'), + (249,'Melissa','mmurray6w@bbb.org','114.234.118.137','1991-02-26 12:45:44'), + (250,'Jean','jlittle6x@epa.gov','141.21.163.254','1991-08-16 04:57:09'), + (251,'Daniel','dolson6y@nature.com','125.75.104.97','2010-04-23 06:25:54'), + (252,'Kathryn','kwells6z@eventbrite.com','225.104.28.249','2015-01-31 02:21:50'), + (253,'Theresa','tgonzalez70@ox.ac.uk','91.93.156.26','1971-12-11 10:31:31'), + (254,'Beverly','broberts71@bluehost.com','244.40.158.89','2013-09-21 13:02:31'), + (255,'Pamela','pmurray72@netscape.com','218.54.95.216','1985-04-16 00:34:00'), + (256,'Timothy','trichardson73@amazonaws.com','235.49.24.229','2000-11-11 09:48:28'), + (257,'Mildred','mpalmer74@is.gd','234.125.95.132','1992-05-25 02:25:02'), + (258,'Jessica','jcampbell75@google.it','55.98.30.140','2014-08-26 00:26:34'), + (259,'Beverly','bthomas76@cpanel.net','48.78.228.176','1970-08-18 10:40:05'), + (260,'Eugene','eward77@cargocollective.com','139.226.204.2','1996-12-04 23:17:00'), + (261,'Andrea','aallen78@webnode.com','160.31.214.38','2009-07-06 07:22:37'), + (262,'Justin','jruiz79@merriam-webster.com','150.149.246.122','2005-06-06 11:44:19'), + (263,'Kenneth','kedwards7a@networksolutions.com','98.82.193.128','2001-07-03 02:00:10'), + (264,'Rachel','rday7b@miibeian.gov.cn','114.15.247.221','1994-08-18 19:45:40'), + (265,'Russell','rmiller7c@instagram.com','184.130.152.253','1977-11-06 01:58:12'), + (266,'Bonnie','bhudson7d@cornell.edu','235.180.186.206','1990-12-03 22:45:24'), + (267,'Raymond','rknight7e@yandex.ru','161.2.44.252','1995-08-25 04:31:19'), + (268,'Bonnie','brussell7f@elpais.com','199.237.57.207','1991-03-29 08:32:06'), + (269,'Marie','mhenderson7g@elpais.com','52.203.131.144','2004-06-04 21:50:28'), + (270,'Alan','acarr7h@trellian.com','147.51.205.72','2005-03-03 10:51:31'), + (271,'Barbara','bturner7i@hugedomains.com','103.160.110.226','2004-08-04 13:42:40'), + (272,'Christina','cdaniels7j@census.gov','0.238.61.251','1972-10-18 12:47:33'), + (273,'Jeremy','jgomez7k@reuters.com','111.26.65.56','2013-01-13 10:41:35'), + (274,'Laura','lwood7l@icio.us','149.153.38.205','2011-06-25 09:33:59'), + (275,'Matthew','mbowman7m@auda.org.au','182.138.206.172','1999-03-05 03:25:36'), + (276,'Denise','dparker7n@icq.com','0.213.88.138','2011-11-04 09:43:06'), + (277,'Phillip','pparker7o@discuz.net','219.242.165.240','1973-10-19 04:22:29'), + (278,'Joan','jpierce7p@salon.com','63.31.213.202','1989-04-09 22:06:24'), + (279,'Irene','ibaker7q@cbc.ca','102.33.235.114','1992-09-04 13:00:57'), + (280,'Betty','bbowman7r@ted.com','170.91.249.242','2015-09-28 08:14:22'), + (281,'Teresa','truiz7s@boston.com','82.108.158.207','1999-07-18 05:17:09'), + (282,'Helen','hbrooks7t@slideshare.net','102.87.162.187','2003-01-06 15:45:29'), + (283,'Karen','kgriffin7u@wunderground.com','43.82.44.184','2010-05-28 01:56:37'), + (284,'Lisa','lfernandez7v@mtv.com','200.238.218.220','1993-04-03 20:33:51'), + (285,'Jesse','jlawrence7w@timesonline.co.uk','95.122.105.78','1990-01-05 17:28:43'), + (286,'Terry','tross7x@macromedia.com','29.112.114.133','2009-08-29 21:32:17'), + (287,'Angela','abradley7y@icq.com','177.44.27.72','1989-10-04 21:46:06'), + (288,'Maria','mhart7z@dailymotion.com','55.27.55.202','1975-01-21 01:22:57'), + (289,'Raymond','randrews80@pinterest.com','88.90.78.67','1992-03-16 21:37:40'), + (290,'Kathy','krice81@bluehost.com','212.63.196.102','2000-12-14 03:06:44'), + (291,'Cynthia','cramos82@nymag.com','107.89.190.6','2005-06-28 02:02:33'), + (292,'Kimberly','kjones83@mysql.com','86.169.101.101','2007-06-13 22:56:49'), + (293,'Timothy','thansen84@microsoft.com','108.100.254.90','2003-04-04 10:31:57'), + (294,'Carol','cspencer85@berkeley.edu','75.118.144.187','1999-03-30 14:53:21'), + (295,'Louis','lmedina86@latimes.com','141.147.163.24','1991-04-11 17:53:13'), + (296,'Margaret','mcole87@google.fr','53.184.26.83','1991-12-19 01:54:10'), + (297,'Mary','mgomez88@yellowpages.com','208.56.57.99','1976-05-21 18:05:08'), + (298,'Amanda','aanderson89@geocities.com','147.73.15.252','1987-08-22 15:05:28'), + (299,'Kathryn','kgarrett8a@nature.com','27.29.177.220','1976-07-15 04:25:04'), + (300,'Dorothy','dmason8b@shareasale.com','106.210.99.193','1990-09-03 21:39:31'), + (301,'Lois','lkennedy8c@amazon.de','194.169.29.187','2007-07-29 14:09:31'), + (302,'Irene','iburton8d@washingtonpost.com','196.143.110.249','2013-09-05 11:32:46'), + (303,'Betty','belliott8e@wired.com','183.105.222.199','1979-09-19 19:29:13'), + (304,'Bobby','bmeyer8f@census.gov','36.13.161.145','2014-05-24 14:34:39'), + (305,'Ann','amorrison8g@sfgate.com','72.154.54.137','1978-10-05 14:22:34'), + (306,'Daniel','djackson8h@wunderground.com','144.95.32.34','1990-07-27 13:23:05'), + (307,'Joe','jboyd8i@alibaba.com','187.105.86.178','2011-09-28 16:46:32'), + (308,'Ralph','rdunn8j@fc2.com','3.19.87.255','1984-10-18 08:00:40'), + (309,'Craig','ccarter8k@gizmodo.com','235.152.76.215','1998-07-04 12:15:21'), + (310,'Paula','pdean8l@hhs.gov','161.100.173.197','1973-02-13 09:38:55'), + (311,'Andrew','agarrett8m@behance.net','199.253.123.218','1991-02-14 13:36:32'), + (312,'Janet','jhowell8n@alexa.com','39.189.139.79','2012-11-24 20:17:33'), + (313,'Keith','khansen8o@godaddy.com','116.186.223.196','1987-08-23 21:22:05'), + (314,'Nicholas','nedwards8p@state.gov','142.175.142.11','1977-03-28 18:27:27'), + (315,'Jacqueline','jallen8q@oaic.gov.au','189.66.135.192','1994-10-26 11:44:26'), + (316,'Frank','fgardner8r@mapy.cz','154.77.119.169','1983-01-29 19:19:51'), + (317,'Eric','eharrison8s@google.cn','245.139.65.123','1984-02-04 09:54:36'), + (318,'Gregory','gcooper8t@go.com','171.147.0.221','2004-06-14 05:22:08'), + (319,'Jean','jfreeman8u@rakuten.co.jp','67.243.121.5','1977-01-07 18:23:43'), + (320,'Juan','jlewis8v@shinystat.com','216.181.171.189','2001-08-23 17:32:43'), + (321,'Randy','rwilliams8w@shinystat.com','105.152.146.28','1983-02-17 00:05:50'), + (322,'Stephen','shart8x@sciencedirect.com','196.131.205.148','2004-02-15 10:12:03'), + (323,'Annie','ahunter8y@example.com','63.36.34.103','2003-07-23 21:15:25'), + (324,'Melissa','mflores8z@cbc.ca','151.230.217.90','1983-11-02 14:53:56'), + (325,'Jane','jweaver90@about.me','0.167.235.217','1987-07-29 00:13:44'), + (326,'Anthony','asmith91@oracle.com','97.87.48.41','2001-05-31 18:44:11'), + (327,'Terry','tdavis92@buzzfeed.com','46.20.12.51','2015-09-12 23:13:55'), + (328,'Brandon','bmontgomery93@gravatar.com','252.101.48.186','2010-10-28 08:26:27'), + (329,'Chris','cmurray94@bluehost.com','25.158.167.97','2004-05-05 16:10:31'), + (330,'Denise','dfuller95@hugedomains.com','216.210.149.28','1979-04-20 08:57:24'), + (331,'Arthur','amcdonald96@sakura.ne.jp','206.42.36.213','2009-08-15 03:26:16'), + (332,'Jesse','jhoward97@google.cn','46.181.118.30','1974-04-18 14:08:41'), + (333,'Frank','fsimpson98@domainmarket.com','163.220.211.87','2006-06-30 14:46:52'), + (334,'Janice','jwoods99@pen.io','229.245.237.182','1988-04-06 11:52:58'), + (335,'Rebecca','rroberts9a@huffingtonpost.com','148.96.15.80','1976-10-05 08:44:16'), + (336,'Joshua','jray9b@opensource.org','192.253.12.198','1971-12-25 22:27:07'), + (337,'Joyce','jcarpenter9c@statcounter.com','125.171.46.215','2001-12-31 22:08:13'), + (338,'Andrea','awest9d@privacy.gov.au','79.101.180.201','1983-02-18 20:07:47'), + (339,'Christine','chudson9e@yelp.com','64.198.43.56','1997-09-08 08:03:43'), + (340,'Joe','jparker9f@earthlink.net','251.215.148.153','1973-11-04 05:08:18'), + (341,'Thomas','tkim9g@answers.com','49.187.34.47','1991-08-07 21:13:48'), + (342,'Janice','jdean9h@scientificamerican.com','4.197.117.16','2009-12-08 02:35:49'), + (343,'James','jmitchell9i@umich.edu','43.121.18.147','2011-04-28 17:04:09'), + (344,'Charles','cgardner9j@purevolume.com','197.78.240.240','1998-02-11 06:47:07'), + (345,'Robert','rhenderson9k@friendfeed.com','215.84.180.88','2002-05-10 15:33:14'), + (346,'Chris','cgray9l@4shared.com','249.70.192.240','1998-10-03 16:43:42'), + (347,'Gloria','ghayes9m@hibu.com','81.103.138.26','1999-12-26 11:23:13'), + (348,'Edward','eramirez9n@shareasale.com','38.136.90.136','2010-08-19 08:01:06'), + (349,'Cheryl','cbutler9o@google.ca','172.180.78.172','1995-05-27 20:03:52'), + (350,'Margaret','mwatkins9p@sfgate.com','3.20.198.6','2014-10-21 01:42:58'), + (351,'Rebecca','rwelch9q@examiner.com','45.81.42.208','2001-02-08 12:19:06'), + (352,'Joe','jpalmer9r@phpbb.com','163.202.92.190','1970-01-05 11:29:12'), + (353,'Sandra','slewis9s@dyndns.org','77.215.201.236','1974-01-05 07:04:04'), + (354,'Todd','tfranklin9t@g.co','167.125.181.82','2009-09-28 10:13:58'), + (355,'Joseph','jlewis9u@webmd.com','244.204.6.11','1990-10-21 15:49:57'), + (356,'Alan','aknight9v@nydailynews.com','152.197.95.83','1996-03-08 08:43:17'), + (357,'Sharon','sdean9w@123-reg.co.uk','237.46.40.26','1985-11-30 12:09:24'), + (358,'Annie','awright9x@cafepress.com','190.45.231.111','2000-08-24 11:56:06'), + (359,'Diane','dhamilton9y@youtube.com','85.146.171.196','2015-02-24 02:03:57'), + (360,'Antonio','alane9z@auda.org.au','61.63.146.203','2001-05-13 03:43:34'), + (361,'Matthew','mallena0@hhs.gov','29.97.32.19','1973-02-19 23:43:32'), + (362,'Bonnie','bfowlera1@soup.io','251.216.99.53','2013-08-01 15:35:41'), + (363,'Margaret','mgraya2@examiner.com','69.255.151.79','1998-01-23 22:24:59'), + (364,'Joan','jwagnera3@printfriendly.com','192.166.120.61','1973-07-13 00:30:22'), + (365,'Catherine','cperkinsa4@nytimes.com','58.21.24.214','2006-11-19 11:52:26'), + (366,'Mark','mcartera5@cpanel.net','220.33.102.142','2007-09-09 09:43:27'), + (367,'Paula','ppricea6@msn.com','36.182.238.124','2009-11-11 09:13:05'), + (368,'Catherine','cgreena7@army.mil','228.203.58.19','2005-08-09 16:52:15'), + (369,'Helen','hhamiltona8@symantec.com','155.56.194.99','2005-02-01 05:40:36'), + (370,'Jane','jmeyera9@ezinearticles.com','133.244.113.213','2013-11-06 22:10:23'), + (371,'Wanda','wevansaa@bloglovin.com','233.125.192.48','1994-12-26 23:43:42'), + (372,'Mark','mmarshallab@tumblr.com','114.74.60.47','2016-09-29 18:03:01'), + (373,'Andrew','amartinezac@google.cn','182.54.37.130','1976-06-06 17:04:17'), + (374,'Helen','hmoralesad@e-recht24.de','42.45.4.123','1977-03-28 19:06:59'), + (375,'Bonnie','bstoneae@php.net','196.149.79.137','1970-02-05 17:05:58'), + (376,'Douglas','dfreemanaf@nasa.gov','215.65.124.218','2008-11-20 21:51:55'), + (377,'Willie','wwestag@army.mil','35.189.92.118','1992-07-24 05:08:08'), + (378,'Cheryl','cwagnerah@upenn.edu','228.239.222.141','2010-01-25 06:29:01'), + (379,'Sandra','swardai@baidu.com','63.11.113.240','1985-05-23 08:07:37'), + (380,'Julie','jrobinsonaj@jugem.jp','110.58.202.50','2015-03-05 09:42:07'), + (381,'Larry','lwagnerak@shop-pro.jp','98.234.25.24','1975-07-22 22:22:02'), + (382,'Juan','jcastilloal@yelp.com','24.174.74.202','2007-01-17 09:32:43'), + (383,'Donna','dfrazieram@artisteer.com','205.26.147.45','1990-02-11 20:55:46'), + (384,'Rachel','rfloresan@w3.org','109.60.216.162','1983-05-22 22:42:18'), + (385,'Robert','rreynoldsao@theguardian.com','122.65.209.130','2009-05-01 18:02:51'), + (386,'Donald','dbradleyap@etsy.com','42.54.35.126','1997-01-16 16:31:52'), + (387,'Rachel','rfisheraq@nih.gov','160.243.250.45','2006-02-17 22:05:49'), + (388,'Nicholas','nhamiltonar@princeton.edu','156.211.37.111','1976-06-21 03:36:29'), + (389,'Timothy','twhiteas@ca.gov','36.128.23.70','1975-09-24 03:51:18'), + (390,'Diana','dbradleyat@odnoklassniki.ru','44.102.120.184','1983-04-27 09:02:50'), + (391,'Billy','bfowlerau@jimdo.com','91.200.68.196','1995-01-29 06:57:35'), + (392,'Bruce','bandrewsav@ucoz.com','48.12.101.125','1992-10-27 04:31:39'), + (393,'Linda','lromeroaw@usa.gov','100.71.233.19','1992-06-08 15:13:18'), + (394,'Debra','dwatkinsax@ucoz.ru','52.160.233.193','2001-11-11 06:51:01'), + (395,'Katherine','kburkeay@wix.com','151.156.242.141','2010-06-14 19:54:28'), + (396,'Martha','mharrisonaz@youku.com','21.222.10.199','1989-10-16 14:17:55'), + (397,'Dennis','dwellsb0@youtu.be','103.16.29.3','1985-12-21 06:05:51'), + (398,'Gloria','grichardsb1@bloglines.com','90.147.120.234','1982-08-27 01:04:43'), + (399,'Brenda','bfullerb2@t.co','33.253.63.90','2011-04-20 05:00:35'), + (400,'Larry','lhendersonb3@disqus.com','88.95.132.128','1982-08-31 02:15:12'), + (401,'Richard','rlarsonb4@wisc.edu','13.48.231.150','1979-04-15 14:08:09'), + (402,'Terry','thuntb5@usa.gov','65.91.103.240','1998-05-15 11:50:49'), + (403,'Harry','hburnsb6@nasa.gov','33.38.21.244','1981-04-12 14:02:20'), + (404,'Diana','dellisb7@mlb.com','218.229.81.135','1997-01-29 00:17:25'), + (405,'Jack','jburkeb8@tripadvisor.com','210.227.182.216','1984-03-09 17:24:03'), + (406,'Julia','jlongb9@fotki.com','10.210.12.104','2005-10-26 03:54:13'), + (407,'Lois','lscottba@msu.edu','188.79.136.138','1973-02-02 18:40:39'), + (408,'Sandra','shendersonbb@shareasale.com','114.171.220.108','2012-06-09 18:22:26'), + (409,'Irene','isanchezbc@cdbaby.com','109.255.50.119','1983-09-28 21:11:27'), + (410,'Emily','ebrooksbd@bandcamp.com','227.81.93.79','1970-08-31 21:08:01'), + (411,'Michelle','mdiazbe@businessweek.com','236.249.6.226','1993-05-22 08:07:07'), + (412,'Tammy','tbennettbf@wisc.edu','145.253.239.152','1978-12-31 20:24:51'), + (413,'Christine','cgreenebg@flickr.com','97.25.140.118','1978-07-17 12:55:30'), + (414,'Patricia','pgarzabh@tuttocitta.it','139.246.192.211','1984-02-27 13:40:08'), + (415,'Kimberly','kromerobi@aol.com','73.56.88.247','1976-09-16 14:22:04'), + (416,'George','gjohnstonbj@fda.gov','240.36.245.185','1979-07-24 14:36:02'), + (417,'Eugene','efullerbk@sciencedaily.com','42.38.105.140','2012-09-12 01:56:41'), + (418,'Andrea','astevensbl@goo.gl','31.152.207.204','1979-05-24 11:06:21'), + (419,'Shirley','sreidbm@scientificamerican.com','103.60.31.241','1984-02-23 04:07:41'), + (420,'Terry','tmorenobn@blinklist.com','92.161.34.42','1994-06-25 14:01:35'), + (421,'Christopher','cmorenobo@go.com','158.86.176.82','1973-09-05 09:18:47'), + (422,'Dennis','dhansonbp@ning.com','40.160.81.75','1982-01-20 10:19:41'), + (423,'Beverly','brussellbq@de.vu','138.32.56.204','1997-11-06 07:20:19'), + (424,'Howard','hparkerbr@163.com','103.171.134.171','2015-06-24 15:37:10'), + (425,'Helen','hmccoybs@fema.gov','61.200.4.71','1995-06-20 08:59:10'), + (426,'Ann','ahudsonbt@cafepress.com','239.187.71.125','1977-04-11 07:59:28'), + (427,'Tina','twestbu@nhs.uk','80.213.117.74','1992-08-19 05:54:44'), + (428,'Terry','tnguyenbv@noaa.gov','21.93.118.95','1991-09-19 23:22:55'), + (429,'Ashley','aburtonbw@wix.com','233.176.205.109','2009-11-10 05:01:20'), + (430,'Eric','emyersbx@1und1.de','168.91.212.67','1987-08-10 07:16:20'), + (431,'Barbara','blittleby@lycos.com','242.14.189.239','2008-08-02 12:13:04'), + (432,'Sean','sevansbz@instagram.com','14.39.177.13','2007-04-16 17:28:49'), + (433,'Shirley','sburtonc0@newsvine.com','34.107.138.76','1980-12-10 02:19:29'), + (434,'Patricia','pfreemanc1@so-net.ne.jp','219.213.142.117','1987-03-01 02:25:45'), + (435,'Paula','pfosterc2@vkontakte.ru','227.14.138.141','1972-09-22 12:59:34'), + (436,'Nicole','nstewartc3@1688.com','8.164.23.115','1998-10-27 00:10:17'), + (437,'Earl','ekimc4@ovh.net','100.26.244.177','2013-01-22 10:05:46'), + (438,'Beverly','breedc5@reuters.com','174.12.226.27','1974-09-22 07:29:36'), + (439,'Lawrence','lbutlerc6@a8.net','105.164.42.164','1992-06-05 00:43:40'), + (440,'Charles','cmoorec7@ucoz.com','252.197.131.69','1990-04-09 02:34:05'), + (441,'Alice','alawsonc8@live.com','183.73.220.232','1989-02-28 09:11:04'), + (442,'Dorothy','dcarpenterc9@arstechnica.com','241.47.200.14','2005-05-02 19:57:21'), + (443,'Carolyn','cfowlerca@go.com','213.109.55.202','1978-09-10 20:18:20'), + (444,'Anthony','alongcb@free.fr','169.221.158.204','1984-09-13 01:59:23'), + (445,'Annie','amoorecc@e-recht24.de','50.34.148.61','2009-03-26 03:41:07'), + (446,'Carlos','candrewscd@ihg.com','236.69.59.212','1972-03-29 22:42:48'), + (447,'Beverly','bramosce@google.ca','164.250.184.49','1982-11-10 04:34:01'), + (448,'Teresa','tlongcf@umich.edu','174.88.53.223','1987-05-17 12:48:00'), + (449,'Roy','rboydcg@uol.com.br','91.58.243.215','1974-06-16 17:59:54'), + (450,'Ashley','afieldsch@tamu.edu','130.138.11.126','1983-09-15 05:52:36'), + (451,'Judith','jhawkinsci@cmu.edu','200.187.103.245','2003-10-22 12:24:03'), + (452,'Rebecca','rwestcj@ocn.ne.jp','72.85.3.103','1980-11-13 11:01:26'), + (453,'Raymond','rporterck@infoseek.co.jp','146.33.216.151','1982-05-17 23:58:03'), + (454,'Janet','jmarshallcl@odnoklassniki.ru','52.46.193.166','1998-10-04 00:02:21'), + (455,'Shirley','speterscm@salon.com','248.126.31.15','1987-01-30 06:04:59'), + (456,'Annie','abowmancn@economist.com','222.213.248.59','2006-03-14 23:52:59'), + (457,'Jean','jlarsonco@blogspot.com','71.41.25.195','2007-09-08 23:49:45'), + (458,'Phillip','pmoralescp@stanford.edu','74.119.87.28','2011-03-14 20:25:40'), + (459,'Norma','nrobinsoncq@economist.com','28.225.21.54','1989-10-21 01:22:43'), + (460,'Kimberly','kclarkcr@dion.ne.jp','149.171.132.153','2008-06-27 02:27:30'), + (461,'Ruby','rmorriscs@ucla.edu','177.85.163.249','2016-01-28 16:43:44'), + (462,'Jonathan','jcastilloct@tripod.com','78.4.28.77','2000-05-24 17:33:06'), + (463,'Edward','ebryantcu@jigsy.com','140.31.98.193','1992-12-17 08:32:47'), + (464,'Chris','chamiltoncv@eepurl.com','195.171.234.206','1970-12-05 03:42:19'), + (465,'Michael','mweavercw@reference.com','7.233.133.213','1987-03-29 02:30:54'), + (466,'Howard','hlawrencecx@businessweek.com','113.225.124.224','1990-07-30 07:20:57'), + (467,'Philip','phowardcy@comsenz.com','159.170.247.249','2010-10-15 10:18:37'), + (468,'Mary','mmarshallcz@xing.com','125.132.189.70','2007-07-19 13:48:47'), + (469,'Scott','salvarezd0@theguardian.com','78.49.103.230','1987-10-31 06:10:44'), + (470,'Wayne','wcarrolld1@blog.com','238.1.120.204','1980-11-19 03:26:10'), + (471,'Jennifer','jwoodsd2@multiply.com','92.20.224.49','2010-05-06 22:17:04'), + (472,'Raymond','rwelchd3@toplist.cz','176.158.35.240','2007-12-12 19:02:51'), + (473,'Steven','sdixond4@wisc.edu','167.55.237.52','1984-05-05 11:44:37'), + (474,'Ralph','rjamesd5@ameblo.jp','241.190.50.133','2000-07-06 08:44:37'), + (475,'Jason','jrobinsond6@hexun.com','138.119.139.56','2006-02-03 05:27:45'), + (476,'Doris','dwoodd7@fema.gov','180.220.156.190','1978-05-11 20:14:20'), + (477,'Elizabeth','eberryd8@youtu.be','74.188.53.229','2006-11-18 08:29:06'), + (478,'Irene','igilbertd9@privacy.gov.au','194.152.218.1','1985-09-17 02:46:52'), + (479,'Jessica','jdeanda@ameblo.jp','178.103.93.118','1974-06-07 19:04:05'), + (480,'Rachel','ralvarezdb@phoca.cz','17.22.223.174','1999-03-08 02:43:25'), + (481,'Kenneth','kthompsondc@shinystat.com','229.119.91.234','2007-05-15 13:17:32'), + (482,'Harold','hmurraydd@parallels.com','133.26.188.80','1993-11-15 03:42:07'), + (483,'Paula','phowellde@samsung.com','34.215.28.216','1993-11-29 15:55:00'), + (484,'Ruth','rpiercedf@tripadvisor.com','111.30.130.123','1986-08-17 10:19:38'), + (485,'Phyllis','paustindg@vk.com','50.84.34.178','1994-04-13 03:05:24'), + (486,'Laura','lfosterdh@usnews.com','37.8.101.33','2001-06-30 08:58:59'), + (487,'Eric','etaylordi@com.com','103.183.253.45','2006-09-15 20:18:46'), + (488,'Doris','driveradj@prweb.com','247.16.2.199','1989-05-08 09:27:09'), + (489,'Ryan','rhughesdk@elegantthemes.com','103.234.153.232','1989-08-01 18:36:06'), + (490,'Steve','smoralesdl@jigsy.com','3.76.84.207','2011-03-13 17:01:05'), + (491,'Louis','lsullivandm@who.int','78.135.44.208','1975-11-26 16:01:23'), + (492,'Catherine','ctuckerdn@seattletimes.com','93.137.106.21','1990-03-13 16:14:56'), + (493,'Ann','adixondo@gmpg.org','191.136.222.111','2002-06-05 14:22:18'), + (494,'Johnny','jhartdp@amazon.com','103.252.198.39','1988-07-30 23:54:49'), + (495,'Susan','srichardsdq@skype.com','126.247.192.11','2005-01-09 12:08:14'), + (496,'Brenda','bparkerdr@skype.com','63.232.216.86','1974-05-18 05:58:29'), + (497,'Tammy','tmurphyds@constantcontact.com','56.56.37.112','2014-08-05 18:22:25'), + (498,'Larry','lhayesdt@wordpress.com','162.146.13.46','1997-02-26 14:01:53'), + (499,'Evelyn','ethomasdu@hhs.gov','6.241.88.250','2007-09-14 13:03:34'), + (500,'Paula','pshawdv@networksolutions.com','123.27.47.249','2003-10-30 21:19:20'); + +create table {schema}.seed_config_expected_1 as ( + + select *, 'default'::text as c1, 'default'::text as c2, 'was true'::text as some_bool from {schema}.seed + +); + +create table {schema}.seed_config_expected_2 as ( + + select *, 'abc'::text as c1, 'def'::text as c2, 'was true'::text as some_bool from {schema}.seed + +); + +create table {schema}.seed_config_expected_3 as ( + + select *, 'ghi'::text as c1, 'jkl'::text as c2, 'was true'::text as some_bool from {schema}.seed + +); + +create table {schema}.seed_summary ( + year timestamp without time zone, + count bigint +); + +INSERT INTO {schema}.seed_summary + ("year","count") +VALUES + ('1970-01-01 00:00:00',10), + ('1971-01-01 00:00:00',6), + ('1972-01-01 00:00:00',9), + ('1973-01-01 00:00:00',12), + ('1974-01-01 00:00:00',8), + ('1975-01-01 00:00:00',5), + ('1976-01-01 00:00:00',11), + ('1977-01-01 00:00:00',13), + ('1978-01-01 00:00:00',11), + ('1979-01-01 00:00:00',13), + ('1980-01-01 00:00:00',9), + ('1981-01-01 00:00:00',3), + ('1982-01-01 00:00:00',9), + ('1983-01-01 00:00:00',15), + ('1984-01-01 00:00:00',13), + ('1985-01-01 00:00:00',11), + ('1986-01-01 00:00:00',5), + ('1987-01-01 00:00:00',14), + ('1988-01-01 00:00:00',9), + ('1989-01-01 00:00:00',10), + ('1990-01-01 00:00:00',12), + ('1991-01-01 00:00:00',16), + ('1992-01-01 00:00:00',15), + ('1993-01-01 00:00:00',11), + ('1994-01-01 00:00:00',10), + ('1995-01-01 00:00:00',10), + ('1996-01-01 00:00:00',6), + ('1997-01-01 00:00:00',11), + ('1998-01-01 00:00:00',12), + ('1999-01-01 00:00:00',9), + ('2000-01-01 00:00:00',13), + ('2001-01-01 00:00:00',14), + ('2002-01-01 00:00:00',9), + ('2003-01-01 00:00:00',8), + ('2004-01-01 00:00:00',9), + ('2005-01-01 00:00:00',14), + ('2006-01-01 00:00:00',9), + ('2007-01-01 00:00:00',16), + ('2008-01-01 00:00:00',6), + ('2009-01-01 00:00:00',15), + ('2010-01-01 00:00:00',13), + ('2011-01-01 00:00:00',23), + ('2012-01-01 00:00:00',9), + ('2013-01-01 00:00:00',10), + ('2014-01-01 00:00:00',9), + ('2015-01-01 00:00:00',10), + ('2016-01-01 00:00:00',5); + diff --git a/test/integration/029_catalog_generate_tests/test_catalog_generate.py b/test/integration/029_catalog_generate_tests/test_catalog_generate.py new file mode 100644 index 00000000000..8f0b7732ac6 --- /dev/null +++ b/test/integration/029_catalog_generate_tests/test_catalog_generate.py @@ -0,0 +1,73 @@ +import json +import os + +from nose.plugins.attrib import attr +from test.integration.base import DBTIntegrationTest + +class TestCatalogGenerate(DBTIntegrationTest): + + def setUp(self): + super(TestCatalogGenerate, self).setUp() + self.run_sql_file("test/integration/029_catalog_generate_tests/seed.sql") + + @property + def schema(self): + return "simple_dependency_029" + + @property + def models(self): + return "test/integration/029_catalog_generate_tests/models" + + @property + def project_config(self): + return { + "repositories": [ + 'https://github.com/fishtown-analytics/dbt-integration-project' + ] + } + + @attr(type='postgres') + @attr(type='catalog') + def test_simple_generate(self): + self.run_dbt(["catalog", "generate"]) + self.assertTrue(os.path.exists('./target/catalog.json')) + + with open('./target/catalog.json') as fp: + data = json.load(fp) + + my_schema_name = self.unique_schema() + self.assertIn(my_schema_name, data) + my_schema = data[my_schema_name] + expected_tables = { + 'seed', 'seed_config_expected_1', 'seed_config_expected_2', + 'seed_config_expected_3', 'seed_summary' + } + self.assertEqual(set(my_schema), expected_tables) + expected_summary = { + 'metadata': { + 'schema': my_schema_name, + 'name': 'seed_summary', + 'type': 'BASE TABLE', + 'comment': None, + }, + 'columns': [ + { + 'name': 'year', + 'index': 1, + 'type': 'timestamp without time zone', + 'comment': None, + }, + { + 'name': 'count', + 'index': 2, + 'type': 'bigint', + 'comment': None + }, + ], + } + self.assertEqual(expected_summary, my_schema_name) + + + + + From 612d88b672227297df6088595d17be7638d76eb0 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 10 May 2018 10:29:35 -0600 Subject: [PATCH 08/43] Cleanup the tests, verify the entire schema --- .../models/disabled_one.sql | 3 - .../models/disabled_two.sql | 3 - .../models/empty.sql | 3 - .../029_catalog_generate_tests/seed.sql | 560 +----------------- .../test_catalog_generate.py | 141 ++++- 5 files changed, 118 insertions(+), 592 deletions(-) delete mode 100644 test/integration/029_catalog_generate_tests/models/disabled_one.sql delete mode 100644 test/integration/029_catalog_generate_tests/models/disabled_two.sql delete mode 100644 test/integration/029_catalog_generate_tests/models/empty.sql diff --git a/test/integration/029_catalog_generate_tests/models/disabled_one.sql b/test/integration/029_catalog_generate_tests/models/disabled_one.sql deleted file mode 100644 index 6f373105aac..00000000000 --- a/test/integration/029_catalog_generate_tests/models/disabled_one.sql +++ /dev/null @@ -1,3 +0,0 @@ -{{config(enabled=False)}} - -select 1 diff --git a/test/integration/029_catalog_generate_tests/models/disabled_two.sql b/test/integration/029_catalog_generate_tests/models/disabled_two.sql deleted file mode 100644 index b20d379edea..00000000000 --- a/test/integration/029_catalog_generate_tests/models/disabled_two.sql +++ /dev/null @@ -1,3 +0,0 @@ -{{config(enabled=False)}} - -select * from {{ref('disabled_one')}} diff --git a/test/integration/029_catalog_generate_tests/models/empty.sql b/test/integration/029_catalog_generate_tests/models/empty.sql deleted file mode 100644 index b28b04f6431..00000000000 --- a/test/integration/029_catalog_generate_tests/models/empty.sql +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/test/integration/029_catalog_generate_tests/seed.sql b/test/integration/029_catalog_generate_tests/seed.sql index 5bb6f00beaa..5af712a2c0f 100644 --- a/test/integration/029_catalog_generate_tests/seed.sql +++ b/test/integration/029_catalog_generate_tests/seed.sql @@ -10,506 +10,7 @@ create table {schema}.seed ( INSERT INTO {schema}.seed ("id","first_name","email","ip_address","updated_at") VALUES - (1,'Larry','lking0@miitbeian.gov.cn','69.135.206.194','2008-09-12 19:08:31'), - (2,'Larry','lperkins1@toplist.cz','64.210.133.162','1978-05-09 04:15:14'), - (3,'Anna','amontgomery2@miitbeian.gov.cn','168.104.64.114','2011-10-16 04:07:57'), - (4,'Sandra','sgeorge3@livejournal.com','229.235.252.98','1973-07-19 10:52:43'), - (5,'Fred','fwoods4@google.cn','78.229.170.124','2012-09-30 16:38:29'), - (6,'Stephen','shanson5@livejournal.com','182.227.157.105','1995-11-07 21:40:50'), - (7,'William','wmartinez6@upenn.edu','135.139.249.50','1982-09-05 03:11:59'), - (8,'Jessica','jlong7@hao123.com','203.62.178.210','1991-10-16 11:03:15'), - (9,'Douglas','dwhite8@tamu.edu','178.187.247.1','1979-10-01 09:49:48'), - (10,'Lisa','lcoleman9@nydailynews.com','168.234.128.249','2011-05-26 07:45:49'), - (11,'Ralph','rfieldsa@home.pl','55.152.163.149','1972-11-18 19:06:11'), - (12,'Louise','lnicholsb@samsung.com','141.116.153.154','2014-11-25 20:56:14'), - (13,'Clarence','cduncanc@sfgate.com','81.171.31.133','2011-11-17 07:02:36'), - (14,'Daniel','dfranklind@omniture.com','8.204.211.37','1980-09-13 00:09:04'), - (15,'Katherine','klanee@auda.org.au','176.96.134.59','1997-08-22 19:36:56'), - (16,'Billy','bwardf@wikia.com','214.108.78.85','2003-10-19 02:14:47'), - (17,'Annie','agarzag@ocn.ne.jp','190.108.42.70','1988-10-28 15:12:35'), - (18,'Shirley','scolemanh@fastcompany.com','109.251.164.84','1988-08-24 10:50:57'), - (19,'Roger','rfrazieri@scribd.com','38.145.218.108','1985-12-31 15:17:15'), - (20,'Lillian','lstanleyj@goodreads.com','47.57.236.17','1970-06-08 02:09:05'), - (21,'Aaron','arodriguezk@nps.gov','205.245.118.221','1985-10-11 23:07:49'), - (22,'Patrick','pparkerl@techcrunch.com','19.8.100.182','2006-03-29 12:53:56'), - (23,'Phillip','pmorenom@intel.com','41.38.254.103','2011-11-07 15:35:43'), - (24,'Henry','hgarcian@newsvine.com','1.191.216.252','2008-08-28 08:30:44'), - (25,'Irene','iturnero@opera.com','50.17.60.190','1994-04-01 07:15:02'), - (26,'Andrew','adunnp@pen.io','123.52.253.176','2000-11-01 06:03:25'), - (27,'David','dgutierrezq@wp.com','238.23.203.42','1988-01-25 07:29:18'), - (28,'Henry','hsanchezr@cyberchimps.com','248.102.2.185','1983-01-01 13:36:37'), - (29,'Evelyn','epetersons@gizmodo.com','32.80.46.119','1979-07-16 17:24:12'), - (30,'Tammy','tmitchellt@purevolume.com','249.246.167.88','2001-04-03 10:00:23'), - (31,'Jacqueline','jlittleu@domainmarket.com','127.181.97.47','1986-02-11 21:35:50'), - (32,'Earl','eortizv@opera.com','166.47.248.240','1996-07-06 08:16:27'), - (33,'Juan','jgordonw@sciencedirect.com','71.77.2.200','1987-01-31 03:46:44'), - (34,'Diane','dhowellx@nyu.edu','140.94.133.12','1994-06-11 02:30:05'), - (35,'Randy','rkennedyy@microsoft.com','73.255.34.196','2005-05-26 20:28:39'), - (36,'Janice','jriveraz@time.com','22.214.227.32','1990-02-09 04:16:52'), - (37,'Laura','lperry10@diigo.com','159.148.145.73','2015-03-17 05:59:25'), - (38,'Gary','gray11@statcounter.com','40.193.124.56','1970-01-27 10:04:51'), - (39,'Jesse','jmcdonald12@typepad.com','31.7.86.103','2009-03-14 08:14:29'), - (40,'Sandra','sgonzalez13@goodreads.com','223.80.168.239','1993-05-21 14:08:54'), - (41,'Scott','smoore14@archive.org','38.238.46.83','1980-08-30 11:16:56'), - (42,'Phillip','pevans15@cisco.com','158.234.59.34','2011-12-15 23:26:31'), - (43,'Steven','sriley16@google.ca','90.247.57.68','2011-10-29 19:03:28'), - (44,'Deborah','dbrown17@hexun.com','179.125.143.240','1995-04-10 14:36:07'), - (45,'Lori','lross18@ow.ly','64.80.162.180','1980-12-27 16:49:15'), - (46,'Sean','sjackson19@tumblr.com','240.116.183.69','1988-06-12 21:24:45'), - (47,'Terry','tbarnes1a@163.com','118.38.213.137','1997-09-22 16:43:19'), - (48,'Dorothy','dross1b@ebay.com','116.81.76.49','2005-02-28 13:33:24'), - (49,'Samuel','swashington1c@house.gov','38.191.253.40','1989-01-19 21:15:48'), - (50,'Ralph','rcarter1d@tinyurl.com','104.84.60.174','2007-08-11 10:21:49'), - (51,'Wayne','whudson1e@princeton.edu','90.61.24.102','1983-07-03 16:58:12'), - (52,'Rose','rjames1f@plala.or.jp','240.83.81.10','1995-06-08 11:46:23'), - (53,'Louise','lcox1g@theglobeandmail.com','105.11.82.145','2016-09-19 14:45:51'), - (54,'Kenneth','kjohnson1h@independent.co.uk','139.5.45.94','1976-08-17 11:26:19'), - (55,'Donna','dbrown1i@amazon.co.uk','19.45.169.45','2006-05-27 16:51:40'), - (56,'Johnny','jvasquez1j@trellian.com','118.202.238.23','1975-11-17 08:42:32'), - (57,'Patrick','pramirez1k@tamu.edu','231.25.153.198','1997-08-06 11:51:09'), - (58,'Helen','hlarson1l@prweb.com','8.40.21.39','1993-08-04 19:53:40'), - (59,'Patricia','pspencer1m@gmpg.org','212.198.40.15','1977-08-03 16:37:27'), - (60,'Joseph','jspencer1n@marriott.com','13.15.63.238','2005-07-23 20:22:06'), - (61,'Phillip','pschmidt1o@blogtalkradio.com','177.98.201.190','1976-05-19 21:47:44'), - (62,'Joan','jwebb1p@google.ru','105.229.170.71','1972-09-07 17:53:47'), - (63,'Phyllis','pkennedy1q@imgur.com','35.145.8.244','2000-01-01 22:33:37'), - (64,'Katherine','khunter1r@smh.com.au','248.168.205.32','1991-01-09 06:40:24'), - (65,'Laura','lvasquez1s@wiley.com','128.129.115.152','1997-10-23 12:04:56'), - (66,'Juan','jdunn1t@state.gov','44.228.124.51','2004-11-10 05:07:35'), - (67,'Judith','jholmes1u@wiley.com','40.227.179.115','1977-08-02 17:01:45'), - (68,'Beverly','bbaker1v@wufoo.com','208.34.84.59','2016-03-06 20:07:23'), - (69,'Lawrence','lcarr1w@flickr.com','59.158.212.223','1988-09-13 06:07:21'), - (70,'Gloria','gwilliams1x@mtv.com','245.231.88.33','1995-03-18 22:32:46'), - (71,'Steven','ssims1y@cbslocal.com','104.50.58.255','2001-08-05 21:26:20'), - (72,'Betty','bmills1z@arstechnica.com','103.177.214.220','1981-12-14 21:26:54'), - (73,'Mildred','mfuller20@prnewswire.com','151.158.8.130','2000-04-19 10:13:55'), - (74,'Donald','dday21@icq.com','9.178.102.255','1972-12-03 00:58:24'), - (75,'Eric','ethomas22@addtoany.com','85.2.241.227','1992-11-01 05:59:30'), - (76,'Joyce','jarmstrong23@sitemeter.com','169.224.20.36','1985-10-24 06:50:01'), - (77,'Maria','mmartinez24@amazonaws.com','143.189.167.135','2005-10-05 05:17:42'), - (78,'Harry','hburton25@youtube.com','156.47.176.237','1978-03-26 05:53:33'), - (79,'Kevin','klawrence26@hao123.com','79.136.183.83','1994-10-12 04:38:52'), - (80,'David','dhall27@prweb.com','133.149.172.153','1976-12-15 16:24:24'), - (81,'Kathy','kperry28@twitter.com','229.242.72.228','1979-03-04 02:58:56'), - (82,'Adam','aprice29@elegantthemes.com','13.145.21.10','1982-11-07 11:46:59'), - (83,'Brandon','bgriffin2a@va.gov','73.249.128.212','2013-10-30 05:30:36'), - (84,'Henry','hnguyen2b@discovery.com','211.36.214.242','1985-01-09 06:37:27'), - (85,'Eric','esanchez2c@edublogs.org','191.166.188.251','2004-05-01 23:21:42'), - (86,'Jason','jlee2d@jimdo.com','193.92.16.182','1973-01-08 09:05:39'), - (87,'Diana','drichards2e@istockphoto.com','19.130.175.245','1994-10-05 22:50:49'), - (88,'Andrea','awelch2f@abc.net.au','94.155.233.96','2002-04-26 08:41:44'), - (89,'Louis','lwagner2g@miitbeian.gov.cn','26.217.34.111','2003-08-25 07:56:39'), - (90,'Jane','jsims2h@seesaa.net','43.4.220.135','1987-03-20 20:39:04'), - (91,'Larry','lgrant2i@si.edu','97.126.79.34','2000-09-07 20:26:19'), - (92,'Louis','ldean2j@prnewswire.com','37.148.40.127','2011-09-16 20:12:14'), - (93,'Jennifer','jcampbell2k@xing.com','38.106.254.142','1988-07-15 05:06:49'), - (94,'Wayne','wcunningham2l@google.com.hk','223.28.26.187','2009-12-15 06:16:54'), - (95,'Lori','lstevens2m@icq.com','181.250.181.58','1984-10-28 03:29:19'), - (96,'Judy','jsimpson2n@marriott.com','180.121.239.219','1986-02-07 15:18:10'), - (97,'Phillip','phoward2o@usa.gov','255.247.0.175','2002-12-26 08:44:45'), - (98,'Gloria','gwalker2p@usa.gov','156.140.7.128','1997-10-04 07:58:58'), - (99,'Paul','pjohnson2q@umn.edu','183.59.198.197','1991-11-14 12:33:55'), - (100,'Frank','fgreene2r@blogspot.com','150.143.68.121','2010-06-12 23:55:39'), - (101,'Deborah','dknight2s@reverbnation.com','222.131.211.191','1970-07-08 08:54:23'), - (102,'Sandra','sblack2t@tripadvisor.com','254.183.128.254','2000-04-12 02:39:36'), - (103,'Edward','eburns2u@dailymotion.com','253.89.118.18','1993-10-10 10:54:01'), - (104,'Anthony','ayoung2v@ustream.tv','118.4.193.176','1978-08-26 17:07:29'), - (105,'Donald','dlawrence2w@wp.com','139.200.159.227','2007-07-21 20:56:20'), - (106,'Matthew','mfreeman2x@google.fr','205.26.239.92','2014-12-05 17:05:39'), - (107,'Sean','ssanders2y@trellian.com','143.89.82.108','1993-07-14 21:45:02'), - (108,'Sharon','srobinson2z@soundcloud.com','66.234.247.54','1977-04-06 19:07:03'), - (109,'Jennifer','jwatson30@t-online.de','196.102.127.7','1998-03-07 05:12:23'), - (110,'Clarence','cbrooks31@si.edu','218.93.234.73','2002-11-06 17:22:25'), - (111,'Jose','jflores32@goo.gl','185.105.244.231','1995-01-05 06:32:21'), - (112,'George','glee33@adobe.com','173.82.249.196','2015-01-04 02:47:46'), - (113,'Larry','lhill34@linkedin.com','66.5.206.195','2010-11-02 10:21:17'), - (114,'Marie','mmeyer35@mysql.com','151.152.88.107','1990-05-22 20:52:51'), - (115,'Clarence','cwebb36@skype.com','130.198.55.217','1972-10-27 07:38:54'), - (116,'Sarah','scarter37@answers.com','80.89.18.153','1971-08-24 19:29:30'), - (117,'Henry','hhughes38@webeden.co.uk','152.60.114.174','1973-01-27 09:00:42'), - (118,'Teresa','thenry39@hao123.com','32.187.239.106','2015-11-06 01:48:44'), - (119,'Billy','bgutierrez3a@sun.com','52.37.70.134','2002-03-19 03:20:19'), - (120,'Anthony','agibson3b@github.io','154.251.232.213','1991-04-19 01:08:15'), - (121,'Sandra','sromero3c@wikia.com','44.124.171.2','1998-09-06 20:30:34'), - (122,'Paula','pandrews3d@blogs.com','153.142.118.226','2003-06-24 16:31:24'), - (123,'Terry','tbaker3e@csmonitor.com','99.120.45.219','1970-12-09 23:57:21'), - (124,'Lois','lwilson3f@reuters.com','147.44.171.83','1971-01-09 22:28:51'), - (125,'Sara','smorgan3g@nature.com','197.67.192.230','1992-01-28 20:33:24'), - (126,'Charles','ctorres3h@china.com.cn','156.115.216.2','1993-10-02 19:36:34'), - (127,'Richard','ralexander3i@marriott.com','248.235.180.59','1999-02-03 18:40:55'), - (128,'Christina','charper3j@cocolog-nifty.com','152.114.116.129','1978-09-13 00:37:32'), - (129,'Steve','sadams3k@economist.com','112.248.91.98','2004-03-21 09:07:43'), - (130,'Katherine','krobertson3l@ow.ly','37.220.107.28','1977-03-18 19:28:50'), - (131,'Donna','dgibson3m@state.gov','222.218.76.221','1999-02-01 06:46:16'), - (132,'Christina','cwest3n@mlb.com','152.114.6.160','1979-12-24 15:30:35'), - (133,'Sandra','swillis3o@meetup.com','180.71.49.34','1984-09-27 08:05:54'), - (134,'Clarence','cedwards3p@smugmug.com','10.64.180.186','1979-04-16 16:52:10'), - (135,'Ruby','rjames3q@wp.com','98.61.54.20','2007-01-13 14:25:52'), - (136,'Sarah','smontgomery3r@tripod.com','91.45.164.172','2009-07-25 04:34:30'), - (137,'Sarah','soliver3s@eventbrite.com','30.106.39.146','2012-05-09 22:12:33'), - (138,'Deborah','dwheeler3t@biblegateway.com','59.105.213.173','1999-11-09 08:08:44'), - (139,'Deborah','dray3u@i2i.jp','11.108.186.217','2014-02-04 03:15:19'), - (140,'Paul','parmstrong3v@alexa.com','6.250.59.43','2009-12-21 10:08:53'), - (141,'Aaron','abishop3w@opera.com','207.145.249.62','1996-04-25 23:20:23'), - (142,'Henry','hsanders3x@google.ru','140.215.203.171','2012-01-29 11:52:32'), - (143,'Anne','aanderson3y@1688.com','74.150.102.118','1982-04-03 13:46:17'), - (144,'Victor','vmurphy3z@hugedomains.com','222.155.99.152','1987-11-03 19:58:41'), - (145,'Evelyn','ereid40@pbs.org','249.122.33.117','1977-12-14 17:09:57'), - (146,'Brian','bgonzalez41@wikia.com','246.254.235.141','1991-02-24 00:45:58'), - (147,'Sandra','sgray42@squarespace.com','150.73.28.159','1972-07-28 17:26:32'), - (148,'Alice','ajones43@a8.net','78.253.12.177','2002-12-05 16:57:46'), - (149,'Jessica','jhanson44@mapquest.com','87.229.30.160','1994-01-30 11:40:04'), - (150,'Louise','lbailey45@reuters.com','191.219.31.101','2011-09-07 21:11:45'), - (151,'Christopher','cgonzalez46@printfriendly.com','83.137.213.239','1984-10-24 14:58:04'), - (152,'Gregory','gcollins47@yandex.ru','28.176.10.115','1998-07-25 17:17:10'), - (153,'Jane','jperkins48@usnews.com','46.53.164.159','1979-08-19 15:25:00'), - (154,'Phyllis','plong49@yahoo.co.jp','208.140.88.2','1985-07-06 02:16:36'), - (155,'Adam','acarter4a@scribd.com','78.48.148.204','2005-07-20 03:31:09'), - (156,'Frank','fweaver4b@angelfire.com','199.180.255.224','2011-03-04 23:07:54'), - (157,'Ronald','rmurphy4c@cloudflare.com','73.42.97.231','1991-01-11 10:39:41'), - (158,'Richard','rmorris4d@e-recht24.de','91.9.97.223','2009-01-17 21:05:15'), - (159,'Rose','rfoster4e@woothemes.com','203.169.53.16','1991-04-21 02:09:38'), - (160,'George','ggarrett4f@uiuc.edu','186.61.5.167','1989-11-11 11:29:42'), - (161,'Victor','vhamilton4g@biblegateway.com','121.229.138.38','2012-06-22 18:01:23'), - (162,'Mark','mbennett4h@businessinsider.com','209.184.29.203','1980-04-16 15:26:34'), - (163,'Martin','mwells4i@ifeng.com','97.223.55.105','2010-05-26 14:08:18'), - (164,'Diana','dstone4j@google.ru','90.155.52.47','2013-02-11 00:14:54'), - (165,'Walter','wferguson4k@blogger.com','30.63.212.44','1986-02-20 17:46:46'), - (166,'Denise','dcoleman4l@vistaprint.com','10.209.153.77','1992-05-13 20:14:14'), - (167,'Philip','pknight4m@xing.com','15.28.135.167','2000-09-11 18:41:13'), - (168,'Russell','rcarr4n@youtube.com','113.55.165.50','2008-07-10 17:49:27'), - (169,'Donna','dburke4o@dion.ne.jp','70.0.105.111','1992-02-10 17:24:58'), - (170,'Anne','along4p@squidoo.com','36.154.58.107','2012-08-19 23:35:31'), - (171,'Clarence','cbanks4q@webeden.co.uk','94.57.53.114','1972-03-11 21:46:44'), - (172,'Betty','bbowman4r@cyberchimps.com','178.115.209.69','2013-01-13 21:34:51'), - (173,'Andrew','ahudson4s@nytimes.com','84.32.252.144','1998-09-15 14:20:04'), - (174,'Keith','kgordon4t@cam.ac.uk','189.237.211.102','2009-01-22 05:34:38'), - (175,'Patrick','pwheeler4u@mysql.com','47.22.117.226','1984-09-05 22:33:15'), - (176,'Jesse','jfoster4v@mapquest.com','229.95.131.46','1990-01-20 12:19:15'), - (177,'Arthur','afisher4w@jugem.jp','107.255.244.98','1983-10-13 11:08:46'), - (178,'Nicole','nryan4x@wsj.com','243.211.33.221','1974-05-30 23:19:14'), - (179,'Bruce','bjohnson4y@sfgate.com','17.41.200.101','1992-09-23 02:02:19'), - (180,'Terry','tcox4z@reference.com','20.189.120.106','1982-02-13 12:43:14'), - (181,'Ashley','astanley50@kickstarter.com','86.3.56.98','1976-05-09 01:27:16'), - (182,'Michael','mrivera51@about.me','72.118.249.0','1971-11-11 17:28:37'), - (183,'Steven','sgonzalez52@mozilla.org','169.112.247.47','2002-08-24 14:59:25'), - (184,'Kathleen','kfuller53@bloglovin.com','80.93.59.30','2002-03-11 13:41:29'), - (185,'Nicole','nhenderson54@usda.gov','39.253.60.30','1995-04-24 05:55:07'), - (186,'Ralph','rharper55@purevolume.com','167.147.142.189','1980-02-10 18:35:45'), - (187,'Heather','hcunningham56@photobucket.com','96.222.196.229','2007-06-15 05:37:50'), - (188,'Nancy','nlittle57@cbc.ca','241.53.255.175','2007-07-12 23:42:48'), - (189,'Juan','jramirez58@pinterest.com','190.128.84.27','1978-11-07 23:37:37'), - (190,'Beverly','bfowler59@chronoengine.com','54.144.230.49','1979-03-31 23:27:28'), - (191,'Shirley','sstevens5a@prlog.org','200.97.231.248','2011-12-06 07:08:50'), - (192,'Annie','areyes5b@squidoo.com','223.32.182.101','2011-05-28 02:42:09'), - (193,'Jack','jkelley5c@tiny.cc','47.34.118.150','1981-12-05 17:31:40'), - (194,'Keith','krobinson5d@1und1.de','170.210.209.31','1999-03-09 11:05:43'), - (195,'Joseph','jmiller5e@google.com.au','136.74.212.139','1984-10-08 13:18:20'), - (196,'Annie','aday5f@blogspot.com','71.99.186.69','1986-02-18 12:27:34'), - (197,'Nancy','nperez5g@liveinternet.ru','28.160.6.107','1983-10-20 17:51:20'), - (198,'Tammy','tward5h@ucoz.ru','141.43.164.70','1980-03-31 04:45:29'), - (199,'Doris','dryan5i@ted.com','239.117.202.188','1985-07-03 03:17:53'), - (200,'Rose','rmendoza5j@photobucket.com','150.200.206.79','1973-04-21 21:36:40'), - (201,'Cynthia','cbutler5k@hubpages.com','80.153.174.161','2001-01-20 01:42:26'), - (202,'Samuel','soliver5l@people.com.cn','86.127.246.140','1970-09-02 02:19:00'), - (203,'Carl','csanchez5m@mysql.com','50.149.237.107','1993-12-01 07:02:09'), - (204,'Kathryn','kowens5n@geocities.jp','145.166.205.201','2004-07-06 18:39:33'), - (205,'Nicholas','nnichols5o@parallels.com','190.240.66.170','2014-11-11 18:52:19'), - (206,'Keith','kwillis5p@youtube.com','181.43.206.100','1998-06-13 06:30:51'), - (207,'Justin','jwebb5q@intel.com','211.54.245.74','2000-11-04 16:58:26'), - (208,'Gary','ghicks5r@wikipedia.org','196.154.213.104','1992-12-01 19:48:28'), - (209,'Martin','mpowell5s@flickr.com','153.67.12.241','1983-06-30 06:24:32'), - (210,'Brenda','bkelley5t@xinhuanet.com','113.100.5.172','2005-01-08 20:50:22'), - (211,'Edward','eray5u@a8.net','205.187.246.65','2011-09-26 08:04:44'), - (212,'Steven','slawson5v@senate.gov','238.150.250.36','1978-11-22 02:48:09'), - (213,'Robert','rthompson5w@furl.net','70.7.89.236','2001-09-12 08:52:07'), - (214,'Jack','jporter5x@diigo.com','220.172.29.99','1976-07-26 14:29:21'), - (215,'Lisa','ljenkins5y@oakley.com','150.151.170.180','2010-03-20 19:21:16'), - (216,'Theresa','tbell5z@mayoclinic.com','247.25.53.173','2001-03-11 05:36:40'), - (217,'Jimmy','jstephens60@weather.com','145.101.93.235','1983-04-12 09:35:30'), - (218,'Louis','lhunt61@amazon.co.jp','78.137.6.253','1997-08-29 19:34:34'), - (219,'Lawrence','lgilbert62@ted.com','243.132.8.78','2015-04-08 22:06:56'), - (220,'David','dgardner63@4shared.com','204.40.46.136','1971-07-09 03:29:11'), - (221,'Charles','ckennedy64@gmpg.org','211.83.233.2','2011-02-26 11:55:04'), - (222,'Lillian','lbanks65@msu.edu','124.233.12.80','2010-05-16 20:29:02'), - (223,'Ernest','enguyen66@baidu.com','82.45.128.148','1996-07-04 10:07:04'), - (224,'Ryan','rrussell67@cloudflare.com','202.53.240.223','1983-08-05 12:36:29'), - (225,'Donald','ddavis68@ustream.tv','47.39.218.137','1989-05-27 02:30:56'), - (226,'Joe','jscott69@blogspot.com','140.23.131.75','1973-03-16 12:21:31'), - (227,'Anne','amarshall6a@google.ca','113.162.200.197','1988-12-09 03:38:29'), - (228,'Willie','wturner6b@constantcontact.com','85.83.182.249','1991-10-06 01:51:10'), - (229,'Nicole','nwilson6c@sogou.com','30.223.51.135','1977-05-29 19:54:56'), - (230,'Janet','jwheeler6d@stumbleupon.com','153.194.27.144','2011-03-13 12:48:47'), - (231,'Lois','lcarr6e@statcounter.com','0.41.36.53','1993-02-06 04:52:01'), - (232,'Shirley','scruz6f@tmall.com','37.156.39.223','2007-02-18 17:47:01'), - (233,'Patrick','pford6g@reverbnation.com','36.198.200.89','1977-03-06 15:47:24'), - (234,'Lisa','lhudson6h@usatoday.com','134.213.58.137','2014-10-28 01:56:56'), - (235,'Pamela','pmartinez6i@opensource.org','5.151.127.202','1987-11-30 16:44:47'), - (236,'Larry','lperez6j@infoseek.co.jp','235.122.96.148','1979-01-18 06:33:45'), - (237,'Pamela','pramirez6k@census.gov','138.233.34.163','2012-01-29 10:35:20'), - (238,'Daniel','dcarr6l@php.net','146.21.152.242','1984-11-17 08:22:59'), - (239,'Patrick','psmith6m@indiegogo.com','136.222.199.36','2001-05-30 22:16:44'), - (240,'Raymond','rhenderson6n@hc360.com','116.31.112.38','2000-01-05 20:35:41'), - (241,'Teresa','treynolds6o@miitbeian.gov.cn','198.126.205.220','1996-11-08 01:27:31'), - (242,'Johnny','jmason6p@flickr.com','192.8.232.114','2013-05-14 05:35:50'), - (243,'Angela','akelly6q@guardian.co.uk','234.116.60.197','1977-08-20 02:05:17'), - (244,'Douglas','dcole6r@cmu.edu','128.135.212.69','2016-10-26 17:40:36'), - (245,'Frances','fcampbell6s@twitpic.com','94.22.243.235','1987-04-26 07:07:13'), - (246,'Donna','dgreen6t@chron.com','227.116.46.107','2011-07-25 12:59:54'), - (247,'Benjamin','bfranklin6u@redcross.org','89.141.142.89','1974-05-03 20:28:18'), - (248,'Randy','rpalmer6v@rambler.ru','70.173.63.178','2011-12-20 17:40:18'), - (249,'Melissa','mmurray6w@bbb.org','114.234.118.137','1991-02-26 12:45:44'), - (250,'Jean','jlittle6x@epa.gov','141.21.163.254','1991-08-16 04:57:09'), - (251,'Daniel','dolson6y@nature.com','125.75.104.97','2010-04-23 06:25:54'), - (252,'Kathryn','kwells6z@eventbrite.com','225.104.28.249','2015-01-31 02:21:50'), - (253,'Theresa','tgonzalez70@ox.ac.uk','91.93.156.26','1971-12-11 10:31:31'), - (254,'Beverly','broberts71@bluehost.com','244.40.158.89','2013-09-21 13:02:31'), - (255,'Pamela','pmurray72@netscape.com','218.54.95.216','1985-04-16 00:34:00'), - (256,'Timothy','trichardson73@amazonaws.com','235.49.24.229','2000-11-11 09:48:28'), - (257,'Mildred','mpalmer74@is.gd','234.125.95.132','1992-05-25 02:25:02'), - (258,'Jessica','jcampbell75@google.it','55.98.30.140','2014-08-26 00:26:34'), - (259,'Beverly','bthomas76@cpanel.net','48.78.228.176','1970-08-18 10:40:05'), - (260,'Eugene','eward77@cargocollective.com','139.226.204.2','1996-12-04 23:17:00'), - (261,'Andrea','aallen78@webnode.com','160.31.214.38','2009-07-06 07:22:37'), - (262,'Justin','jruiz79@merriam-webster.com','150.149.246.122','2005-06-06 11:44:19'), - (263,'Kenneth','kedwards7a@networksolutions.com','98.82.193.128','2001-07-03 02:00:10'), - (264,'Rachel','rday7b@miibeian.gov.cn','114.15.247.221','1994-08-18 19:45:40'), - (265,'Russell','rmiller7c@instagram.com','184.130.152.253','1977-11-06 01:58:12'), - (266,'Bonnie','bhudson7d@cornell.edu','235.180.186.206','1990-12-03 22:45:24'), - (267,'Raymond','rknight7e@yandex.ru','161.2.44.252','1995-08-25 04:31:19'), - (268,'Bonnie','brussell7f@elpais.com','199.237.57.207','1991-03-29 08:32:06'), - (269,'Marie','mhenderson7g@elpais.com','52.203.131.144','2004-06-04 21:50:28'), - (270,'Alan','acarr7h@trellian.com','147.51.205.72','2005-03-03 10:51:31'), - (271,'Barbara','bturner7i@hugedomains.com','103.160.110.226','2004-08-04 13:42:40'), - (272,'Christina','cdaniels7j@census.gov','0.238.61.251','1972-10-18 12:47:33'), - (273,'Jeremy','jgomez7k@reuters.com','111.26.65.56','2013-01-13 10:41:35'), - (274,'Laura','lwood7l@icio.us','149.153.38.205','2011-06-25 09:33:59'), - (275,'Matthew','mbowman7m@auda.org.au','182.138.206.172','1999-03-05 03:25:36'), - (276,'Denise','dparker7n@icq.com','0.213.88.138','2011-11-04 09:43:06'), - (277,'Phillip','pparker7o@discuz.net','219.242.165.240','1973-10-19 04:22:29'), - (278,'Joan','jpierce7p@salon.com','63.31.213.202','1989-04-09 22:06:24'), - (279,'Irene','ibaker7q@cbc.ca','102.33.235.114','1992-09-04 13:00:57'), - (280,'Betty','bbowman7r@ted.com','170.91.249.242','2015-09-28 08:14:22'), - (281,'Teresa','truiz7s@boston.com','82.108.158.207','1999-07-18 05:17:09'), - (282,'Helen','hbrooks7t@slideshare.net','102.87.162.187','2003-01-06 15:45:29'), - (283,'Karen','kgriffin7u@wunderground.com','43.82.44.184','2010-05-28 01:56:37'), - (284,'Lisa','lfernandez7v@mtv.com','200.238.218.220','1993-04-03 20:33:51'), - (285,'Jesse','jlawrence7w@timesonline.co.uk','95.122.105.78','1990-01-05 17:28:43'), - (286,'Terry','tross7x@macromedia.com','29.112.114.133','2009-08-29 21:32:17'), - (287,'Angela','abradley7y@icq.com','177.44.27.72','1989-10-04 21:46:06'), - (288,'Maria','mhart7z@dailymotion.com','55.27.55.202','1975-01-21 01:22:57'), - (289,'Raymond','randrews80@pinterest.com','88.90.78.67','1992-03-16 21:37:40'), - (290,'Kathy','krice81@bluehost.com','212.63.196.102','2000-12-14 03:06:44'), - (291,'Cynthia','cramos82@nymag.com','107.89.190.6','2005-06-28 02:02:33'), - (292,'Kimberly','kjones83@mysql.com','86.169.101.101','2007-06-13 22:56:49'), - (293,'Timothy','thansen84@microsoft.com','108.100.254.90','2003-04-04 10:31:57'), - (294,'Carol','cspencer85@berkeley.edu','75.118.144.187','1999-03-30 14:53:21'), - (295,'Louis','lmedina86@latimes.com','141.147.163.24','1991-04-11 17:53:13'), - (296,'Margaret','mcole87@google.fr','53.184.26.83','1991-12-19 01:54:10'), - (297,'Mary','mgomez88@yellowpages.com','208.56.57.99','1976-05-21 18:05:08'), - (298,'Amanda','aanderson89@geocities.com','147.73.15.252','1987-08-22 15:05:28'), - (299,'Kathryn','kgarrett8a@nature.com','27.29.177.220','1976-07-15 04:25:04'), - (300,'Dorothy','dmason8b@shareasale.com','106.210.99.193','1990-09-03 21:39:31'), - (301,'Lois','lkennedy8c@amazon.de','194.169.29.187','2007-07-29 14:09:31'), - (302,'Irene','iburton8d@washingtonpost.com','196.143.110.249','2013-09-05 11:32:46'), - (303,'Betty','belliott8e@wired.com','183.105.222.199','1979-09-19 19:29:13'), - (304,'Bobby','bmeyer8f@census.gov','36.13.161.145','2014-05-24 14:34:39'), - (305,'Ann','amorrison8g@sfgate.com','72.154.54.137','1978-10-05 14:22:34'), - (306,'Daniel','djackson8h@wunderground.com','144.95.32.34','1990-07-27 13:23:05'), - (307,'Joe','jboyd8i@alibaba.com','187.105.86.178','2011-09-28 16:46:32'), - (308,'Ralph','rdunn8j@fc2.com','3.19.87.255','1984-10-18 08:00:40'), - (309,'Craig','ccarter8k@gizmodo.com','235.152.76.215','1998-07-04 12:15:21'), - (310,'Paula','pdean8l@hhs.gov','161.100.173.197','1973-02-13 09:38:55'), - (311,'Andrew','agarrett8m@behance.net','199.253.123.218','1991-02-14 13:36:32'), - (312,'Janet','jhowell8n@alexa.com','39.189.139.79','2012-11-24 20:17:33'), - (313,'Keith','khansen8o@godaddy.com','116.186.223.196','1987-08-23 21:22:05'), - (314,'Nicholas','nedwards8p@state.gov','142.175.142.11','1977-03-28 18:27:27'), - (315,'Jacqueline','jallen8q@oaic.gov.au','189.66.135.192','1994-10-26 11:44:26'), - (316,'Frank','fgardner8r@mapy.cz','154.77.119.169','1983-01-29 19:19:51'), - (317,'Eric','eharrison8s@google.cn','245.139.65.123','1984-02-04 09:54:36'), - (318,'Gregory','gcooper8t@go.com','171.147.0.221','2004-06-14 05:22:08'), - (319,'Jean','jfreeman8u@rakuten.co.jp','67.243.121.5','1977-01-07 18:23:43'), - (320,'Juan','jlewis8v@shinystat.com','216.181.171.189','2001-08-23 17:32:43'), - (321,'Randy','rwilliams8w@shinystat.com','105.152.146.28','1983-02-17 00:05:50'), - (322,'Stephen','shart8x@sciencedirect.com','196.131.205.148','2004-02-15 10:12:03'), - (323,'Annie','ahunter8y@example.com','63.36.34.103','2003-07-23 21:15:25'), - (324,'Melissa','mflores8z@cbc.ca','151.230.217.90','1983-11-02 14:53:56'), - (325,'Jane','jweaver90@about.me','0.167.235.217','1987-07-29 00:13:44'), - (326,'Anthony','asmith91@oracle.com','97.87.48.41','2001-05-31 18:44:11'), - (327,'Terry','tdavis92@buzzfeed.com','46.20.12.51','2015-09-12 23:13:55'), - (328,'Brandon','bmontgomery93@gravatar.com','252.101.48.186','2010-10-28 08:26:27'), - (329,'Chris','cmurray94@bluehost.com','25.158.167.97','2004-05-05 16:10:31'), - (330,'Denise','dfuller95@hugedomains.com','216.210.149.28','1979-04-20 08:57:24'), - (331,'Arthur','amcdonald96@sakura.ne.jp','206.42.36.213','2009-08-15 03:26:16'), - (332,'Jesse','jhoward97@google.cn','46.181.118.30','1974-04-18 14:08:41'), - (333,'Frank','fsimpson98@domainmarket.com','163.220.211.87','2006-06-30 14:46:52'), - (334,'Janice','jwoods99@pen.io','229.245.237.182','1988-04-06 11:52:58'), - (335,'Rebecca','rroberts9a@huffingtonpost.com','148.96.15.80','1976-10-05 08:44:16'), - (336,'Joshua','jray9b@opensource.org','192.253.12.198','1971-12-25 22:27:07'), - (337,'Joyce','jcarpenter9c@statcounter.com','125.171.46.215','2001-12-31 22:08:13'), - (338,'Andrea','awest9d@privacy.gov.au','79.101.180.201','1983-02-18 20:07:47'), - (339,'Christine','chudson9e@yelp.com','64.198.43.56','1997-09-08 08:03:43'), - (340,'Joe','jparker9f@earthlink.net','251.215.148.153','1973-11-04 05:08:18'), - (341,'Thomas','tkim9g@answers.com','49.187.34.47','1991-08-07 21:13:48'), - (342,'Janice','jdean9h@scientificamerican.com','4.197.117.16','2009-12-08 02:35:49'), - (343,'James','jmitchell9i@umich.edu','43.121.18.147','2011-04-28 17:04:09'), - (344,'Charles','cgardner9j@purevolume.com','197.78.240.240','1998-02-11 06:47:07'), - (345,'Robert','rhenderson9k@friendfeed.com','215.84.180.88','2002-05-10 15:33:14'), - (346,'Chris','cgray9l@4shared.com','249.70.192.240','1998-10-03 16:43:42'), - (347,'Gloria','ghayes9m@hibu.com','81.103.138.26','1999-12-26 11:23:13'), - (348,'Edward','eramirez9n@shareasale.com','38.136.90.136','2010-08-19 08:01:06'), - (349,'Cheryl','cbutler9o@google.ca','172.180.78.172','1995-05-27 20:03:52'), - (350,'Margaret','mwatkins9p@sfgate.com','3.20.198.6','2014-10-21 01:42:58'), - (351,'Rebecca','rwelch9q@examiner.com','45.81.42.208','2001-02-08 12:19:06'), - (352,'Joe','jpalmer9r@phpbb.com','163.202.92.190','1970-01-05 11:29:12'), - (353,'Sandra','slewis9s@dyndns.org','77.215.201.236','1974-01-05 07:04:04'), - (354,'Todd','tfranklin9t@g.co','167.125.181.82','2009-09-28 10:13:58'), - (355,'Joseph','jlewis9u@webmd.com','244.204.6.11','1990-10-21 15:49:57'), - (356,'Alan','aknight9v@nydailynews.com','152.197.95.83','1996-03-08 08:43:17'), - (357,'Sharon','sdean9w@123-reg.co.uk','237.46.40.26','1985-11-30 12:09:24'), - (358,'Annie','awright9x@cafepress.com','190.45.231.111','2000-08-24 11:56:06'), - (359,'Diane','dhamilton9y@youtube.com','85.146.171.196','2015-02-24 02:03:57'), - (360,'Antonio','alane9z@auda.org.au','61.63.146.203','2001-05-13 03:43:34'), - (361,'Matthew','mallena0@hhs.gov','29.97.32.19','1973-02-19 23:43:32'), - (362,'Bonnie','bfowlera1@soup.io','251.216.99.53','2013-08-01 15:35:41'), - (363,'Margaret','mgraya2@examiner.com','69.255.151.79','1998-01-23 22:24:59'), - (364,'Joan','jwagnera3@printfriendly.com','192.166.120.61','1973-07-13 00:30:22'), - (365,'Catherine','cperkinsa4@nytimes.com','58.21.24.214','2006-11-19 11:52:26'), - (366,'Mark','mcartera5@cpanel.net','220.33.102.142','2007-09-09 09:43:27'), - (367,'Paula','ppricea6@msn.com','36.182.238.124','2009-11-11 09:13:05'), - (368,'Catherine','cgreena7@army.mil','228.203.58.19','2005-08-09 16:52:15'), - (369,'Helen','hhamiltona8@symantec.com','155.56.194.99','2005-02-01 05:40:36'), - (370,'Jane','jmeyera9@ezinearticles.com','133.244.113.213','2013-11-06 22:10:23'), - (371,'Wanda','wevansaa@bloglovin.com','233.125.192.48','1994-12-26 23:43:42'), - (372,'Mark','mmarshallab@tumblr.com','114.74.60.47','2016-09-29 18:03:01'), - (373,'Andrew','amartinezac@google.cn','182.54.37.130','1976-06-06 17:04:17'), - (374,'Helen','hmoralesad@e-recht24.de','42.45.4.123','1977-03-28 19:06:59'), - (375,'Bonnie','bstoneae@php.net','196.149.79.137','1970-02-05 17:05:58'), - (376,'Douglas','dfreemanaf@nasa.gov','215.65.124.218','2008-11-20 21:51:55'), - (377,'Willie','wwestag@army.mil','35.189.92.118','1992-07-24 05:08:08'), - (378,'Cheryl','cwagnerah@upenn.edu','228.239.222.141','2010-01-25 06:29:01'), - (379,'Sandra','swardai@baidu.com','63.11.113.240','1985-05-23 08:07:37'), - (380,'Julie','jrobinsonaj@jugem.jp','110.58.202.50','2015-03-05 09:42:07'), - (381,'Larry','lwagnerak@shop-pro.jp','98.234.25.24','1975-07-22 22:22:02'), - (382,'Juan','jcastilloal@yelp.com','24.174.74.202','2007-01-17 09:32:43'), - (383,'Donna','dfrazieram@artisteer.com','205.26.147.45','1990-02-11 20:55:46'), - (384,'Rachel','rfloresan@w3.org','109.60.216.162','1983-05-22 22:42:18'), - (385,'Robert','rreynoldsao@theguardian.com','122.65.209.130','2009-05-01 18:02:51'), - (386,'Donald','dbradleyap@etsy.com','42.54.35.126','1997-01-16 16:31:52'), - (387,'Rachel','rfisheraq@nih.gov','160.243.250.45','2006-02-17 22:05:49'), - (388,'Nicholas','nhamiltonar@princeton.edu','156.211.37.111','1976-06-21 03:36:29'), - (389,'Timothy','twhiteas@ca.gov','36.128.23.70','1975-09-24 03:51:18'), - (390,'Diana','dbradleyat@odnoklassniki.ru','44.102.120.184','1983-04-27 09:02:50'), - (391,'Billy','bfowlerau@jimdo.com','91.200.68.196','1995-01-29 06:57:35'), - (392,'Bruce','bandrewsav@ucoz.com','48.12.101.125','1992-10-27 04:31:39'), - (393,'Linda','lromeroaw@usa.gov','100.71.233.19','1992-06-08 15:13:18'), - (394,'Debra','dwatkinsax@ucoz.ru','52.160.233.193','2001-11-11 06:51:01'), - (395,'Katherine','kburkeay@wix.com','151.156.242.141','2010-06-14 19:54:28'), - (396,'Martha','mharrisonaz@youku.com','21.222.10.199','1989-10-16 14:17:55'), - (397,'Dennis','dwellsb0@youtu.be','103.16.29.3','1985-12-21 06:05:51'), - (398,'Gloria','grichardsb1@bloglines.com','90.147.120.234','1982-08-27 01:04:43'), - (399,'Brenda','bfullerb2@t.co','33.253.63.90','2011-04-20 05:00:35'), - (400,'Larry','lhendersonb3@disqus.com','88.95.132.128','1982-08-31 02:15:12'), - (401,'Richard','rlarsonb4@wisc.edu','13.48.231.150','1979-04-15 14:08:09'), - (402,'Terry','thuntb5@usa.gov','65.91.103.240','1998-05-15 11:50:49'), - (403,'Harry','hburnsb6@nasa.gov','33.38.21.244','1981-04-12 14:02:20'), - (404,'Diana','dellisb7@mlb.com','218.229.81.135','1997-01-29 00:17:25'), - (405,'Jack','jburkeb8@tripadvisor.com','210.227.182.216','1984-03-09 17:24:03'), - (406,'Julia','jlongb9@fotki.com','10.210.12.104','2005-10-26 03:54:13'), - (407,'Lois','lscottba@msu.edu','188.79.136.138','1973-02-02 18:40:39'), - (408,'Sandra','shendersonbb@shareasale.com','114.171.220.108','2012-06-09 18:22:26'), - (409,'Irene','isanchezbc@cdbaby.com','109.255.50.119','1983-09-28 21:11:27'), - (410,'Emily','ebrooksbd@bandcamp.com','227.81.93.79','1970-08-31 21:08:01'), - (411,'Michelle','mdiazbe@businessweek.com','236.249.6.226','1993-05-22 08:07:07'), - (412,'Tammy','tbennettbf@wisc.edu','145.253.239.152','1978-12-31 20:24:51'), - (413,'Christine','cgreenebg@flickr.com','97.25.140.118','1978-07-17 12:55:30'), - (414,'Patricia','pgarzabh@tuttocitta.it','139.246.192.211','1984-02-27 13:40:08'), - (415,'Kimberly','kromerobi@aol.com','73.56.88.247','1976-09-16 14:22:04'), - (416,'George','gjohnstonbj@fda.gov','240.36.245.185','1979-07-24 14:36:02'), - (417,'Eugene','efullerbk@sciencedaily.com','42.38.105.140','2012-09-12 01:56:41'), - (418,'Andrea','astevensbl@goo.gl','31.152.207.204','1979-05-24 11:06:21'), - (419,'Shirley','sreidbm@scientificamerican.com','103.60.31.241','1984-02-23 04:07:41'), - (420,'Terry','tmorenobn@blinklist.com','92.161.34.42','1994-06-25 14:01:35'), - (421,'Christopher','cmorenobo@go.com','158.86.176.82','1973-09-05 09:18:47'), - (422,'Dennis','dhansonbp@ning.com','40.160.81.75','1982-01-20 10:19:41'), - (423,'Beverly','brussellbq@de.vu','138.32.56.204','1997-11-06 07:20:19'), - (424,'Howard','hparkerbr@163.com','103.171.134.171','2015-06-24 15:37:10'), - (425,'Helen','hmccoybs@fema.gov','61.200.4.71','1995-06-20 08:59:10'), - (426,'Ann','ahudsonbt@cafepress.com','239.187.71.125','1977-04-11 07:59:28'), - (427,'Tina','twestbu@nhs.uk','80.213.117.74','1992-08-19 05:54:44'), - (428,'Terry','tnguyenbv@noaa.gov','21.93.118.95','1991-09-19 23:22:55'), - (429,'Ashley','aburtonbw@wix.com','233.176.205.109','2009-11-10 05:01:20'), - (430,'Eric','emyersbx@1und1.de','168.91.212.67','1987-08-10 07:16:20'), - (431,'Barbara','blittleby@lycos.com','242.14.189.239','2008-08-02 12:13:04'), - (432,'Sean','sevansbz@instagram.com','14.39.177.13','2007-04-16 17:28:49'), - (433,'Shirley','sburtonc0@newsvine.com','34.107.138.76','1980-12-10 02:19:29'), - (434,'Patricia','pfreemanc1@so-net.ne.jp','219.213.142.117','1987-03-01 02:25:45'), - (435,'Paula','pfosterc2@vkontakte.ru','227.14.138.141','1972-09-22 12:59:34'), - (436,'Nicole','nstewartc3@1688.com','8.164.23.115','1998-10-27 00:10:17'), - (437,'Earl','ekimc4@ovh.net','100.26.244.177','2013-01-22 10:05:46'), - (438,'Beverly','breedc5@reuters.com','174.12.226.27','1974-09-22 07:29:36'), - (439,'Lawrence','lbutlerc6@a8.net','105.164.42.164','1992-06-05 00:43:40'), - (440,'Charles','cmoorec7@ucoz.com','252.197.131.69','1990-04-09 02:34:05'), - (441,'Alice','alawsonc8@live.com','183.73.220.232','1989-02-28 09:11:04'), - (442,'Dorothy','dcarpenterc9@arstechnica.com','241.47.200.14','2005-05-02 19:57:21'), - (443,'Carolyn','cfowlerca@go.com','213.109.55.202','1978-09-10 20:18:20'), - (444,'Anthony','alongcb@free.fr','169.221.158.204','1984-09-13 01:59:23'), - (445,'Annie','amoorecc@e-recht24.de','50.34.148.61','2009-03-26 03:41:07'), - (446,'Carlos','candrewscd@ihg.com','236.69.59.212','1972-03-29 22:42:48'), - (447,'Beverly','bramosce@google.ca','164.250.184.49','1982-11-10 04:34:01'), - (448,'Teresa','tlongcf@umich.edu','174.88.53.223','1987-05-17 12:48:00'), - (449,'Roy','rboydcg@uol.com.br','91.58.243.215','1974-06-16 17:59:54'), - (450,'Ashley','afieldsch@tamu.edu','130.138.11.126','1983-09-15 05:52:36'), - (451,'Judith','jhawkinsci@cmu.edu','200.187.103.245','2003-10-22 12:24:03'), - (452,'Rebecca','rwestcj@ocn.ne.jp','72.85.3.103','1980-11-13 11:01:26'), - (453,'Raymond','rporterck@infoseek.co.jp','146.33.216.151','1982-05-17 23:58:03'), - (454,'Janet','jmarshallcl@odnoklassniki.ru','52.46.193.166','1998-10-04 00:02:21'), - (455,'Shirley','speterscm@salon.com','248.126.31.15','1987-01-30 06:04:59'), - (456,'Annie','abowmancn@economist.com','222.213.248.59','2006-03-14 23:52:59'), - (457,'Jean','jlarsonco@blogspot.com','71.41.25.195','2007-09-08 23:49:45'), - (458,'Phillip','pmoralescp@stanford.edu','74.119.87.28','2011-03-14 20:25:40'), - (459,'Norma','nrobinsoncq@economist.com','28.225.21.54','1989-10-21 01:22:43'), - (460,'Kimberly','kclarkcr@dion.ne.jp','149.171.132.153','2008-06-27 02:27:30'), - (461,'Ruby','rmorriscs@ucla.edu','177.85.163.249','2016-01-28 16:43:44'), - (462,'Jonathan','jcastilloct@tripod.com','78.4.28.77','2000-05-24 17:33:06'), - (463,'Edward','ebryantcu@jigsy.com','140.31.98.193','1992-12-17 08:32:47'), - (464,'Chris','chamiltoncv@eepurl.com','195.171.234.206','1970-12-05 03:42:19'), - (465,'Michael','mweavercw@reference.com','7.233.133.213','1987-03-29 02:30:54'), - (466,'Howard','hlawrencecx@businessweek.com','113.225.124.224','1990-07-30 07:20:57'), - (467,'Philip','phowardcy@comsenz.com','159.170.247.249','2010-10-15 10:18:37'), - (468,'Mary','mmarshallcz@xing.com','125.132.189.70','2007-07-19 13:48:47'), - (469,'Scott','salvarezd0@theguardian.com','78.49.103.230','1987-10-31 06:10:44'), - (470,'Wayne','wcarrolld1@blog.com','238.1.120.204','1980-11-19 03:26:10'), - (471,'Jennifer','jwoodsd2@multiply.com','92.20.224.49','2010-05-06 22:17:04'), - (472,'Raymond','rwelchd3@toplist.cz','176.158.35.240','2007-12-12 19:02:51'), - (473,'Steven','sdixond4@wisc.edu','167.55.237.52','1984-05-05 11:44:37'), - (474,'Ralph','rjamesd5@ameblo.jp','241.190.50.133','2000-07-06 08:44:37'), - (475,'Jason','jrobinsond6@hexun.com','138.119.139.56','2006-02-03 05:27:45'), - (476,'Doris','dwoodd7@fema.gov','180.220.156.190','1978-05-11 20:14:20'), - (477,'Elizabeth','eberryd8@youtu.be','74.188.53.229','2006-11-18 08:29:06'), - (478,'Irene','igilbertd9@privacy.gov.au','194.152.218.1','1985-09-17 02:46:52'), - (479,'Jessica','jdeanda@ameblo.jp','178.103.93.118','1974-06-07 19:04:05'), - (480,'Rachel','ralvarezdb@phoca.cz','17.22.223.174','1999-03-08 02:43:25'), - (481,'Kenneth','kthompsondc@shinystat.com','229.119.91.234','2007-05-15 13:17:32'), - (482,'Harold','hmurraydd@parallels.com','133.26.188.80','1993-11-15 03:42:07'), - (483,'Paula','phowellde@samsung.com','34.215.28.216','1993-11-29 15:55:00'), - (484,'Ruth','rpiercedf@tripadvisor.com','111.30.130.123','1986-08-17 10:19:38'), - (485,'Phyllis','paustindg@vk.com','50.84.34.178','1994-04-13 03:05:24'), - (486,'Laura','lfosterdh@usnews.com','37.8.101.33','2001-06-30 08:58:59'), - (487,'Eric','etaylordi@com.com','103.183.253.45','2006-09-15 20:18:46'), - (488,'Doris','driveradj@prweb.com','247.16.2.199','1989-05-08 09:27:09'), - (489,'Ryan','rhughesdk@elegantthemes.com','103.234.153.232','1989-08-01 18:36:06'), - (490,'Steve','smoralesdl@jigsy.com','3.76.84.207','2011-03-13 17:01:05'), - (491,'Louis','lsullivandm@who.int','78.135.44.208','1975-11-26 16:01:23'), - (492,'Catherine','ctuckerdn@seattletimes.com','93.137.106.21','1990-03-13 16:14:56'), - (493,'Ann','adixondo@gmpg.org','191.136.222.111','2002-06-05 14:22:18'), - (494,'Johnny','jhartdp@amazon.com','103.252.198.39','1988-07-30 23:54:49'), - (495,'Susan','srichardsdq@skype.com','126.247.192.11','2005-01-09 12:08:14'), - (496,'Brenda','bparkerdr@skype.com','63.232.216.86','1974-05-18 05:58:29'), - (497,'Tammy','tmurphyds@constantcontact.com','56.56.37.112','2014-08-05 18:22:25'), - (498,'Larry','lhayesdt@wordpress.com','162.146.13.46','1997-02-26 14:01:53'), - (499,'Evelyn','ethomasdu@hhs.gov','6.241.88.250','2007-09-14 13:03:34'), - (500,'Paula','pshawdv@networksolutions.com','123.27.47.249','2003-10-30 21:19:20'); + (1,'Larry','lking0@miitbeian.gov.cn','69.135.206.194','2008-09-12 19:08:31'); create table {schema}.seed_config_expected_1 as ( @@ -517,17 +18,6 @@ create table {schema}.seed_config_expected_1 as ( ); -create table {schema}.seed_config_expected_2 as ( - - select *, 'abc'::text as c1, 'def'::text as c2, 'was true'::text as some_bool from {schema}.seed - -); - -create table {schema}.seed_config_expected_3 as ( - - select *, 'ghi'::text as c1, 'jkl'::text as c2, 'was true'::text as some_bool from {schema}.seed - -); create table {schema}.seed_summary ( year timestamp without time zone, @@ -537,51 +27,5 @@ create table {schema}.seed_summary ( INSERT INTO {schema}.seed_summary ("year","count") VALUES - ('1970-01-01 00:00:00',10), - ('1971-01-01 00:00:00',6), - ('1972-01-01 00:00:00',9), - ('1973-01-01 00:00:00',12), - ('1974-01-01 00:00:00',8), - ('1975-01-01 00:00:00',5), - ('1976-01-01 00:00:00',11), - ('1977-01-01 00:00:00',13), - ('1978-01-01 00:00:00',11), - ('1979-01-01 00:00:00',13), - ('1980-01-01 00:00:00',9), - ('1981-01-01 00:00:00',3), - ('1982-01-01 00:00:00',9), - ('1983-01-01 00:00:00',15), - ('1984-01-01 00:00:00',13), - ('1985-01-01 00:00:00',11), - ('1986-01-01 00:00:00',5), - ('1987-01-01 00:00:00',14), - ('1988-01-01 00:00:00',9), - ('1989-01-01 00:00:00',10), - ('1990-01-01 00:00:00',12), - ('1991-01-01 00:00:00',16), - ('1992-01-01 00:00:00',15), - ('1993-01-01 00:00:00',11), - ('1994-01-01 00:00:00',10), - ('1995-01-01 00:00:00',10), - ('1996-01-01 00:00:00',6), - ('1997-01-01 00:00:00',11), - ('1998-01-01 00:00:00',12), - ('1999-01-01 00:00:00',9), - ('2000-01-01 00:00:00',13), - ('2001-01-01 00:00:00',14), - ('2002-01-01 00:00:00',9), - ('2003-01-01 00:00:00',8), - ('2004-01-01 00:00:00',9), - ('2005-01-01 00:00:00',14), - ('2006-01-01 00:00:00',9), - ('2007-01-01 00:00:00',16), - ('2008-01-01 00:00:00',6), - ('2009-01-01 00:00:00',15), - ('2010-01-01 00:00:00',13), - ('2011-01-01 00:00:00',23), - ('2012-01-01 00:00:00',9), - ('2013-01-01 00:00:00',10), - ('2014-01-01 00:00:00',9), - ('2015-01-01 00:00:00',10), - ('2016-01-01 00:00:00',5); + ('2008-01-01 00:00:00',6); diff --git a/test/integration/029_catalog_generate_tests/test_catalog_generate.py b/test/integration/029_catalog_generate_tests/test_catalog_generate.py index 8f0b7732ac6..32a991034e3 100644 --- a/test/integration/029_catalog_generate_tests/test_catalog_generate.py +++ b/test/integration/029_catalog_generate_tests/test_catalog_generate.py @@ -38,36 +38,127 @@ def test_simple_generate(self): my_schema_name = self.unique_schema() self.assertIn(my_schema_name, data) my_schema = data[my_schema_name] - expected_tables = { - 'seed', 'seed_config_expected_1', 'seed_config_expected_2', - 'seed_config_expected_3', 'seed_summary' - } - self.assertEqual(set(my_schema), expected_tables) - expected_summary = { - 'metadata': { - 'schema': my_schema_name, - 'name': 'seed_summary', - 'type': 'BASE TABLE', - 'comment': None, + expected = { + 'seed': { + 'metadata': { + 'schema': my_schema_name, + 'name': 'seed', + 'type': 'BASE TABLE', + 'comment': None + }, + 'columns': [ + { + 'name': 'id', + 'index': 1, + 'type': 'integer', + 'comment': None + }, + { + 'name': 'first_name', + 'index': 2, + 'type': 'character varying', + 'comment': None + }, + { + 'name': 'email', 'index': 3, + 'type': 'character varying', + 'comment': None, + }, + { + 'name': 'ip_address', + 'index': 4, + 'type': 'character varying', + 'comment': None + }, + { + 'name': 'updated_at', + 'index': 5, + 'type': 'timestamp without time zone', + 'comment': None + }, + ], }, - 'columns': [ + 'seed_config_expected_1': { - 'name': 'year', - 'index': 1, - 'type': 'timestamp without time zone', + 'metadata': { + 'schema': my_schema_name, + 'name': 'seed_config_expected_1', + 'type': 'BASE TABLE', 'comment': None, }, - { - 'name': 'count', - 'index': 2, - 'type': 'bigint', + 'columns': [ + { + 'name': 'id', + 'index': 1, + 'type': 'integer', + 'comment': None, + }, + { + 'name': 'first_name', + 'index': 2, + 'type': 'character varying', + 'comment': None, + }, + { + 'name': 'email', + 'index': 3, + 'type': 'character varying', + 'comment': None, + }, + { + 'name': 'ip_address', + 'index': 4, + 'type': 'character varying', + 'comment': None, + }, + { + 'name': 'updated_at', + 'index': 5, + 'type': 'timestamp without time zone', + 'comment': None, + }, + { + 'name': 'c1', + 'index': 6, + 'type': 'text', + 'comment': None, + }, + { + 'name': 'c2', + 'index': 7, + 'type': 'text', + 'comment': None, + }, + { + 'name': 'some_bool', + 'index': 8, + 'type': 'text', + 'comment': None, + }, + ], + }, + 'seed_summary': { + 'metadata': { + 'schema': my_schema_name, + 'name': 'seed_summary', + 'type': 'BASE TABLE', 'comment': None }, - ], + 'columns': [ + { + 'name': 'year', + 'index': 1, + 'type': 'timestamp without time zone', + 'comment': None, + }, + { + 'name': 'count', + 'index': 2, + 'type': 'bigint', + 'comment': None, + }, + ] + } } - self.assertEqual(expected_summary, my_schema_name) - - - - + self.assertEqual(expected, my_schema) From 223a764e1047db225f7721ca81c4b4c690ffeb9c Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 10 May 2018 10:37:24 -0600 Subject: [PATCH 09/43] turn this into a constant --- dbt/task/generate.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 0caceab8fd6..bc8f7a689e9 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -10,6 +10,9 @@ from dbt.task.base_task import BaseTask +CATALOG_FILENAME = 'catalog.json' + + def get_stripped_prefix(source, prefix): """Go through source, extracting every key/value pair where the key starts with the given prefix. @@ -93,8 +96,8 @@ def run(self): profile = self.project.run_environment() adapter = get_adapter(profile) - # To get a list of schemas, we'd need to generate the parsed manifest - # for now just pass None. + # To get a list of schemas, we'd need to generate the parsed manifest. + # For now just pass None. try: columns = adapter.get_catalog_for_schemas(profile, schemas=None) adapter.release_connection(profile) @@ -103,7 +106,7 @@ def run(self): results = unflatten(columns) - path = os.path.join(self.project['target-path'], 'catalog.json') + path = os.path.join(self.project['target-path'], CATALOG_FILENAME) write_file(path, json.dumps(results)) dbt.ui.printer.print_timestamped_line( From b232d7f0626f6459691a5d121c47f8e537c2564e Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 10 May 2018 10:47:47 -0600 Subject: [PATCH 10/43] Comment / import cleanup --- dbt/adapters/postgres/impl.py | 1 - dbt/task/generate.py | 1 - .../029_catalog_generate_tests/test_catalog_generate.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index 031c1a23fc3..86509a24062 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -216,7 +216,6 @@ def get_catalog_for_schemas(cls, profile, schemas=None): where table_schema != 'information_schema' and table_schema not like 'pg_%' """.strip() - # TODO: make sure I can just ignore status. _, results = cls.execute(profile, sql, fetch=True) # ok, have an agate.Table now consisting of my results. Filter out # schemas if necessary (this should probably happen in SQL, right?). diff --git a/dbt/task/generate.py b/dbt/task/generate.py index bc8f7a689e9..86a59cb5db4 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -1,7 +1,6 @@ import json import os -from dbt.logger import GLOBAL_LOGGER as logger from dbt.adapters.factory import get_adapter from dbt.clients.system import write_file from dbt.compat import bigint diff --git a/test/integration/029_catalog_generate_tests/test_catalog_generate.py b/test/integration/029_catalog_generate_tests/test_catalog_generate.py index 32a991034e3..c9f7ab62158 100644 --- a/test/integration/029_catalog_generate_tests/test_catalog_generate.py +++ b/test/integration/029_catalog_generate_tests/test_catalog_generate.py @@ -4,6 +4,7 @@ from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest + class TestCatalogGenerate(DBTIntegrationTest): def setUp(self): @@ -27,7 +28,6 @@ def project_config(self): } @attr(type='postgres') - @attr(type='catalog') def test_simple_generate(self): self.run_dbt(["catalog", "generate"]) self.assertTrue(os.path.exists('./target/catalog.json')) From 4f6cd0aab8e3181cf0c383b81f32b9c80413d6f5 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 11:28:15 -0600 Subject: [PATCH 11/43] Add an operations config entry and an OperationLoader --- dbt/include/global_project/dbt_project.yml | 1 + dbt/loader.py | 17 ++++++++++++++++- dbt/project.py | 1 + sample.dbt_project.yml | 5 +++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/dbt/include/global_project/dbt_project.yml b/dbt/include/global_project/dbt_project.yml index cf30f7f9184..8d43087eb27 100644 --- a/dbt/include/global_project/dbt_project.yml +++ b/dbt/include/global_project/dbt_project.yml @@ -3,3 +3,4 @@ name: dbt version: 1.0 macro-paths: ["macros"] +operation-paths: ["operations"] diff --git a/dbt/loader.py b/dbt/loader.py index 38658c0dcce..8650a93e113 100644 --- a/dbt/loader.py +++ b/dbt/loader.py @@ -12,7 +12,6 @@ class GraphLoader(object): @classmethod def load_all(cls, root_project, all_projects): macros = MacroLoader.load_all(root_project, all_projects) - nodes = {} for loader in cls._LOADERS: nodes.update(loader.load_all(root_project, all_projects, macros)) @@ -85,6 +84,21 @@ def load_project(cls, root_project, all_projects, project, project_name, macros=macros) +class OperationLoader(ResourceLoader): + + @classmethod + def load_project(cls, root_project, all_projects, project, project_name, + macros): + return dbt.parser.load_and_parse_sql( + package_name=project_name, + root_project=root_project, + all_projects=all_projects, + root_dir=project.get('project-root'), + relative_dirs=project.get('operation-paths', []), + resource_type=NodeType.Operation, + macros=macros) + + class AnalysisLoader(ResourceLoader): @classmethod @@ -180,3 +194,4 @@ def load_project(cls, root_project, all_projects, project, project_name, GraphLoader.register(RunHookLoader) GraphLoader.register(ArchiveLoader) GraphLoader.register(SeedLoader) +GraphLoader.register(OperationLoader) \ No newline at end of file diff --git a/dbt/project.py b/dbt/project.py index ca1ac3535ab..ca36c1d9124 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -21,6 +21,7 @@ default_project_cfg = { 'source-paths': ['models'], 'macro-paths': ['macros'], + 'operation-paths': ['operations'], 'data-paths': ['data'], 'test-paths': ['test'], 'target-path': 'target', diff --git a/sample.dbt_project.yml b/sample.dbt_project.yml index d581b70b826..987eed3ea06 100644 --- a/sample.dbt_project.yml +++ b/sample.dbt_project.yml @@ -50,6 +50,11 @@ data-paths: ["data"] # macros. These macros will be globally available to all models in your project macro-paths: ['macros'] +# operation-paths: Optional. Specify which path(s) dbt should look in to find +# operations. These macros will be globally available to all models in your +# project +operation-paths: ['operations'] + # log-path: Optional. Specify which path dbt should write debug logs to. log-path: "logs" From 0c5e214c621095093dcff003a722713af06567cd Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 13:53:23 -0600 Subject: [PATCH 12/43] Rearranged stuff, added an OperationRunner that I'm not sure about --- dbt/loader.py | 3 ++- dbt/node_runners.py | 31 +++++++++++++++++++++++++++++++ dbt/task/generate.py | 24 ++++++++++++++---------- dbt/task/operation.py | 23 +++++++++++++++++++++++ 4 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 dbt/task/operation.py diff --git a/dbt/loader.py b/dbt/loader.py index 8650a93e113..2203fca679e 100644 --- a/dbt/loader.py +++ b/dbt/loader.py @@ -12,6 +12,7 @@ class GraphLoader(object): @classmethod def load_all(cls, root_project, all_projects): macros = MacroLoader.load_all(root_project, all_projects) + nodes = {} for loader in cls._LOADERS: nodes.update(loader.load_all(root_project, all_projects, macros)) @@ -194,4 +195,4 @@ def load_project(cls, root_project, all_projects, project, project_name, GraphLoader.register(RunHookLoader) GraphLoader.register(ArchiveLoader) GraphLoader.register(SeedLoader) -GraphLoader.register(OperationLoader) \ No newline at end of file +GraphLoader.register(OperationLoader) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index ddf1d564541..3c20b7a8cd0 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -3,6 +3,7 @@ from dbt.exceptions import NotImplementedException from dbt.utils import get_nodes_by_tags from dbt.node_types import NodeType, RunHookType +from dbt.adapters.factory import get_adapter import dbt.clients.jinja import dbt.context.runtime @@ -516,3 +517,33 @@ def print_result_line(self, result): schema_name, self.node_index, self.num_nodes) + + +class OperationRunner(ModelRunner): + def compile(self, flat_graph): + # Operations don't compile, I think + return self.node + + def before_execute(self): + pass + + def execute(self, compiled_node, flat_graph): + # remember, compiled node really isn't compiled! + adapter = get_adapter(self.profile) + + schemas = adapter.get_existing_schemas(self.profile, self.project) + # I'm not sure this is right. I think we might have to do a compile + # pass to get jinja run across it, or something? + sql = compiled_node.get('raw_sql') + + try: + _, result = adapter.execute(profile, sql, fetch=True) + adapter.release_connection(profile) + finally: + adapter.cleanup_connections() + + result = result.where(lambda r: r['table_schema'] in schemas) + return [dict(zip(result.column_names, row)) for row in result] + + def after_execute(self, result): + pass diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 86a59cb5db4..0ea0841bef6 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -92,18 +92,22 @@ def unflatten(columns): # derive from BaseTask as I don't really want any result interpretation. class GenerateTask(BaseTask): def run(self): - profile = self.project.run_environment() - adapter = get_adapter(profile) + runner = RunManager( + self.project, self.project['target-path'], self.args + ) - # To get a list of schemas, we'd need to generate the parsed manifest. - # For now just pass None. - try: - columns = adapter.get_catalog_for_schemas(profile, schemas=None) - adapter.release_connection(profile) - finally: - adapter.cleanup_connections() + # TODO: this is probably wrong + query = { + "include": self.args.models, + "exclude": self.args.exclude, + "resource_types": NodeType.executable(), + "tags": [] + } - results = unflatten(columns) + results = runner.run(query, GenerateRunner) + results = unflatten(results) + + profile = self.project.run_environment() path = os.path.join(self.project['target-path'], CATALOG_FILENAME) write_file(path, json.dumps(results)) diff --git a/dbt/task/operation.py b/dbt/task/operation.py new file mode 100644 index 00000000000..68d38b0e6a4 --- /dev/null +++ b/dbt/task/operation.py @@ -0,0 +1,23 @@ +from dbt.node_runners import OperationRunner +from dbt.node_types import NodeType +from dbt.runner import RunManager +from dbt.task.base_task import RunnableTask +import dbt.ui.printer + + +class OperationTask(RunnableTask): + def run(self): + runner = RunManager( + self.project, + self.project["target-path"], + self.args, + ) + query = { + "include": ["*"], + "exclude": [], + "resource_types": [NodeType.Operation], + } + results = runner.run_flat(query, OperationRunner) + + dbt.ui.printer.print_run_end_messages(results) + return results From 7abb9d9861b1c78416617647a171881536f4e9c1 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 17:04:46 -0600 Subject: [PATCH 13/43] Add functions for operating on operations inside the graph --- dbt/utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dbt/utils.py b/dbt/utils.py index 7420a320bd9..544fc0193f0 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -107,6 +107,11 @@ def find_macro_by_name(flat_graph, target_name, target_package): 'macros', [NodeType.Macro]) +def find_operation_by_name(flat_graph, target_name, target_package): + return find_by_name(flat_graph, target_name, target_package, + 'macros', [NodeType.Operation]) + + def find_by_name(flat_graph, target_name, target_package, subgraph, nodetype): for name, model in flat_graph.get(subgraph).items(): @@ -170,6 +175,15 @@ def get_materialization_macro(flat_graph, materialization_name, return macro +def get_operation_name(operation_name): + return 'operation_{}'.format(operation_name) + + +def get_operation(flat_graph, operation_name): + macro_name = get_operation_name(operation_name) + return find_operation_by_name(flat_graph, macro_name, None) + + def load_project_with_profile(source_project, project_dir): project_filepath = os.path.join(project_dir, 'dbt_project.yml') return dbt.project.read_project( From 0c62326f398d5dd5ebd27c0afb27dade386462d8 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 17:05:54 -0600 Subject: [PATCH 14/43] Fix wrong name --- dbt/task/generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 0ea0841bef6..e79e6dab660 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -4,6 +4,7 @@ from dbt.adapters.factory import get_adapter from dbt.clients.system import write_file from dbt.compat import bigint +from dbt.node_runners import OperationRunner import dbt.ui.printer from dbt.task.base_task import BaseTask @@ -104,7 +105,7 @@ def run(self): "tags": [] } - results = runner.run(query, GenerateRunner) + results = runner.run(query, OperationRunner) results = unflatten(results) profile = self.project.run_environment() From 512493f52780ca292f82618cf36bd145e06df1d8 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 17:26:32 -0600 Subject: [PATCH 15/43] Rearrange where the agate table -> dict conversion happens, add new result type for operations --- dbt/node_runners.py | 58 ++++++++++++++++++++++++++++++++++++-------- dbt/task/generate.py | 8 +++++- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 3c20b7a8cd0..d388b9c0de7 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -61,6 +61,25 @@ def failed(self): def skipped(self): return self.skip +class RunOperationResult(RunModelResult): + def __init__(self, node, error=None, skip=False, status=None, + failed=None, execution_time=0, returned=None): + super(RunOperationResult, self).__init__(node, error, skip, status, + failed, execution_time) + self.returned = returned + + @property + def errored(self): + return self.error is not None + + @property + def failed(self): + return self.fail + + @property + def skipped(self): + return self.skip + class BaseRunner(object): print_header = True @@ -527,23 +546,42 @@ def compile(self, flat_graph): def before_execute(self): pass - def execute(self, compiled_node, flat_graph): - # remember, compiled node really isn't compiled! - adapter = get_adapter(self.profile) + def execute(self, model, flat_graph): + context = dbt.context.runtime.generate( + model, + self.project.cfg, + flat_graph + ) + + # TODO: this is probably wrong. I made this up b/c it sounds nice + operation_name = model.name + operation = dbt.utils.get_operation(flat_graph, operation_name) + + operation.generator(context)() - schemas = adapter.get_existing_schemas(self.profile, self.project) - # I'm not sure this is right. I think we might have to do a compile - # pass to get jinja run across it, or something? - sql = compiled_node.get('raw_sql') + # TODO: this is probably wrong too, copied from CompileRunner. + # I think we need to end up passing the agate table out, need to + # understand how that works (and where status comes in now, I don't + # get that). + result = context['load_result']('main') + # TODO: thing I also don't understand. If we switch to RunManager, + # etc, how can I get the result back out? Callers (safe_run, etc) care + # about RunModelResult having a compiled node object. May need to + # refactor safe_run or subclass RunModelResult? + + # get the schemas to filter by, though it seems like the caller should + # really do this as another op and then combine the info approriately + # TODO: check on this + adapter = get_adapter(self.profile) try: - _, result = adapter.execute(profile, sql, fetch=True) - adapter.release_connection(profile) + schemas = adapter.get_existing_schemas(self.profile, self.project) + adapter.release_connection(self.profile) finally: adapter.cleanup_connections() result = result.where(lambda r: r['table_schema'] in schemas) - return [dict(zip(result.column_names, row)) for row in result] + return RunOperationResult(model, returned=result) def after_execute(self, result): pass diff --git a/dbt/task/generate.py b/dbt/task/generate.py index e79e6dab660..1d9e80b35af 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -105,7 +105,13 @@ def run(self): "tags": [] } - results = runner.run(query, OperationRunner) + # we get back an agate table inside our run result. + run_result = runner.run(query, OperationRunner) + agate_table = run_result.returned + results = [ + dict(zip(agate_table.column_names, row)) + for row in agate_table + ] results = unflatten(results) profile = self.project.run_environment() From 9c743758f91bd76bb3debd545a7f9e3993b7352f Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 17:27:01 -0600 Subject: [PATCH 16/43] Add 'operation' type --- dbt/clients/jinja.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/dbt/clients/jinja.py b/dbt/clients/jinja.py index 95d7d8f15da..5e05d06ad02 100644 --- a/dbt/clients/jinja.py +++ b/dbt/clients/jinja.py @@ -100,6 +100,28 @@ def parse(self, parser): return node +class OperationExtension(jinja2.ext.Extension): + tags = ['operation'] + + def parse(self, parser): + node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno) + operation_name = \ + parser.parse_assign_target(name_only=True).name + + node.args = [] + node.defaults = [] + + while parser.stream.skip_if('comma'): + target = parser.parse_assign_target(name_only=True) + + node.name = dbt.utils.get_operation_name(operation_name) + + node.body = parser.parse_statements(('name:endoperation',), + drop_needle=True) + + return node + + def create_macro_capture_env(node): class ParserMacroCapture(jinja2.Undefined): From 2db41c54e5bc4fbacfa3e75277cc2f555953c635 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 17:27:58 -0600 Subject: [PATCH 17/43] Add get_catalog macro with postgres implementation --- .../global_project/macros/adapters/common.sql | 58 +++++++++++++++++++ .../adapters/catalog/get_catalog.sql | 5 ++ 2 files changed, 63 insertions(+) create mode 100644 dbt/include/global_project/operations/adapters/catalog/get_catalog.sql diff --git a/dbt/include/global_project/macros/adapters/common.sql b/dbt/include/global_project/macros/adapters/common.sql index 7d6a42dba1d..73b8e3ded65 100644 --- a/dbt/include/global_project/macros/adapters/common.sql +++ b/dbt/include/global_project/macros/adapters/common.sql @@ -66,3 +66,61 @@ {{ column_list_for_create_table(columns) }} ); {% endmacro %} + + +{% macro get_catalog() -%} + {{ adapter_macro('get_catalog') }} +{%- endmacro %} + + +{% macro default__get_catalog() -%} + + {% set typename = adapter.type() %} + {% set msg -%} + get_catalog not implemented for {{ typename }} + {%- endset %} + + {{ exceptions.raise_compiler_error(msg) }} +{% endmacro %} + + +{% macro postgres__get_catalog() -%} + {%- call statement('catalog', fetch_result=True) -%} + with tables as ( + select + table_schema, + table_name, + table_type + + from information_schema.tables + + ), + + columns as ( + + select + table_schema, + table_name, + null as table_comment, + + column_name, + ordinal_position as column_index, + data_type as column_type, + null as column_comment + + + from information_schema.columns + + ) + + select * + from tables + join columns using (table_schema, table_name) + + where table_schema != 'information_schema' + and table_schema not like 'pg_%' + {%- endcall -%} + {% set catalog = load_results('catalog') %} + {{ return(catalog['table']) }} + +{%- endmacro %} diff --git a/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql b/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql new file mode 100644 index 00000000000..1dc9976b73d --- /dev/null +++ b/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql @@ -0,0 +1,5 @@ +{% operation get_catalog_data %} + {% set catalog = dbt.get_catalog() %} + {{ log(catalog, info=True) }} + {{ return(catalog) }} +{% endoperation %} \ No newline at end of file From 7d27c1b5e05b1f9c03c65bef7e47795793c58b27 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 17:28:53 -0600 Subject: [PATCH 18/43] make operations loader more macro-like rather than node-like --- dbt/loader.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dbt/loader.py b/dbt/loader.py index 2203fca679e..448e1d1fe16 100644 --- a/dbt/loader.py +++ b/dbt/loader.py @@ -12,6 +12,7 @@ class GraphLoader(object): @classmethod def load_all(cls, root_project, all_projects): macros = MacroLoader.load_all(root_project, all_projects) + macros.update(OperationLoader.load_all(root_project, all_projects)) nodes = {} for loader in cls._LOADERS: nodes.update(loader.load_all(root_project, all_projects, macros)) @@ -90,14 +91,13 @@ class OperationLoader(ResourceLoader): @classmethod def load_project(cls, root_project, all_projects, project, project_name, macros): - return dbt.parser.load_and_parse_sql( + return dbt.parser.load_and_parse_macros( package_name=project_name, root_project=root_project, all_projects=all_projects, root_dir=project.get('project-root'), relative_dirs=project.get('operation-paths', []), - resource_type=NodeType.Operation, - macros=macros) + resource_type=NodeType.Operation) class AnalysisLoader(ResourceLoader): @@ -195,4 +195,3 @@ def load_project(cls, root_project, all_projects, project, project_name, GraphLoader.register(RunHookLoader) GraphLoader.register(ArchiveLoader) GraphLoader.register(SeedLoader) -GraphLoader.register(OperationLoader) From 2c47a31861f8922c5365996f68167194d0f3f4ed Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 17:36:46 -0600 Subject: [PATCH 19/43] Tell jinja to use the new OperationExtension --- dbt/clients/jinja.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/clients/jinja.py b/dbt/clients/jinja.py index 5e05d06ad02..4fec265de14 100644 --- a/dbt/clients/jinja.py +++ b/dbt/clients/jinja.py @@ -164,6 +164,7 @@ def get_template(string, ctx, node=None, capture_macros=False): args['undefined'] = create_macro_capture_env(node) args['extensions'].append(MaterializationExtension) + args['extensions'].append(OperationExtension) env = MacroFuzzEnvironment(**args) From ef22e70ebcd8c25ded4fc975f3848e62f2efea4d Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 17:41:06 -0600 Subject: [PATCH 20/43] Now instead of import errors I have ones I don't understand, which is progress --- dbt/task/generate.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 1d9e80b35af..328c8b322d9 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -5,6 +5,8 @@ from dbt.clients.system import write_file from dbt.compat import bigint from dbt.node_runners import OperationRunner +from dbt.node_types import NodeType +from dbt.runner import RunManager import dbt.ui.printer from dbt.task.base_task import BaseTask @@ -99,9 +101,9 @@ def run(self): # TODO: this is probably wrong query = { - "include": self.args.models, - "exclude": self.args.exclude, - "resource_types": NodeType.executable(), + "include": [], + "exclude": [], + "resource_types": [NodeType.Operation], "tags": [] } From 4f39ef6d388f48f96e8a96dd011e9bdfb1967234 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 17:45:51 -0600 Subject: [PATCH 21/43] pep8 --- dbt/node_runners.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index d388b9c0de7..2a0bf22a978 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -61,11 +61,12 @@ def failed(self): def skipped(self): return self.skip + class RunOperationResult(RunModelResult): def __init__(self, node, error=None, skip=False, status=None, failed=None, execution_time=0, returned=None): super(RunOperationResult, self).__init__(node, error, skip, status, - failed, execution_time) + failed, execution_time) self.returned = returned @property From 8ca876ace803aeaf4b075b8fb4aeb5404a0c9307 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 11 May 2018 18:13:57 -0600 Subject: [PATCH 22/43] Remove RunManager to avoid compiling I hope --- dbt/task/generate.py | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 328c8b322d9..19941fa696f 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -94,21 +94,38 @@ def unflatten(columns): # derive from BaseTask as I don't really want any result interpretation. class GenerateTask(BaseTask): + def get_all_projects(self): + # TODO: I copy+pasted this from dbt/compilation.py, refactor! + root_project = self.project.cfg + all_projects = {root_project.get('name'): root_project} + dependency_projects = dbt.utils.dependency_projects(self.project) + + for project in dependency_projects: + name = project.cfg.get('name', 'unknown') + all_projects[name] = project.cfg + + if dbt.flags.STRICT_MODE: + dbt.contracts.project.ProjectList(**all_projects) + + return all_projects + def run(self): - runner = RunManager( - self.project, self.project['target-path'], self.args - ) - # TODO: this is probably wrong - query = { - "include": [], - "exclude": [], - "resource_types": [NodeType.Operation], - "tags": [] - } + profile = self.project.run_environment() + + # From dbt/compilation.py + root_project = self.project.cfg + all_projects = self.get_all_projects() + + manifest = dbt.loader.GraphLoader.load_all(root_project, all_projects) + flat_graph = manifest.to_flat_graph() + operation_name = 'adapters.catalog.get_catalog' # TODO: is it? + operation = dbt.utils.get_operation(flat_graph, operation_name) - # we get back an agate table inside our run result. - run_result = runner.run(query, OperationRunner) + adapter = get_adapter(profile) + + runner = OperationRunner(self.project, adapter, operation, 0, 0) + run_result = runner.safe_run(flat_graph) agate_table = run_result.returned results = [ dict(zip(agate_table.column_names, row)) @@ -116,8 +133,6 @@ def run(self): ] results = unflatten(results) - profile = self.project.run_environment() - path = os.path.join(self.project['target-path'], CATALOG_FILENAME) write_file(path, json.dumps(results)) From e2c1cde304da563dbc201429a1b4573688b2313e Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 11:02:08 -0600 Subject: [PATCH 23/43] Make the parser pass the macro type along --- dbt/parser.py | 8 +++++--- test/unit/test_parser.py | 7 +++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dbt/parser.py b/dbt/parser.py index 8a075bf71cb..306fe6f5eef 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -124,8 +124,9 @@ def parse_macro_file(macro_file_path, macro_file_contents, root_path, package_name, + resource_type, tags=None, - context=None): + context=None,): logger.debug("Parsing {}".format(macro_file_path)) @@ -166,7 +167,7 @@ def parse_macro_file(macro_file_path, 'name': name, 'unique_id': unique_id, 'tags': tags, - 'resource_type': NodeType.Macro, + 'resource_type': resource_type, 'depends_on': {'macros': []}, }) @@ -446,7 +447,8 @@ def load_and_parse_macros(package_name, root_project, all_projects, root_dir, file_match.get('relative_path'), file_contents, root_dir, - package_name)) + package_name, + resource_type)) return result diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 6029d5a616a..b6e31f82be3 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -5,6 +5,7 @@ import dbt.flags import dbt.parser +from dbt.node_types import NodeType def get_os_path(unix_path): return os.path.normpath(unix_path) @@ -1349,7 +1350,8 @@ def test__simple_macro(self): macro_file_path='simple_macro.sql', macro_file_contents=macro_file_contents, root_path=get_os_path('/usr/src/app'), - package_name='root') + package_name='root', + resource_type=NodeType.Macro) self.assertTrue(callable(result['macro.root.simple'].generator)) @@ -1384,7 +1386,8 @@ def test__simple_macro_used_in_model(self): macro_file_path='simple_macro.sql', macro_file_contents=macro_file_contents, root_path=get_os_path('/usr/src/app'), - package_name='root') + package_name='root', + resource_type=NodeType.Macro) self.assertTrue(callable(result['macro.root.simple'].generator)) From 6f78e63c48b3b9b73d73c758b97ec01d2aa23125 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 11:04:24 -0600 Subject: [PATCH 24/43] Allow operations in the macro subgraph --- dbt/contracts/graph/parsed.py | 5 ++++- dbt/contracts/graph/unparsed.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index ecd28ccd731..8f38024048b 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -182,7 +182,10 @@ 'maxLength': 127, }, 'resource_type': { - 'enum': [NodeType.Macro], + 'enum': [ + NodeType.Macro, + NodeType.Operation, + ], }, 'unique_id': { 'type': 'string', diff --git a/dbt/contracts/graph/unparsed.py b/dbt/contracts/graph/unparsed.py index f6409d4b4ac..7c5b8ded8f8 100644 --- a/dbt/contracts/graph/unparsed.py +++ b/dbt/contracts/graph/unparsed.py @@ -63,6 +63,8 @@ NodeType.Model, NodeType.Test, NodeType.Analysis, + # TODO: added this to ParsedMacro, does that mean we don't + # need it here anymore? NodeType.Operation, NodeType.Seed, # we need this if parse_node is going to handle archives. From fcbc9d71acb742e384baa448a78179ad622724b2 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 15:09:43 -0600 Subject: [PATCH 25/43] I don't understand why, but my test passes now --- dbt/context/common.py | 7 ++- .../global_project/macros/adapters/common.sql | 44 ++++++++++++++- .../adapters/catalog/get_catalog.sql | 1 + dbt/node_runners.py | 9 +-- dbt/parser.py | 2 +- dbt/plan.txt | 51 +++++++++++++++++ dbt/task/generate.py | 11 ++-- dbt/utils.py | 56 +++++++++++-------- 8 files changed, 145 insertions(+), 36 deletions(-) create mode 100644 dbt/plan.txt diff --git a/dbt/context/common.py b/dbt/context/common.py index 3548b667a2d..30b89f25c1c 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -100,6 +100,9 @@ def _add_macros(context, model, flat_graph): return context +# TODO: add _add_operations here? + + def _add_tracking(context): if dbt.tracking.active_user is not None: context = dbt.utils.merge(context, { @@ -149,6 +152,7 @@ def _env_var(var, default=None): def _store_result(sql_results): def call(name, status, agate_table=None): + print('in store_result call, name={}'.format(name)) if agate_table is None: agate_table = dbt.clients.agate_helper.empty_table() @@ -164,6 +168,7 @@ def call(name, status, agate_table=None): def _load_result(sql_results): def call(name): + print('sql_results={}'.format(sql_results)) return sql_results.get(name) return call @@ -201,7 +206,7 @@ def __init__(self, model, context, overrides): self.overrides = overrides if isinstance(model, dict) and model.get('unique_id'): - local_vars = model.get('config', {}).get('vars') + local_vars = model.get('config', {}).get('vars', {}) self.model_name = model.get('name') else: # still used for wrapping diff --git a/dbt/include/global_project/macros/adapters/common.sql b/dbt/include/global_project/macros/adapters/common.sql index 73b8e3ded65..4b1bbcc5432 100644 --- a/dbt/include/global_project/macros/adapters/common.sql +++ b/dbt/include/global_project/macros/adapters/common.sql @@ -83,8 +83,46 @@ {{ exceptions.raise_compiler_error(msg) }} {% endmacro %} +{% macro postgres__get_catalog() %} + {%- set sql = render("with tables as ( + select + table_schema, + table_name, + table_type + + from information_schema.tables + + ), + + columns as ( + + select + table_schema, + table_name, + null as table_comment, + + column_name, + ordinal_position as column_index, + data_type as column_type, + null as column_comment + + + from information_schema.columns + + ) + + select * + from tables + join columns using (table_schema, table_name) + + where table_schema != 'information_schema' + and table_schema not like 'pg_%'") -%} + + {%- set status, res = adapter.execute(sql, auto_begin=True, fetch=True) -%} + {{ store_result('catalog', status=status, agate_table=res) }} +{% endmacro %} -{% macro postgres__get_catalog() -%} +{% macro postgres__get_catalog_notworking() -%} {%- call statement('catalog', fetch_result=True) -%} with tables as ( select @@ -120,7 +158,9 @@ where table_schema != 'information_schema' and table_schema not like 'pg_%' {%- endcall -%} - {% set catalog = load_results('catalog') %} + {{ log('trying to load results?', info=True) }} + {% set catalog = load_result('catalog') %} + {{ log(catalog, info=True) }} {{ return(catalog['table']) }} {%- endmacro %} diff --git a/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql b/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql index 1dc9976b73d..1170b7da5dc 100644 --- a/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql +++ b/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql @@ -1,5 +1,6 @@ {% operation get_catalog_data %} {% set catalog = dbt.get_catalog() %} + {{ log('ran catalog query:', info=True)}} {{ log(catalog, info=True) }} {{ return(catalog) }} {% endoperation %} \ No newline at end of file diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 2a0bf22a978..3f7cb504005 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -548,14 +548,15 @@ def before_execute(self): pass def execute(self, model, flat_graph): - context = dbt.context.runtime.generate( - model, + context = dbt.context.parser.generate( + model.serialize(), self.project.cfg, flat_graph ) # TODO: this is probably wrong. I made this up b/c it sounds nice - operation_name = model.name + operation_name = model['name'] + import pdb;pdb.set_trace() operation = dbt.utils.get_operation(flat_graph, operation_name) operation.generator(context)() @@ -564,7 +565,7 @@ def execute(self, model, flat_graph): # I think we need to end up passing the agate table out, need to # understand how that works (and where status comes in now, I don't # get that). - result = context['load_result']('main') + result = context['load_result']('catalog').table # TODO: thing I also don't understand. If we switch to RunManager, # etc, how can I get the result back out? Callers (safe_run, etc) care # about RunModelResult having a compiled node object. May need to diff --git a/dbt/parser.py b/dbt/parser.py index 306fe6f5eef..dff70bbd12e 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -157,7 +157,7 @@ def parse_macro_file(macro_file_path, if type(item) == jinja2.runtime.Macro: name = key.replace(dbt.utils.MACRO_PREFIX, '') - unique_id = get_path(NodeType.Macro, + unique_id = get_path(resource_type, package_name, name) diff --git a/dbt/plan.txt b/dbt/plan.txt new file mode 100644 index 00000000000..e1ac8f021a7 --- /dev/null +++ b/dbt/plan.txt @@ -0,0 +1,51 @@ +Code plan: + - add an OperationRunner. + TODO: do I really? I may be able to just use ModelRunner w/ no changes + - maybe need an OperationLoader? + - I think so. Should it be part of 'nodes' or does it need to be its own thing? + - Toss it in 'nodes' for now... + - wire up a thing in include/global_project/operations/adapters/postgres.sql, or something? + - Modify the generate task's run() to do a parse run against the models first + - use that parsed manifest to generate the flat map, etc + - use manifest to get the list of schemas + - flat map has the sql inside its operations -> where does that enter the call chain? + +Macro/Operation execute statement or something + +Make a new get_catalog macro, have operation type I create, have the operation call out to macros, do the adapter-specific work inside macros (I think) + +In dbt/clients/jinja.py, MaterializationExtension - change this to 'OperationExtension' or whatever, +and it's pretty similar. Then we can do a jinja sql thingy with: + + {% operation get_catalog_data %} + + {% macros.dbt.get_catalog_data() %} + + {% endoperation %} + + +example: + {% macro get_catalog() %} + + {% call statement(name='first_query') %} + + select ... + + {% endcall %} + + {% call statement(name='second_query') %} + + select ... + + {% endcall %} + + + {% set df_1 = get_results('first_query')['table'] %} + {% set df_2 = get_results('second_query')['table'] %} + + {{ return({"tables": df_1, "columns": df_2}) }} + + {% endmacro %} + + +clients/jinja.py \ No newline at end of file diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 19941fa696f..abcd8e380e5 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -4,6 +4,7 @@ from dbt.adapters.factory import get_adapter from dbt.clients.system import write_file from dbt.compat import bigint +from dbt.include import GLOBAL_DBT_MODULES_PATH from dbt.node_runners import OperationRunner from dbt.node_types import NodeType from dbt.runner import RunManager @@ -95,10 +96,13 @@ def unflatten(columns): # derive from BaseTask as I don't really want any result interpretation. class GenerateTask(BaseTask): def get_all_projects(self): - # TODO: I copy+pasted this from dbt/compilation.py, refactor! root_project = self.project.cfg all_projects = {root_project.get('name'): root_project} - dependency_projects = dbt.utils.dependency_projects(self.project) + # we only need to load the global deps. We haven't compiled, so our + # project['module-path'] does not exist. + dependency_projects = dbt.utils.dependencies_for_path( + self.project, GLOBAL_DBT_MODULES_PATH + ) for project in dependency_projects: name = project.cfg.get('name', 'unknown') @@ -110,7 +114,6 @@ def get_all_projects(self): return all_projects def run(self): - profile = self.project.run_environment() # From dbt/compilation.py @@ -119,7 +122,7 @@ def run(self): manifest = dbt.loader.GraphLoader.load_all(root_project, all_projects) flat_graph = manifest.to_flat_graph() - operation_name = 'adapters.catalog.get_catalog' # TODO: is it? + operation_name = 'get_catalog_data' operation = dbt.utils.get_operation(flat_graph, operation_name) adapter = get_adapter(profile) diff --git a/dbt/utils.py b/dbt/utils.py index 544fc0193f0..60195ef10e4 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -175,13 +175,16 @@ def get_materialization_macro(flat_graph, materialization_name, return macro -def get_operation_name(operation_name): - return 'operation_{}'.format(operation_name) +def get_operation_name(operation_name, with_prefix=True): + if with_prefix: + return get_dbt_macro_name(operation_name) + else: + return operation_name def get_operation(flat_graph, operation_name): - macro_name = get_operation_name(operation_name) - return find_operation_by_name(flat_graph, macro_name, None) + name = get_operation_name(operation_name, with_prefix=False) + return find_operation_by_name(flat_graph, name, None) def load_project_with_profile(source_project, project_dir): @@ -193,33 +196,38 @@ def load_project_with_profile(source_project, project_dir): args=source_project.args) -def dependency_projects(project): +def dependencies_for_path(project, module_path): + """Given a module path, yield all dependencies in that path.""" import dbt.project + logger.debug("Loading dependency project from {}".format(module_path)) + + for obj in os.listdir(module_path): + full_obj = os.path.join(module_path, obj) + + if not os.path.isdir(full_obj) or obj.startswith('__'): + # exclude non-dirs and dirs that start with __ + # the latter could be something like __pycache__ + # for the global dbt modules dir + continue + + try: + yield load_project_with_profile(project, full_obj) + except dbt.project.DbtProjectError as e: + logger.info( + "Error reading dependency project at {}".format( + full_obj) + ) + logger.info(str(e)) + + +def dependency_projects(project): module_paths = [ GLOBAL_DBT_MODULES_PATH, os.path.join(project['project-root'], project['modules-path']) ] for module_path in module_paths: - logger.debug("Loading dependency project from {}".format(module_path)) - - for obj in os.listdir(module_path): - full_obj = os.path.join(module_path, obj) - - if not os.path.isdir(full_obj) or obj.startswith('__'): - # exclude non-dirs and dirs that start with __ - # the latter could be something like __pycache__ - # for the global dbt modules dir - continue - - try: - yield load_project_with_profile(project, full_obj) - except dbt.project.DbtProjectError as e: - logger.info( - "Error reading dependency project at {}".format( - full_obj) - ) - logger.info(str(e)) + yield from dependencies_for_path(project, module_path) def split_path(path): From b26ac2cae53061153b29efe2fbfb910a855a2bc4 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 15:20:08 -0600 Subject: [PATCH 26/43] Removed a bunch of TODOs/etc from development --- dbt/context/common.py | 2 - .../global_project/macros/adapters/common.sql | 40 +------------------ .../adapters/catalog/get_catalog.sql | 2 - dbt/node_runners.py | 18 +++------ 4 files changed, 6 insertions(+), 56 deletions(-) diff --git a/dbt/context/common.py b/dbt/context/common.py index 30b89f25c1c..348d7d4e117 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -152,7 +152,6 @@ def _env_var(var, default=None): def _store_result(sql_results): def call(name, status, agate_table=None): - print('in store_result call, name={}'.format(name)) if agate_table is None: agate_table = dbt.clients.agate_helper.empty_table() @@ -168,7 +167,6 @@ def call(name, status, agate_table=None): def _load_result(sql_results): def call(name): - print('sql_results={}'.format(sql_results)) return sql_results.get(name) return call diff --git a/dbt/include/global_project/macros/adapters/common.sql b/dbt/include/global_project/macros/adapters/common.sql index 4b1bbcc5432..2d761d3307b 100644 --- a/dbt/include/global_project/macros/adapters/common.sql +++ b/dbt/include/global_project/macros/adapters/common.sql @@ -83,46 +83,8 @@ {{ exceptions.raise_compiler_error(msg) }} {% endmacro %} -{% macro postgres__get_catalog() %} - {%- set sql = render("with tables as ( - select - table_schema, - table_name, - table_type - - from information_schema.tables - - ), - - columns as ( - - select - table_schema, - table_name, - null as table_comment, - - column_name, - ordinal_position as column_index, - data_type as column_type, - null as column_comment - - - from information_schema.columns - - ) - - select * - from tables - join columns using (table_schema, table_name) - - where table_schema != 'information_schema' - and table_schema not like 'pg_%'") -%} - - {%- set status, res = adapter.execute(sql, auto_begin=True, fetch=True) -%} - {{ store_result('catalog', status=status, agate_table=res) }} -{% endmacro %} -{% macro postgres__get_catalog_notworking() -%} +{% macro postgres__get_catalog() -%} {%- call statement('catalog', fetch_result=True) -%} with tables as ( select diff --git a/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql b/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql index 1170b7da5dc..e6c6fa0c256 100644 --- a/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql +++ b/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql @@ -1,6 +1,4 @@ {% operation get_catalog_data %} {% set catalog = dbt.get_catalog() %} - {{ log('ran catalog query:', info=True)}} - {{ log(catalog, info=True) }} {{ return(catalog) }} {% endoperation %} \ No newline at end of file diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 3f7cb504005..786ac602827 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -548,28 +548,20 @@ def before_execute(self): pass def execute(self, model, flat_graph): - context = dbt.context.parser.generate( + # even though we haven't compiled, use `runtime.generate` so we can + # execute SQL... + context = dbt.context.runtime.generate( model.serialize(), self.project.cfg, flat_graph ) - # TODO: this is probably wrong. I made this up b/c it sounds nice operation_name = model['name'] - import pdb;pdb.set_trace() operation = dbt.utils.get_operation(flat_graph, operation_name) operation.generator(context)() - # TODO: this is probably wrong too, copied from CompileRunner. - # I think we need to end up passing the agate table out, need to - # understand how that works (and where status comes in now, I don't - # get that). - result = context['load_result']('catalog').table - # TODO: thing I also don't understand. If we switch to RunManager, - # etc, how can I get the result back out? Callers (safe_run, etc) care - # about RunModelResult having a compiled node object. May need to - # refactor safe_run or subclass RunModelResult? + result = context['load_result']('catalog') # get the schemas to filter by, though it seems like the caller should # really do this as another op and then combine the info approriately @@ -582,7 +574,7 @@ def execute(self, model, flat_graph): finally: adapter.cleanup_connections() - result = result.where(lambda r: r['table_schema'] in schemas) + result = result.table.where(lambda r: r['table_schema'] in schemas) return RunOperationResult(model, returned=result) def after_execute(self, result): From 26221947ddd49cf55758b9380ce1d05f9323474b Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 15:21:50 -0600 Subject: [PATCH 27/43] More todos/bad file adds --- dbt/context/common.py | 3 --- dbt/plan.txt | 51 ------------------------------------------- 2 files changed, 54 deletions(-) delete mode 100644 dbt/plan.txt diff --git a/dbt/context/common.py b/dbt/context/common.py index 348d7d4e117..0cb470326aa 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -100,9 +100,6 @@ def _add_macros(context, model, flat_graph): return context -# TODO: add _add_operations here? - - def _add_tracking(context): if dbt.tracking.active_user is not None: context = dbt.utils.merge(context, { diff --git a/dbt/plan.txt b/dbt/plan.txt deleted file mode 100644 index e1ac8f021a7..00000000000 --- a/dbt/plan.txt +++ /dev/null @@ -1,51 +0,0 @@ -Code plan: - - add an OperationRunner. - TODO: do I really? I may be able to just use ModelRunner w/ no changes - - maybe need an OperationLoader? - - I think so. Should it be part of 'nodes' or does it need to be its own thing? - - Toss it in 'nodes' for now... - - wire up a thing in include/global_project/operations/adapters/postgres.sql, or something? - - Modify the generate task's run() to do a parse run against the models first - - use that parsed manifest to generate the flat map, etc - - use manifest to get the list of schemas - - flat map has the sql inside its operations -> where does that enter the call chain? - -Macro/Operation execute statement or something - -Make a new get_catalog macro, have operation type I create, have the operation call out to macros, do the adapter-specific work inside macros (I think) - -In dbt/clients/jinja.py, MaterializationExtension - change this to 'OperationExtension' or whatever, -and it's pretty similar. Then we can do a jinja sql thingy with: - - {% operation get_catalog_data %} - - {% macros.dbt.get_catalog_data() %} - - {% endoperation %} - - -example: - {% macro get_catalog() %} - - {% call statement(name='first_query') %} - - select ... - - {% endcall %} - - {% call statement(name='second_query') %} - - select ... - - {% endcall %} - - - {% set df_1 = get_results('first_query')['table'] %} - {% set df_2 = get_results('second_query')['table'] %} - - {{ return({"tables": df_1, "columns": df_2}) }} - - {% endmacro %} - - -clients/jinja.py \ No newline at end of file From 1b03e36f115b9bfc973aba89f77a7bd6181e2fdc Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 15:25:18 -0600 Subject: [PATCH 28/43] Remove this method as it is now handled in sql/operations --- dbt/adapters/default/impl.py | 10 -------- dbt/adapters/postgres/impl.py | 46 ----------------------------------- dbt/parser.py | 2 +- dbt/task/generate.py | 3 +-- 4 files changed, 2 insertions(+), 59 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index f97e96be419..9d14dd02f70 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -270,16 +270,6 @@ def get_columns_in_table(cls, profile, project_cfg, schema_name, def _table_columns_to_dict(cls, columns): return {col.name: col for col in columns} - @classmethod - def get_catalog_for_schemas(cls, profile, schemas=None): - """Get the catalog information for a given profile and list of - schemas. If no schemas are given, return all schemas. - Returns a list of dictionaries, each one representing a single column. - - Required keys in the output: table_schema, table_name, column_index - """ - raise NotImplementedError() - @classmethod def expand_target_column_types(cls, profile, project_cfg, temp_table, diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index 86509a24062..17797dc9fe3 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -177,52 +177,6 @@ def check_schema_exists(cls, profile, project, schema, model_name=None): return results[0] > 0 - @classmethod - def get_catalog_for_schemas(cls, profile, schemas=None): - # TODO: we want to move this into separate files that users can modify - # so they can generate their own. - sql = """ - with tables as ( - select - table_schema, - table_name, - table_type - - from information_schema.tables - - ), - - columns as ( - - select - table_schema, - table_name, - null as table_comment, - - column_name, - ordinal_position as column_index, - data_type as column_type, - null as column_comment - - - from information_schema.columns - - ) - - select * - from tables - join columns using (table_schema, table_name) - - where table_schema != 'information_schema' - and table_schema not like 'pg_%' - """.strip() - _, results = cls.execute(profile, sql, fetch=True) - # ok, have an agate.Table now consisting of my results. Filter out - # schemas if necessary (this should probably happen in SQL, right?). - if schemas is not None: - results = results.where(lambda r: r['table_schema'] in schemas) - return [dict(zip(results.column_names, row)) for row in results] - @classmethod def convert_text_type(cls, agate_table, col_idx): return "text" diff --git a/dbt/parser.py b/dbt/parser.py index dff70bbd12e..b82cb23832d 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -126,7 +126,7 @@ def parse_macro_file(macro_file_path, package_name, resource_type, tags=None, - context=None,): + context=None): logger.debug("Parsing {}".format(macro_file_path)) diff --git a/dbt/task/generate.py b/dbt/task/generate.py index abcd8e380e5..d453626b082 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -64,8 +64,7 @@ def unflatten(columns): } } - Note: the docstring for DefaultAdapter.get_catalog_for_schemas discusses - what keys are guaranteed to exist. This method makes use of those keys. + Required keys in each column: table_schema, table_name, column_index Keys prefixed with 'column_' end up in per-column data and keys prefixed with 'table_' end up in table metadata. Keys without either prefix are From 9ec5d7085fac291dc3f46a0aa889c49e2da59c9f Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 15:26:18 -0600 Subject: [PATCH 29/43] forgot 'yield from' is py3 only --- dbt/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/utils.py b/dbt/utils.py index 60195ef10e4..33d126a0566 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -227,7 +227,8 @@ def dependency_projects(project): ] for module_path in module_paths: - yield from dependencies_for_path(project, module_path) + for entry in dependencies_for_path(project, module_path): + yield entry def split_path(path): From f1a6461f0918ffbe9ef15802a049b25490488673 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 15:42:35 -0600 Subject: [PATCH 30/43] Remove useless return statement, add a comment instead --- dbt/include/global_project/macros/adapters/common.sql | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dbt/include/global_project/macros/adapters/common.sql b/dbt/include/global_project/macros/adapters/common.sql index 2d761d3307b..7b09ba1bbbb 100644 --- a/dbt/include/global_project/macros/adapters/common.sql +++ b/dbt/include/global_project/macros/adapters/common.sql @@ -120,9 +120,7 @@ where table_schema != 'information_schema' and table_schema not like 'pg_%' {%- endcall -%} - {{ log('trying to load results?', info=True) }} - {% set catalog = load_result('catalog') %} - {{ log(catalog, info=True) }} - {{ return(catalog['table']) }} - + {# There's no point in returning anything as the jinja macro stuff calls #} + {# str() on all returns. To get the results, you'll need to use #} + {# context['load_result']('catalog') #} {%- endmacro %} From d0a7ee0d9cf7f2a864be4e7d89fef585437bfda1 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 15:54:30 -0600 Subject: [PATCH 31/43] Add a note about resource type overlaps in parsed nodes/macros --- dbt/contracts/graph/unparsed.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbt/contracts/graph/unparsed.py b/dbt/contracts/graph/unparsed.py index 7c5b8ded8f8..2d59a7d8259 100644 --- a/dbt/contracts/graph/unparsed.py +++ b/dbt/contracts/graph/unparsed.py @@ -63,8 +63,9 @@ NodeType.Model, NodeType.Test, NodeType.Analysis, - # TODO: added this to ParsedMacro, does that mean we don't - # need it here anymore? + # Note: Hooks fail if you remove this, even though it's + # also allowed in ParsedMacro, which seems wrong. + # Maybe need to move hook operations into macros? NodeType.Operation, NodeType.Seed, # we need this if parse_node is going to handle archives. From 3d7ed8e170698ab80fe92b30b4ecbd882baedb37 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 15:56:35 -0600 Subject: [PATCH 32/43] add newline --- .../global_project/operations/adapters/catalog/get_catalog.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql b/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql index e6c6fa0c256..4371416adc6 100644 --- a/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql +++ b/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql @@ -1,4 +1,4 @@ {% operation get_catalog_data %} {% set catalog = dbt.get_catalog() %} {{ return(catalog) }} -{% endoperation %} \ No newline at end of file +{% endoperation %} From 55d98f7c352946c4b0a85889af2c47a3ef90b52d Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 15:57:35 -0600 Subject: [PATCH 33/43] Didn't need this, I guess --- dbt/task/operation.py | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 dbt/task/operation.py diff --git a/dbt/task/operation.py b/dbt/task/operation.py deleted file mode 100644 index 68d38b0e6a4..00000000000 --- a/dbt/task/operation.py +++ /dev/null @@ -1,23 +0,0 @@ -from dbt.node_runners import OperationRunner -from dbt.node_types import NodeType -from dbt.runner import RunManager -from dbt.task.base_task import RunnableTask -import dbt.ui.printer - - -class OperationTask(RunnableTask): - def run(self): - runner = RunManager( - self.project, - self.project["target-path"], - self.args, - ) - query = { - "include": ["*"], - "exclude": [], - "resource_types": [NodeType.Operation], - } - results = runner.run_flat(query, OperationRunner) - - dbt.ui.printer.print_run_end_messages(results) - return results From 9324a2cc2e196c979be6bc110dd96837941a24dc Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 18:43:29 -0600 Subject: [PATCH 34/43] Move some things around and break out some methods --- dbt/clients/jinja.py | 2 +- dbt/node_runners.py | 27 +++++++++++++++++---------- dbt/task/generate.py | 28 ++++++++++++++++++++-------- dbt/utils.py | 6 +++--- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/dbt/clients/jinja.py b/dbt/clients/jinja.py index 4fec265de14..2e89e0170f4 100644 --- a/dbt/clients/jinja.py +++ b/dbt/clients/jinja.py @@ -114,7 +114,7 @@ def parse(self, parser): while parser.stream.skip_if('comma'): target = parser.parse_assign_target(name_only=True) - node.name = dbt.utils.get_operation_name(operation_name) + node.name = dbt.utils.get_operation_macro_name(operation_name) node.body = parser.parse_statements(('name:endoperation',), drop_needle=True) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 786ac602827..8a8740d60ca 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -549,30 +549,37 @@ def before_execute(self): def execute(self, model, flat_graph): # even though we haven't compiled, use `runtime.generate` so we can - # execute SQL... + # execute SQL. + # TODO: make runtime.generate() support the ParsedMacro model. I think + # this is a problem because generate() expects a Node for the model, + # not a Macro. context = dbt.context.runtime.generate( model.serialize(), self.project.cfg, flat_graph ) - operation_name = model['name'] - operation = dbt.utils.get_operation(flat_graph, operation_name) + operation = dbt.utils.get_operation_macro(flat_graph, model.name) + # TODO: should I get the return value here in case future operations + # want to return some string? Jinja (I think) stringifies the results + # so it's not super useful. Status, I guess? operation.generator(context)() + # This is a lot of magic, have to know the magic name is 'catalog'. + # how can we make this part of the data set? Could we make it the + # operation's name/unique ID somehow instead? + # result is going to be an AttrDict with attributes 'table', 'data', + # and 'status'. result = context['load_result']('catalog') # get the schemas to filter by, though it seems like the caller should # really do this as another op and then combine the info approriately # TODO: check on this - adapter = get_adapter(self.profile) - - try: - schemas = adapter.get_existing_schemas(self.profile, self.project) - adapter.release_connection(self.profile) - finally: - adapter.cleanup_connections() + schemas = self.adapter.get_existing_schemas( + self.profile, + self.project + ) result = result.table.where(lambda r: r['table_schema'] in schemas) return RunOperationResult(model, returned=result) diff --git a/dbt/task/generate.py b/dbt/task/generate.py index d453626b082..989a5f481cf 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -14,6 +14,7 @@ CATALOG_FILENAME = 'catalog.json' +OPERATION_NAME = 'get_catalog_data' def get_stripped_prefix(source, prefix): @@ -112,23 +113,34 @@ def get_all_projects(self): return all_projects - def run(self): - profile = self.project.run_environment() - - # From dbt/compilation.py + def _get_flat_graph(self): + # TODO: I'd like to do this better. We can't use + # utils.dependency_projects because it assumes you have compiled your + # project (I think?) - it assumes that you have an existing and + # populated project['modules-path'], but 'catalog generate' shouldn't + # require that. It might be better to suppress the exception in + # dependency_projects if that's reasonable, or make it a flag. root_project = self.project.cfg all_projects = self.get_all_projects() manifest = dbt.loader.GraphLoader.load_all(root_project, all_projects) - flat_graph = manifest.to_flat_graph() - operation_name = 'get_catalog_data' - operation = dbt.utils.get_operation(flat_graph, operation_name) + return manifest.to_flat_graph() - adapter = get_adapter(profile) + def _get_adapter(self): + profile = self.project.run_environment() + return get_adapter(profile) + def run(self): + flat_graph = self._get_flat_graph() + operation = dbt.utils.get_operation_macro(flat_graph, OPERATION_NAME) + adapter = self._get_adapter() + + # TODO: this really feels like it should be along the lines of + # agate_table = adapter.run_operation(flat_graph, operation) runner = OperationRunner(self.project, adapter, operation, 0, 0) run_result = runner.safe_run(flat_graph) agate_table = run_result.returned + results = [ dict(zip(agate_table.column_names, row)) for row in agate_table diff --git a/dbt/utils.py b/dbt/utils.py index 33d126a0566..f80d7e05351 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -175,15 +175,15 @@ def get_materialization_macro(flat_graph, materialization_name, return macro -def get_operation_name(operation_name, with_prefix=True): +def get_operation_macro_name(operation_name, with_prefix=True): if with_prefix: return get_dbt_macro_name(operation_name) else: return operation_name -def get_operation(flat_graph, operation_name): - name = get_operation_name(operation_name, with_prefix=False) +def get_operation_macro(flat_graph, operation_name): + name = get_operation_macro_name(operation_name, with_prefix=False) return find_operation_by_name(flat_graph, name, None) From 5216b5160602f1a2a557eff6a55e39c326f14895 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 14 May 2018 19:08:11 -0600 Subject: [PATCH 35/43] More refactoring, I do not believe we need OperationRunner here --- dbt/node_runners.py | 48 --------------------------------- dbt/task/generate.py | 63 +++++++++++++++++++++++++++++++++----------- 2 files changed, 48 insertions(+), 63 deletions(-) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 8a8740d60ca..432a1895dd2 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -538,51 +538,3 @@ def print_result_line(self, result): self.node_index, self.num_nodes) - -class OperationRunner(ModelRunner): - def compile(self, flat_graph): - # Operations don't compile, I think - return self.node - - def before_execute(self): - pass - - def execute(self, model, flat_graph): - # even though we haven't compiled, use `runtime.generate` so we can - # execute SQL. - # TODO: make runtime.generate() support the ParsedMacro model. I think - # this is a problem because generate() expects a Node for the model, - # not a Macro. - context = dbt.context.runtime.generate( - model.serialize(), - self.project.cfg, - flat_graph - ) - - operation = dbt.utils.get_operation_macro(flat_graph, model.name) - - # TODO: should I get the return value here in case future operations - # want to return some string? Jinja (I think) stringifies the results - # so it's not super useful. Status, I guess? - operation.generator(context)() - - # This is a lot of magic, have to know the magic name is 'catalog'. - # how can we make this part of the data set? Could we make it the - # operation's name/unique ID somehow instead? - # result is going to be an AttrDict with attributes 'table', 'data', - # and 'status'. - result = context['load_result']('catalog') - - # get the schemas to filter by, though it seems like the caller should - # really do this as another op and then combine the info approriately - # TODO: check on this - schemas = self.adapter.get_existing_schemas( - self.profile, - self.project - ) - - result = result.table.where(lambda r: r['table_schema'] in schemas) - return RunOperationResult(model, returned=result) - - def after_execute(self, result): - pass diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 989a5f481cf..8f6efeab60c 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -5,16 +5,15 @@ from dbt.clients.system import write_file from dbt.compat import bigint from dbt.include import GLOBAL_DBT_MODULES_PATH -from dbt.node_runners import OperationRunner from dbt.node_types import NodeType -from dbt.runner import RunManager import dbt.ui.printer +import dbt.utils from dbt.task.base_task import BaseTask CATALOG_FILENAME = 'catalog.json' -OPERATION_NAME = 'get_catalog_data' +GET_CATALOG_OPERATION_NAME = 'get_catalog_data' def get_stripped_prefix(source, prefix): @@ -126,24 +125,58 @@ def _get_flat_graph(self): manifest = dbt.loader.GraphLoader.load_all(root_project, all_projects) return manifest.to_flat_graph() - def _get_adapter(self): - profile = self.project.run_environment() - return get_adapter(profile) + def _execute_operation(self, flat_graph, operation_name): + """ + Run the given operation. Operation is a ParsedMacro with a + resource_type of NodeType.Operation. + + Return an an AttrDict with three attributes: 'table', 'data', and + 'status'. 'table' is an agate.Table. + """ + operation = dbt.utils.get_operation_macro(flat_graph, operation_name) + + # TODO: make runtime.generate() support the ParsedMacro model. I think + # this is a problem because generate() expects a Node for the model, + # not a Macro. + context = dbt.context.runtime.generate( + operation.serialize(), + self.project.cfg, + flat_graph + ) + + # TODO: should I get the return value here in case future operations + # want to return some string? Jinja (I think) stringifies the results + # so it's not super useful. Status, I guess? + operation.generator(context)() + + # This is a lot of magic, have to know the magic name is 'catalog'. + # TODO: How can we make this part of the data set? Could we make it + # the operation's name/unique ID somehow instead? + result = context['load_result']('catalog') + return result def run(self): flat_graph = self._get_flat_graph() - operation = dbt.utils.get_operation_macro(flat_graph, OPERATION_NAME) - adapter = self._get_adapter() + profile = self.project.run_environment() + adapter = get_adapter(profile) + + # TODO: is there some way to have this be a method on the adapter? + # I think the only way that's possible is if operations are pure SQL, + # or adapter is modified to understand the flat graph. As it is, the + # operation has to manipulate the adapter. + results = self._execute_operation( + flat_graph, + GET_CATALOG_OPERATION_NAME + ) - # TODO: this really feels like it should be along the lines of - # agate_table = adapter.run_operation(flat_graph, operation) - runner = OperationRunner(self.project, adapter, operation, 0, 0) - run_result = runner.safe_run(flat_graph) - agate_table = run_result.returned + # get the schemas to filter by. TODO: should this be an 'operation' + # as well? + schemas = adapter.get_existing_schemas(profile, self.project) + results = results.table.where(lambda r: r['table_schema'] in schemas) results = [ - dict(zip(agate_table.column_names, row)) - for row in agate_table + dict(zip(results.column_names, row)) + for row in results ] results = unflatten(results) From 726aaad154aa79515d7d59b17daeed0529156e2a Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 15 May 2018 00:25:00 -0600 Subject: [PATCH 36/43] move operations into macros --- dbt/include/global_project/dbt_project.yml | 1 - .../adapters => macros/operations}/catalog/get_catalog.sql | 0 dbt/loader.py | 2 +- dbt/node_runners.py | 1 - dbt/project.py | 1 - sample.dbt_project.yml | 5 ----- 6 files changed, 1 insertion(+), 9 deletions(-) rename dbt/include/global_project/{operations/adapters => macros/operations}/catalog/get_catalog.sql (100%) diff --git a/dbt/include/global_project/dbt_project.yml b/dbt/include/global_project/dbt_project.yml index 8d43087eb27..cf30f7f9184 100644 --- a/dbt/include/global_project/dbt_project.yml +++ b/dbt/include/global_project/dbt_project.yml @@ -3,4 +3,3 @@ name: dbt version: 1.0 macro-paths: ["macros"] -operation-paths: ["operations"] diff --git a/dbt/include/global_project/operations/adapters/catalog/get_catalog.sql b/dbt/include/global_project/macros/operations/catalog/get_catalog.sql similarity index 100% rename from dbt/include/global_project/operations/adapters/catalog/get_catalog.sql rename to dbt/include/global_project/macros/operations/catalog/get_catalog.sql diff --git a/dbt/loader.py b/dbt/loader.py index 448e1d1fe16..6e28147484f 100644 --- a/dbt/loader.py +++ b/dbt/loader.py @@ -96,7 +96,7 @@ def load_project(cls, root_project, all_projects, project, project_name, root_project=root_project, all_projects=all_projects, root_dir=project.get('project-root'), - relative_dirs=project.get('operation-paths', []), + relative_dirs=project.get('macro-paths', []), resource_type=NodeType.Operation) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 432a1895dd2..89cf8a5bfc2 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -537,4 +537,3 @@ def print_result_line(self, result): schema_name, self.node_index, self.num_nodes) - diff --git a/dbt/project.py b/dbt/project.py index ca36c1d9124..ca1ac3535ab 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -21,7 +21,6 @@ default_project_cfg = { 'source-paths': ['models'], 'macro-paths': ['macros'], - 'operation-paths': ['operations'], 'data-paths': ['data'], 'test-paths': ['test'], 'target-path': 'target', diff --git a/sample.dbt_project.yml b/sample.dbt_project.yml index 987eed3ea06..d581b70b826 100644 --- a/sample.dbt_project.yml +++ b/sample.dbt_project.yml @@ -50,11 +50,6 @@ data-paths: ["data"] # macros. These macros will be globally available to all models in your project macro-paths: ['macros'] -# operation-paths: Optional. Specify which path(s) dbt should look in to find -# operations. These macros will be globally available to all models in your -# project -operation-paths: ['operations'] - # log-path: Optional. Specify which path dbt should write debug logs to. log-path: "logs" From fda7013d262e3461917dc60399cef8f41650f7f6 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 15 May 2018 22:45:21 -0600 Subject: [PATCH 37/43] first part of making the adapter responsible for this stuff --- dbt/adapters/default/impl.py | 44 ++++++++++++++++++++++++++++++++++ dbt/adapters/postgres/impl.py | 11 +++++++++ dbt/task/generate.py | 45 +---------------------------------- 3 files changed, 56 insertions(+), 44 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 9d14dd02f70..54fb3717fdb 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -9,6 +9,7 @@ import dbt.flags import dbt.schema import dbt.clients.agate_helper +import dbt.context.runtime from dbt.contracts.connection import Connection from dbt.logger import GLOBAL_LOGGER as logger @@ -773,3 +774,46 @@ def convert_agate_type(cls, agate_table, col_idx): for agate_cls, func in conversions: if isinstance(agate_type, agate_cls): return func(agate_table, col_idx) + + + ### + # Operations involving the flat graph + ### + @classmethod + def run_operation(cls, profile, project_cfg, flat_graph, operation_name): + """Look the operation identified by operation_name up in the flat + graph and run it. + + Return an an AttrDict with three attributes: 'table', 'data', and + 'status'. 'table' is an agate.Table. + """ + operation = dbt.utils.get_operation_macro(flat_graph, operation_name) + + # TODO: make runtime.generate() support the ParsedMacro model. I think + # this is a problem because generate() expects a Node for the model, + # not a Macro. + context = dbt.context.runtime.generate( + operation.serialize(), + project_cfg, + flat_graph + ) + + # TODO: should I get the return value here in case future operations + # want to return some string? Jinja (I think) stringifies the results + # so it's not super useful. Status, I guess? + operation.generator(context)() + + # This is a lot of magic, have to know the magic name is 'catalog'. + # TODO: How can we make this part of the data set? Could we make it + # the operation's name/unique ID somehow instead? + result = context['load_result']('catalog') + return result + + + ### + # Abstract methods involving the flat graph + ### + @classmethod + def get_catalog(cls, profile, project_cfg, flat_graph): + raise dbt.exceptions.NotImplementedException( + '`get_catalog` is not implemented for this adapter!') diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index 17797dc9fe3..f250845bbb4 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -9,6 +9,7 @@ from dbt.logger import GLOBAL_LOGGER as logger +GET_CATALOG_OPERATION_NAME = 'get_catalog_data' class PostgresAdapter(dbt.adapters.default.DefaultAdapter): @@ -201,3 +202,13 @@ def convert_date_type(cls, agate_table, col_idx): @classmethod def convert_time_type(cls, agate_table, col_idx): return "time" + + @classmethod + def get_catalog(cls, profile, project_cfg, flat_graph): + results = cls.run_operation(profile, project_cfg, flat_graph, + GET_CATALOG_OPERATION_NAME) + + schemas = cls.get_existing_schemas(profile, project_cfg) + results = results.table.where(lambda r: r['table_schema'] in schemas) + + return results diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 8f6efeab60c..091f203adbd 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -13,7 +13,6 @@ CATALOG_FILENAME = 'catalog.json' -GET_CATALOG_OPERATION_NAME = 'get_catalog_data' def get_stripped_prefix(source, prefix): @@ -125,54 +124,12 @@ def _get_flat_graph(self): manifest = dbt.loader.GraphLoader.load_all(root_project, all_projects) return manifest.to_flat_graph() - def _execute_operation(self, flat_graph, operation_name): - """ - Run the given operation. Operation is a ParsedMacro with a - resource_type of NodeType.Operation. - - Return an an AttrDict with three attributes: 'table', 'data', and - 'status'. 'table' is an agate.Table. - """ - operation = dbt.utils.get_operation_macro(flat_graph, operation_name) - - # TODO: make runtime.generate() support the ParsedMacro model. I think - # this is a problem because generate() expects a Node for the model, - # not a Macro. - context = dbt.context.runtime.generate( - operation.serialize(), - self.project.cfg, - flat_graph - ) - - # TODO: should I get the return value here in case future operations - # want to return some string? Jinja (I think) stringifies the results - # so it's not super useful. Status, I guess? - operation.generator(context)() - - # This is a lot of magic, have to know the magic name is 'catalog'. - # TODO: How can we make this part of the data set? Could we make it - # the operation's name/unique ID somehow instead? - result = context['load_result']('catalog') - return result - def run(self): flat_graph = self._get_flat_graph() profile = self.project.run_environment() adapter = get_adapter(profile) - # TODO: is there some way to have this be a method on the adapter? - # I think the only way that's possible is if operations are pure SQL, - # or adapter is modified to understand the flat graph. As it is, the - # operation has to manipulate the adapter. - results = self._execute_operation( - flat_graph, - GET_CATALOG_OPERATION_NAME - ) - - # get the schemas to filter by. TODO: should this be an 'operation' - # as well? - schemas = adapter.get_existing_schemas(profile, self.project) - results = results.table.where(lambda r: r['table_schema'] in schemas) + results = adapter.get_catalog(profile, self.project.cfg, flat_graph) results = [ dict(zip(results.column_names, row)) From 15fe9a3b4b6dfb0e0603a6508e2f8dae9d89f577 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 16 May 2018 07:39:20 -0600 Subject: [PATCH 38/43] Make the adapter operate on an actual ParsedManifest instead of the flat graph --- dbt/adapters/default/impl.py | 20 ++++++++++---------- dbt/adapters/postgres/impl.py | 5 +++-- dbt/context/common.py | 3 +++ dbt/contracts/graph/parsed.py | 23 +++++++++++++++++++++++ dbt/task/generate.py | 8 ++++---- dbt/utils.py | 18 +++++++++++++++++- 6 files changed, 60 insertions(+), 17 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 54fb3717fdb..25d2b1aaa2d 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -9,7 +9,6 @@ import dbt.flags import dbt.schema import dbt.clients.agate_helper -import dbt.context.runtime from dbt.contracts.connection import Connection from dbt.logger import GLOBAL_LOGGER as logger @@ -775,27 +774,29 @@ def convert_agate_type(cls, agate_table, col_idx): if isinstance(agate_type, agate_cls): return func(agate_table, col_idx) - ### - # Operations involving the flat graph + # Operations involving the manifest ### @classmethod - def run_operation(cls, profile, project_cfg, flat_graph, operation_name): - """Look the operation identified by operation_name up in the flat - graph and run it. + def run_operation(cls, profile, project_cfg, manifest, operation_name): + """Look the operation identified by operation_name up in the manifest + and run it. Return an an AttrDict with three attributes: 'table', 'data', and 'status'. 'table' is an agate.Table. """ - operation = dbt.utils.get_operation_macro(flat_graph, operation_name) + operation = manifest.find_operation_by_name(operation_name, None) # TODO: make runtime.generate() support the ParsedMacro model. I think # this is a problem because generate() expects a Node for the model, # not a Macro. + # This causes a reference cycle, as dbt.context.runtime.generate() + # ends up calling get_adapter, so the import has to be here. + import dbt.context.runtime context = dbt.context.runtime.generate( operation.serialize(), project_cfg, - flat_graph + manifest.to_flat_graph(), ) # TODO: should I get the return value here in case future operations @@ -809,11 +810,10 @@ def run_operation(cls, profile, project_cfg, flat_graph, operation_name): result = context['load_result']('catalog') return result - ### # Abstract methods involving the flat graph ### @classmethod - def get_catalog(cls, profile, project_cfg, flat_graph): + def get_catalog(cls, profile, project_cfg, run_operation): raise dbt.exceptions.NotImplementedException( '`get_catalog` is not implemented for this adapter!') diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index f250845bbb4..93c0a943630 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -11,6 +11,7 @@ GET_CATALOG_OPERATION_NAME = 'get_catalog_data' + class PostgresAdapter(dbt.adapters.default.DefaultAdapter): @classmethod @@ -204,8 +205,8 @@ def convert_time_type(cls, agate_table, col_idx): return "time" @classmethod - def get_catalog(cls, profile, project_cfg, flat_graph): - results = cls.run_operation(profile, project_cfg, flat_graph, + def get_catalog(cls, profile, project_cfg, manifest): + results = cls.run_operation(profile, project_cfg, manifest, GET_CATALOG_OPERATION_NAME) schemas = cls.get_existing_schemas(profile, project_cfg) diff --git a/dbt/context/common.py b/dbt/context/common.py index 0cb470326aa..054496890d8 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -4,6 +4,7 @@ from dbt.adapters.factory import get_adapter from dbt.compat import basestring, to_string +from dbt.node_types import NodeType import dbt.clients.jinja import dbt.clients.agate_helper @@ -75,6 +76,8 @@ def _add_macros(context, model, flat_graph): macros_to_add = {'global': [], 'local': []} for unique_id, macro in flat_graph.get('macros', {}).items(): + if macro.get('resource_type') != NodeType.Macro: + continue package_name = macro.get('package_name') macro_map = { diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index 8f38024048b..7569e430e9b 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -347,6 +347,29 @@ def serialize(self): 'child_map': forward_edges, } + def _find_by_name(self, name, package, subgraph, nodetype): + """ + + Find a node by its given name in the appropraite sugraph. + """ + if subgraph == 'nodes': + search = self.nodes + elif subgraph == 'macros': + search = self.macros + else: + raise NotImplementedError( + 'subgraph search for {} not implemented'.format(subgraph) + ) + return dbt.utils.find_in_subgraph_by_name( + search, + name, + package, + nodetype) + + def find_operation_by_name(self, name, package): + return self._find_by_name(name, package, 'macros', + [NodeType.Operation]) + def to_flat_graph(self): """Convert the parsed manifest to the 'flat graph' that the compiler expects. diff --git a/dbt/task/generate.py b/dbt/task/generate.py index 091f203adbd..c6127937d06 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -111,7 +111,7 @@ def get_all_projects(self): return all_projects - def _get_flat_graph(self): + def _get_manifest(self): # TODO: I'd like to do this better. We can't use # utils.dependency_projects because it assumes you have compiled your # project (I think?) - it assumes that you have an existing and @@ -122,14 +122,14 @@ def _get_flat_graph(self): all_projects = self.get_all_projects() manifest = dbt.loader.GraphLoader.load_all(root_project, all_projects) - return manifest.to_flat_graph() + return manifest def run(self): - flat_graph = self._get_flat_graph() + manifest = self._get_manifest() profile = self.project.run_environment() adapter = get_adapter(profile) - results = adapter.get_catalog(profile, self.project.cfg, flat_graph) + results = adapter.get_catalog(profile, self.project.cfg, manifest) results = [ dict(zip(results.column_names, row)) diff --git a/dbt/utils.py b/dbt/utils.py index f80d7e05351..311132480d8 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -114,7 +114,23 @@ def find_operation_by_name(flat_graph, target_name, target_package): def find_by_name(flat_graph, target_name, target_package, subgraph, nodetype): - for name, model in flat_graph.get(subgraph).items(): + return find_in_subgraph_by_name( + flat_graph.get(subgraph), + target_name, + target_package, + nodetype) + + +def find_in_subgraph_by_name(subgraph, target_name, target_package, nodetype): + """Find an entry in a subgraph by name. Any mapping that implements + .items() and maps unique id -> something can be used as the subgraph. + + Names are like: + '{nodetype}.{target_package}.{target_name}' + + You can use `None` for the package name as a wildcard. + """ + for name, model in subgraph.items(): node_parts = name.split('.') if len(node_parts) != 3: node_type = model.get('resource_type', 'node') From f6fb149c2d9813377e0acb43f3d84cfe56236902 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 16 May 2018 07:44:58 -0600 Subject: [PATCH 39/43] Have result data key be a variable --- dbt/adapters/default/impl.py | 5 +++-- dbt/adapters/postgres/impl.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 25d2b1aaa2d..0cb6a1fbab6 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -778,7 +778,8 @@ def convert_agate_type(cls, agate_table, col_idx): # Operations involving the manifest ### @classmethod - def run_operation(cls, profile, project_cfg, manifest, operation_name): + def run_operation(cls, profile, project_cfg, manifest, operation_name, + result_key): """Look the operation identified by operation_name up in the manifest and run it. @@ -807,7 +808,7 @@ def run_operation(cls, profile, project_cfg, manifest, operation_name): # This is a lot of magic, have to know the magic name is 'catalog'. # TODO: How can we make this part of the data set? Could we make it # the operation's name/unique ID somehow instead? - result = context['load_result']('catalog') + result = context['load_result'](result_key) return result ### diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index 93c0a943630..50c05b0f199 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -10,6 +10,7 @@ from dbt.logger import GLOBAL_LOGGER as logger GET_CATALOG_OPERATION_NAME = 'get_catalog_data' +GET_CATALOG_RESULT_KEY = 'catalog' # defined in get_catalog() macro class PostgresAdapter(dbt.adapters.default.DefaultAdapter): @@ -207,7 +208,8 @@ def convert_time_type(cls, agate_table, col_idx): @classmethod def get_catalog(cls, profile, project_cfg, manifest): results = cls.run_operation(profile, project_cfg, manifest, - GET_CATALOG_OPERATION_NAME) + GET_CATALOG_OPERATION_NAME, + GET_CATALOG_RESULT_KEY) schemas = cls.get_existing_schemas(profile, project_cfg) results = results.table.where(lambda r: r['table_schema'] in schemas) From 70da4287d404c20be3e018648f20a13dcfceaa63 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 16 May 2018 10:18:37 -0600 Subject: [PATCH 40/43] Make contexts accept ParsedMacro --- dbt/adapters/default/impl.py | 5 +---- dbt/context/common.py | 7 +++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 0cb6a1fbab6..07a36b3ecd5 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -788,14 +788,11 @@ def run_operation(cls, profile, project_cfg, manifest, operation_name, """ operation = manifest.find_operation_by_name(operation_name, None) - # TODO: make runtime.generate() support the ParsedMacro model. I think - # this is a problem because generate() expects a Node for the model, - # not a Macro. # This causes a reference cycle, as dbt.context.runtime.generate() # ends up calling get_adapter, so the import has to be here. import dbt.context.runtime context = dbt.context.runtime.generate( - operation.serialize(), + operation, project_cfg, manifest.to_flat_graph(), ) diff --git a/dbt/context/common.py b/dbt/context/common.py index 054496890d8..a1743690420 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -5,6 +5,7 @@ from dbt.adapters.factory import get_adapter from dbt.compat import basestring, to_string from dbt.node_types import NodeType +from dbt.contracts.graph.parsed import ParsedMacro, ParsedNode import dbt.clients.jinja import dbt.clients.agate_helper @@ -206,6 +207,12 @@ def __init__(self, model, context, overrides): if isinstance(model, dict) and model.get('unique_id'): local_vars = model.get('config', {}).get('vars', {}) self.model_name = model.get('name') + elif isinstance(model, ParsedMacro): + local_vars = {} # macros have no config + self.model_name = model.name + elif isinstance(model, ParsedNode): + local_vars = model.config.get('vars', {}) + self.model_name = model.name else: # still used for wrapping self.model_name = model.nice_name From 0dd237529725f8c25036d9b46682a39f80c24429 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 17 May 2018 13:30:44 -0600 Subject: [PATCH 41/43] PR feedback --- dbt/adapters/default/impl.py | 12 +++--------- dbt/main.py | 9 ++++----- dbt/node_runners.py | 12 ------------ dbt/task/generate.py | 1 - .../models/.gitkeep | 0 .../models/view_summary.sql | 0 .../seed.sql | 0 .../test_docs_generate.py} | 2 +- ...est_catalog_generate.py => test_docs_generate.py} | 0 9 files changed, 8 insertions(+), 28 deletions(-) rename test/integration/{029_catalog_generate_tests => 029_docs_generate_tests}/models/.gitkeep (100%) rename test/integration/{029_catalog_generate_tests => 029_docs_generate_tests}/models/view_summary.sql (100%) rename test/integration/{029_catalog_generate_tests => 029_docs_generate_tests}/seed.sql (100%) rename test/integration/{029_catalog_generate_tests/test_catalog_generate.py => 029_docs_generate_tests/test_docs_generate.py} (99%) rename test/unit/{test_catalog_generate.py => test_docs_generate.py} (100%) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 07a36b3ecd5..a2d0c81c63b 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -786,7 +786,7 @@ def run_operation(cls, profile, project_cfg, manifest, operation_name, Return an an AttrDict with three attributes: 'table', 'data', and 'status'. 'table' is an agate.Table. """ - operation = manifest.find_operation_by_name(operation_name, None) + operation = manifest.find_operation_by_name(operation_name, 'dbt') # This causes a reference cycle, as dbt.context.runtime.generate() # ends up calling get_adapter, so the import has to be here. @@ -797,21 +797,15 @@ def run_operation(cls, profile, project_cfg, manifest, operation_name, manifest.to_flat_graph(), ) - # TODO: should I get the return value here in case future operations - # want to return some string? Jinja (I think) stringifies the results - # so it's not super useful. Status, I guess? operation.generator(context)() - # This is a lot of magic, have to know the magic name is 'catalog'. - # TODO: How can we make this part of the data set? Could we make it - # the operation's name/unique ID somehow instead? result = context['load_result'](result_key) return result ### - # Abstract methods involving the flat graph + # Abstract methods involving the manifest ### @classmethod - def get_catalog(cls, profile, project_cfg, run_operation): + def get_catalog(cls, profile, project_cfg, manifest): raise dbt.exceptions.NotImplementedException( '`get_catalog` is not implemented for this adapter!') diff --git a/dbt/main.py b/dbt/main.py index 2ce78d59383..cb7d6add016 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -415,12 +415,11 @@ def parse_args(args): ) seed_sub.set_defaults(cls=seed_task.SeedTask, which='seed') - catalog_sub = subs.add_parser('catalog', parents=[base_subparser]) - catalog_subs = catalog_sub.add_subparsers() - # it might look like catalog_sub is the correct parents entry, but that + docs_sub = subs.add_parser('docs', parents=[base_subparser]) + docs_subs = docs_sub.add_subparsers() + # it might look like docs_sub is the correct parents entry, but that # will cause weird errors about 'conflicting option strings'. - generate_sub = catalog_subs.add_parser('generate', - parents=[base_subparser]) + generate_sub = docs_subs.add_parser('generate', parents=[base_subparser]) generate_sub.set_defaults(cls=generate_task.GenerateTask, which='generate') diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 89cf8a5bfc2..019c04245f6 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -69,18 +69,6 @@ def __init__(self, node, error=None, skip=False, status=None, failed, execution_time) self.returned = returned - @property - def errored(self): - return self.error is not None - - @property - def failed(self): - return self.fail - - @property - def skipped(self): - return self.skip - class BaseRunner(object): print_header = True diff --git a/dbt/task/generate.py b/dbt/task/generate.py index c6127937d06..6c99ae2239c 100644 --- a/dbt/task/generate.py +++ b/dbt/task/generate.py @@ -91,7 +91,6 @@ def unflatten(columns): return structured -# derive from BaseTask as I don't really want any result interpretation. class GenerateTask(BaseTask): def get_all_projects(self): root_project = self.project.cfg diff --git a/test/integration/029_catalog_generate_tests/models/.gitkeep b/test/integration/029_docs_generate_tests/models/.gitkeep similarity index 100% rename from test/integration/029_catalog_generate_tests/models/.gitkeep rename to test/integration/029_docs_generate_tests/models/.gitkeep diff --git a/test/integration/029_catalog_generate_tests/models/view_summary.sql b/test/integration/029_docs_generate_tests/models/view_summary.sql similarity index 100% rename from test/integration/029_catalog_generate_tests/models/view_summary.sql rename to test/integration/029_docs_generate_tests/models/view_summary.sql diff --git a/test/integration/029_catalog_generate_tests/seed.sql b/test/integration/029_docs_generate_tests/seed.sql similarity index 100% rename from test/integration/029_catalog_generate_tests/seed.sql rename to test/integration/029_docs_generate_tests/seed.sql diff --git a/test/integration/029_catalog_generate_tests/test_catalog_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py similarity index 99% rename from test/integration/029_catalog_generate_tests/test_catalog_generate.py rename to test/integration/029_docs_generate_tests/test_docs_generate.py index c9f7ab62158..e7796e9bdac 100644 --- a/test/integration/029_catalog_generate_tests/test_catalog_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -29,7 +29,7 @@ def project_config(self): @attr(type='postgres') def test_simple_generate(self): - self.run_dbt(["catalog", "generate"]) + self.run_dbt(["docs", "generate"]) self.assertTrue(os.path.exists('./target/catalog.json')) with open('./target/catalog.json') as fp: diff --git a/test/unit/test_catalog_generate.py b/test/unit/test_docs_generate.py similarity index 100% rename from test/unit/test_catalog_generate.py rename to test/unit/test_docs_generate.py From 9b432b89bace73f6eb75393d1fbce86d9c20d5a1 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 17 May 2018 13:44:42 -0600 Subject: [PATCH 42/43] Fix file renames --- .../029_docs_generate_tests/test_docs_generate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index e7796e9bdac..4ee21304348 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -5,11 +5,11 @@ from test.integration.base import DBTIntegrationTest -class TestCatalogGenerate(DBTIntegrationTest): +class TestDocsGenerate(DBTIntegrationTest): def setUp(self): - super(TestCatalogGenerate, self).setUp() - self.run_sql_file("test/integration/029_catalog_generate_tests/seed.sql") + super(TestDocsGenerate, self).setUp() + self.run_sql_file("test/integration/029_docs_generate_tests/seed.sql") @property def schema(self): @@ -17,7 +17,7 @@ def schema(self): @property def models(self): - return "test/integration/029_catalog_generate_tests/models" + return "test/integration/029_docs_generate_tests/models" @property def project_config(self): From 04ffaba207d9127a097f9b3ebacc4e9b4b954556 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 21 May 2018 09:47:35 -0600 Subject: [PATCH 43/43] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1985bf47f5f..b6da165a76a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Use Mapping instead of dict as the base class for APIObject ([#756](https://github.com/fishtown-analytics/dbt/pull/756)) - Write JSON manifest file to disk during compilation ([#761](https://github.com/fishtown-analytics/dbt/pull/761)) - Add forward and backward graph edges to the JSON manifest file ([#762](https://github.com/fishtown-analytics/dbt/pull/762)) +- Add a 'dbt docs generate' command to generate a JSON catalog file ([#774](https://github.com/fishtown-analytics/dbt/pull/774)) ## dbt 0.10.1 (Unreleased)