From bd9960c521e172e513d4fe0a203a38b53a9765a6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 15:09:49 +0000 Subject: [PATCH 1/2] Update dependency cognite-sdk to v6.32.2 --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index a1c9b3790..ad82c4db5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -187,13 +187,13 @@ files = [ [[package]] name = "cognite-sdk" -version = "6.32.1" +version = "6.32.2" description = "Cognite Python SDK" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "cognite_sdk-6.32.1-py3-none-any.whl", hash = "sha256:cfa40be84b77dba08780468630174c142627bf779142cbbc065c8403ee35a6a8"}, - {file = "cognite_sdk-6.32.1.tar.gz", hash = "sha256:de56de49d3be9a30bcd2836c9487ff34adbc856bb23630c12d111d440a5b2b27"}, + {file = "cognite_sdk-6.32.2-py3-none-any.whl", hash = "sha256:f888b779a37ddadaabd7e56ef6c3b9f3f1e75ae5cae031f60875bc4968d904dd"}, + {file = "cognite_sdk-6.32.2.tar.gz", hash = "sha256:c59edcb28736a83587a5f8fdc7f3413c786728a92ddd997c748d166d410ad549"}, ] [package.dependencies] @@ -779,4 +779,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "0bd4efd1bbfd77ddaac57335253ed533cc1140b91e97016d973cb107c8b48116" +content-hash = "8450eff11cf2b5422048fba35e691764f356afed74c03476fbce2562e3872645" diff --git a/pyproject.toml b/pyproject.toml index ca01d394e..a59f8ad66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ readme = "README.md" [tool.poetry.dependencies] python = ">=3.9,<3.12" python-dotenv = "^1.0.0" -cognite-sdk = {version = "6.32.1", extras = ["pandas"]} +cognite-sdk = {version = "6.32.2", extras = ["pandas"]} pandas = "^2.1.1" pyyaml = "^6.0.1" dacite = "^1.8.1" From 1560a268c88d067567de93c977f1e9f9b6e3ef91 Mon Sep 17 00:00:00 2001 From: Greger Teigre Wedel Date: Sat, 14 Oct 2023 09:45:32 +0200 Subject: [PATCH 2/2] Gtw/clean_up_tooling (#22) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Move apm_simple configs into module dir * Factor out example stuff from data-model-examples --------- Co-authored-by: Pål Rønning --- clean.py | 2 +- config.yaml | 9 -- deploy.py | 42 +++++-- modules/cdf_apm_simple/config.yaml | 9 ++ .../domain_models/Asset.container.yaml | 2 +- .../domain_models/WorkItem.container.yaml | 2 +- .../domain_models/WorkOrder.container.yaml | 2 +- .../solution_models/Asset.view.yaml | 32 ++--- .../solution_models/WorkItem.view.yaml | 28 ++--- .../solution_models/WorkOrder.view.yaml | 50 ++++---- .../solution_models/apm_simple.datamodel.yaml | 20 +-- .../transformations/assets.yaml | 12 +- .../transformations/link_asset2children.yaml | 14 +-- .../link_workorders2assets.yaml | 14 +-- .../transformations/timeseries2assets.yaml | 10 +- .../tutorial-load-asset2children.json | 6 +- .../transformations/tutorial-load-assets.json | 6 +- .../tutorial-load-timeseries2assets.json | 6 +- .../tutorial-load-workitems.json | 6 +- .../tutorial-load-workitems2assets.json | 6 +- .../tutorial-load-workitems2workorders.json | 6 +- .../tutorial-load-workorders.json | 6 +- .../tutorial-load-workorders2assets.json | 6 +- .../transformations/workitems.yaml | 12 +- .../transformations/workitems2assets.yaml | 14 +-- .../transformations/workitems2workorders.yaml | 14 +-- .../transformations/workorders.yaml | 10 +- scripts/datamodel.py | 76 ++++++++---- scripts/delete.py | 98 ++++++--------- scripts/load.py | 82 ++++++------- scripts/transformations.py | 14 +-- scripts/utils.py | 116 +++++------------- 32 files changed, 349 insertions(+), 383 deletions(-) diff --git a/clean.py b/clean.py index 4adae50be..58d089ce8 100755 --- a/clean.py +++ b/clean.py @@ -37,7 +37,7 @@ def run(build_dir: str) -> None: print( " The current utils/ delete tooling needs to be adapted to pick up configurations in" ) - print(" ./build/ directory and not (as today) use hard-coded examples directory.") + print(" ./build/ directory.") if ToolGlobals.failed: print(f"Failure to load as expected.") exit(1) diff --git a/config.yaml b/config.yaml index 68374e3ef..47a504faa 100644 --- a/config.yaml +++ b/config.yaml @@ -1,10 +1 @@ -apm_simple_raw_db: tutorial_apm -apm_simple_datamodel: tutorial_apm_simple -apm_simple_space: tutorial_apm_simple -apm_simple_data_set: Valhall_System_23 -apm_simple_data_set_desc: Valhall_System_23 -apm_simple_datamodel_version: "1" -apm_simple_view_Asset_version: "1" -apm_simple_view_WorkOrder_version: "1" -apm_simple_view_WorkItem_version: "1" cdf_cluster: ${CDF_CLUSTER} diff --git a/deploy.py b/deploy.py index 51cc67b9a..92ca7d849 100755 --- a/deploy.py +++ b/deploy.py @@ -21,7 +21,7 @@ load_dotenv(".env") -def run(build_dir: str) -> None: +def run(build_dir: str, drop: bool = True, dry_run: bool = True) -> None: print(f"Deploying config files from {build_dir}...") # Configure a client and load credentials from environment build_path = Path(__file__).parent / build_dir @@ -29,8 +29,6 @@ def run(build_dir: str) -> None: print(f"{build_dir} does not exists.") exit(1) ToolGlobals = CDFToolConfig(client_name="cdf-project-templates") - # TODO: #14 This is confusing heritage from data-model-examples. Refactor to use config.yaml and module structure. - ToolGlobals.example = "default" print("Using following configurations: ") print(ToolGlobals) # TODO: #6 This is a very limited support. Needs to be expanded to support configurable groups. @@ -43,18 +41,34 @@ def run(build_dir: str) -> None: ) if Path(f"{build_dir}/raw").is_dir(): # TODO: #7 load_raw only loads one database as configured in ToolGlobals.config, needs more dynamic support - load_raw(ToolGlobals, drop=True, file=None, directory=f"f{build_dir}/raw") + # Now hardcoded to load into "default" database. This must be configurable in the module (config.yaml) + load_raw( + ToolGlobals, + raw_db="default", + drop=drop, + file=None, + directory=f"f{build_dir}/raw", + ) + # TODO: #21 Implement dry-run consistently across. if Path(f"{build_dir}/timeseries").is_dir(): load_timeseries_metadata( - ToolGlobals, drop=True, file=None, directory=f"f{build_dir}/timeseries" + ToolGlobals, drop=drop, file=None, directory=f"f{build_dir}/timeseries" ) if Path(f"{build_dir}/transformations").is_dir(): load_transformations_dump( - ToolGlobals, file=None, drop=True, directory=f"{build_dir}/transformations" + ToolGlobals, file=None, drop=drop, directory=f"{build_dir}/transformations" + ) + if (models_dir := Path(f"{build_dir}/source_models")).is_dir(): + load_datamodel_dump( + ToolGlobals, drop=drop, directory=models_dir, dry_run=dry_run + ) + if (models_dir := Path(f"{build_dir}/domain_models")).is_dir(): + load_datamodel_dump( + ToolGlobals, drop=drop, directory=models_dir, dry_run=dry_run ) - if (models_dir := Path(f"{build_dir}/data_models")).is_dir(): + if (models_dir := Path(f"{build_dir}/solution_models")).is_dir(): load_datamodel_dump( - ToolGlobals, drop=True, directory=models_dir, dry_run=True + ToolGlobals, drop=drop, directory=models_dir, dry_run=dry_run ) if ToolGlobals.failed: print(f"Failure to load as expected.") @@ -63,6 +77,12 @@ def run(build_dir: str) -> None: if __name__ == "__main__": parser = argparse.ArgumentParser(epilog="Further functionality to be added") + parser.add_argument( + "--dry-run", help="whether to do a dry-run", type=bool, default=False + ) + parser.add_argument( + "--drop", help="whether to drop existing data", type=bool, default=True + ) parser.add_argument( "build_dir", default="./build", @@ -70,4 +90,8 @@ def run(build_dir: str) -> None: help="Where to pick up the config files to deploy", ) args, unknown_args = parser.parse_known_args() - run(args.build_dir) + run( + build_dir=args.build_dir, + dry_run=args.dry_run, + drop=args.drop, + ) diff --git a/modules/cdf_apm_simple/config.yaml b/modules/cdf_apm_simple/config.yaml index dd92648c1..9279e9f27 100644 --- a/modules/cdf_apm_simple/config.yaml +++ b/modules/cdf_apm_simple/config.yaml @@ -1,4 +1,13 @@ # Only valid for this module, loads template variables from environment +raw_db: tutorial_apm +datamodel: tutorial_apm_simple +space: tutorial_apm_simple +data_set: Valhall_System_23 +data_set_desc: Valhall_System_23 +datamodel_version: "1" +view_Asset_version: "1" +view_WorkOrder_version: "1" +view_WorkItem_version: "1" clientId: ${TRANSFORMATIONS_CLIENT_ID} clientSecret: ${TRANSFORMATIONS_CLIENT_SECRET} tokenUrl: ${TRANSFORMATIONS_TOKEN_URL} diff --git a/modules/cdf_apm_simple/data_models/domain_models/Asset.container.yaml b/modules/cdf_apm_simple/data_models/domain_models/Asset.container.yaml index 0a82a5a68..972c7d2b6 100644 --- a/modules/cdf_apm_simple/data_models/domain_models/Asset.container.yaml +++ b/modules/cdf_apm_simple/data_models/domain_models/Asset.container.yaml @@ -112,5 +112,5 @@ properties: type: list: false type: timestamp -space: '{{apm_simple_space}}' +space: '{{space}}' usedFor: node diff --git a/modules/cdf_apm_simple/data_models/domain_models/WorkItem.container.yaml b/modules/cdf_apm_simple/data_models/domain_models/WorkItem.container.yaml index 04db7db1b..87610f0d9 100644 --- a/modules/cdf_apm_simple/data_models/domain_models/WorkItem.container.yaml +++ b/modules/cdf_apm_simple/data_models/domain_models/WorkItem.container.yaml @@ -88,5 +88,5 @@ properties: type: container: null type: direct -space: '{{apm_simple_space}}' +space: '{{space}}' usedFor: node diff --git a/modules/cdf_apm_simple/data_models/domain_models/WorkOrder.container.yaml b/modules/cdf_apm_simple/data_models/domain_models/WorkOrder.container.yaml index ee623f12e..79e11f20e 100644 --- a/modules/cdf_apm_simple/data_models/domain_models/WorkOrder.container.yaml +++ b/modules/cdf_apm_simple/data_models/domain_models/WorkOrder.container.yaml @@ -179,5 +179,5 @@ properties: collation: ucs_basic list: false type: text -space: '{{apm_simple_space}}' +space: '{{space}}' usedFor: node diff --git a/modules/cdf_apm_simple/data_models/solution_models/Asset.view.yaml b/modules/cdf_apm_simple/data_models/solution_models/Asset.view.yaml index 4f3c48494..7a67c5110 100644 --- a/modules/cdf_apm_simple/data_models/solution_models/Asset.view.yaml +++ b/modules/cdf_apm_simple/data_models/solution_models/Asset.view.yaml @@ -1,19 +1,19 @@ externalId: Asset name: Asset -space: '{{apm_simple_space}}' -version: "{{apm_simple_view_Asset_version}}" +space: '{{space}}' +version: "{{view_Asset_version}}" properties: areaId: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: areaId name: areaId categoryId: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: categoryId name: categoryId @@ -22,79 +22,79 @@ properties: name: children source: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: view version: 8069f1498c7f9a type: externalId: Asset.children - space: '{{apm_simple_space}}' + space: '{{space}}' createdDate: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: createdDate name: createdDate description: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: description name: description documents: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: documents name: documents isActive: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: isActive name: isActive isCriticalLine: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: isCriticalLine name: isCriticalLine metrics: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: metrics name: metrics parent: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: parent name: parent sourceDb: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: sourceDb name: sourceDb tag: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: tag name: tag updatedDate: container: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: updatedDate name: updatedDate diff --git a/modules/cdf_apm_simple/data_models/solution_models/WorkItem.view.yaml b/modules/cdf_apm_simple/data_models/solution_models/WorkItem.view.yaml index 253f725f9..ec531767e 100644 --- a/modules/cdf_apm_simple/data_models/solution_models/WorkItem.view.yaml +++ b/modules/cdf_apm_simple/data_models/solution_models/WorkItem.view.yaml @@ -1,40 +1,40 @@ externalId: WorkItem name: WorkItem -space: '{{apm_simple_space}}' -version: "{{apm_simple_view_WorkItem_version}}" +space: '{{space}}' +version: "{{view_WorkItem_version}}" properties: criticality: container: externalId: WorkItem - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: criticality name: criticality description: container: externalId: WorkItem - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: description name: description isCompleted: container: externalId: WorkItem - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: isCompleted name: isCompleted itemInfo: container: externalId: WorkItem - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: itemInfo name: itemInfo itemName: container: externalId: WorkItem - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: itemName name: itemName @@ -43,37 +43,37 @@ properties: name: linkedAssets source: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: view - version: '{{apm_simple_view_Asset_version}}' + version: '{{view_Asset_version}}' type: externalId: WorkItem.linkedAssets - space: '{{apm_simple_space}}' + space: '{{space}}' method: container: externalId: WorkItem - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: method name: method title: container: externalId: WorkItem - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: title name: title toBeDone: container: externalId: WorkItem - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: toBeDone name: toBeDone workOrder: container: externalId: WorkItem - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: workOrder name: workOrder diff --git a/modules/cdf_apm_simple/data_models/solution_models/WorkOrder.view.yaml b/modules/cdf_apm_simple/data_models/solution_models/WorkOrder.view.yaml index a975df574..a5f123b59 100644 --- a/modules/cdf_apm_simple/data_models/solution_models/WorkOrder.view.yaml +++ b/modules/cdf_apm_simple/data_models/solution_models/WorkOrder.view.yaml @@ -1,75 +1,75 @@ externalId: WorkOrder name: WorkOrder -space: '{{apm_simple_space}}' -version: "{{apm_simple_view_WorkOrder_version}}" +space: '{{space}}' +version: "{{view_WorkOrder_version}}" properties: actualHours: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: actualHours name: actualHours createdDate: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: createdDate name: createdDate description: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: description name: description dueDate: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: dueDate name: dueDate durationHours: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: durationHours name: durationHours endTime: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: endTime name: endTime isActive: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: isActive name: isActive isCancelled: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: isCancelled name: isCancelled isCompleted: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: isCompleted name: isCompleted isSafetyCritical: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: isSafetyCritical name: isSafetyCritical @@ -78,58 +78,58 @@ properties: name: linkedAssets source: externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: view version: 8069f1498c7f9a type: externalId: WorkOrder.linkedAssets - space: '{{apm_simple_space}}' + space: '{{space}}' percentageProgress: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: percentageProgress name: percentageProgress plannedStart: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: plannedStart name: plannedStart priorityDescription: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: priorityDescription name: priorityDescription programNumber: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: programNumber name: programNumber startTime: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: startTime name: startTime status: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: status name: status title: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: title name: title @@ -138,23 +138,23 @@ properties: name: workItems source: externalId: WorkItem - space: '{{apm_simple_space}}' + space: '{{space}}' type: view version: 94d2b7121128a2 type: externalId: WorkOrder.workItems - space: '{{apm_simple_space}}' + space: '{{space}}' workOrderNumber: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: workOrderNumber name: workOrderNumber workPackageNumber: container: externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: container containerPropertyIdentifier: workPackageNumber name: workPackageNumber diff --git a/modules/cdf_apm_simple/data_models/solution_models/apm_simple.datamodel.yaml b/modules/cdf_apm_simple/data_models/solution_models/apm_simple.datamodel.yaml index bc87c88f7..d91dea5cd 100644 --- a/modules/cdf_apm_simple/data_models/solution_models/apm_simple.datamodel.yaml +++ b/modules/cdf_apm_simple/data_models/solution_models/apm_simple.datamodel.yaml @@ -1,17 +1,17 @@ -externalId: '{{apm_simple_datamodel}}' -name: '{{apm_simple_datamodel}}' -space: '{{apm_simple_space}}' -version: '{{apm_simple_datamodel_version}}' +externalId: '{{datamodel}}' +name: '{{datamodel}}' +space: '{{space}}' +version: '{{datamodel_version}}' views: - externalId: Asset - space: '{{apm_simple_space}}' + space: '{{space}}' type: view - version: 8069f1498c7f9a + version: "{{view_Asset_version}}" - externalId: WorkOrder - space: '{{apm_simple_space}}' + space: '{{space}}' type: view - version: 659c59a4a8afe9 + version: "{{view_WorkOrder_version}}" - externalId: WorkItem - space: '{{apm_simple_space}}' + space: '{{space}}' type: view - version: 94d2b7121128a2 + version: "{{view_WorkItem_version}}" diff --git a/modules/cdf_apm_simple/transformations/assets.yaml b/modules/cdf_apm_simple/transformations/assets.yaml index 95a16dd98..478e72d28 100644 --- a/modules/cdf_apm_simple/transformations/assets.yaml +++ b/modules/cdf_apm_simple/transformations/assets.yaml @@ -14,7 +14,7 @@ query: >- {"from":"tag","to":"tag","asType":"STRING"}, {"from":"areaId","to":"areaId","asType":"INT"}, {"from":"isActive","to":"isActive","asType":"BOOLEAN"}], - "sourceLevel1":"{{apm_simple_raw_db}}","sourceLevel2":"assets"} */ + "sourceLevel1":"{{raw_db}}","sourceLevel2":"assets"} */ select cast(`externalId` as STRING) as externalId, cast(`categoryId` as INT) as categoryId, @@ -22,20 +22,20 @@ query: >- cast(`sourceDb` as STRING) as sourceDb, cast(`updatedDate` as TIMESTAMP) as updatedDate, cast(`createdDate` as TIMESTAMP) as createdDate, - node_reference('{{apm_simple_space}}', `parentExternalId`) as parent, + node_reference('{{space}}', `parentExternalId`) as parent, cast(`description` as STRING) as description, cast(`tag` as STRING) as tag, cast(`areaId` as INT) as areaId, cast(`isActive` as BOOLEAN) as isActive from - `{{apm_simple_raw_db}}`.`assets`; + `{{raw_db}}`.`assets`; destination: dataModel: - space: {{apm_simple_space}} - externalId: {{apm_simple_datamodel}} + space: {{space}} + externalId: {{datamodel}} version: "1" destinationType: Asset - instanceSpace: {{apm_simple_space}} + instanceSpace: {{space}} type: instances ignoreNullFields: true shared: true diff --git a/modules/cdf_apm_simple/transformations/link_asset2children.yaml b/modules/cdf_apm_simple/transformations/link_asset2children.yaml index 2999854b4..ea3f30d31 100644 --- a/modules/cdf_apm_simple/transformations/link_asset2children.yaml +++ b/modules/cdf_apm_simple/transformations/link_asset2children.yaml @@ -6,21 +6,21 @@ query: >- {"from":"externalId","to":"externalId","asType":"STRING"}, {"from":"sourceExternalId","to":"startNode","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"}, {"from":"targetExternalId","to":"endNode","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"}], - "sourceLevel1":"{{apm_simple_raw_db}}","sourceLevel2":"asset2children"} */ + "sourceLevel1":"{{raw_db}}","sourceLevel2":"asset2children"} */ select cast(`externalId` as STRING) as externalId, - node_reference('{{apm_simple_space}}', `sourceExternalId`) as startNode, - node_reference('{{apm_simple_space}}', `targetExternalId`) as endNode + node_reference('{{space}}', `sourceExternalId`) as startNode, + node_reference('{{space}}', `targetExternalId`) as endNode from - `{{apm_simple_raw_db}}`.`asset2children`; + `{{raw_db}}`.`asset2children`; destination: dataModel: - space: {{apm_simple_space}} - externalId: {{apm_simple_datamodel}} + space: {{space}} + externalId: {{datamodel}} version: "1" destinationType: Asset destinationRelationshipFromType: children - instanceSpace: {{apm_simple_space}} + instanceSpace: {{space}} type: instances ignoreNullFields: true shared: true diff --git a/modules/cdf_apm_simple/transformations/link_workorders2assets.yaml b/modules/cdf_apm_simple/transformations/link_workorders2assets.yaml index ea4a77955..f67ed9ee1 100644 --- a/modules/cdf_apm_simple/transformations/link_workorders2assets.yaml +++ b/modules/cdf_apm_simple/transformations/link_workorders2assets.yaml @@ -2,21 +2,21 @@ externalId: tutorial-load-workorders2assets name: tutorial-load-Workorders2assets query: >- /* MAPPING_MODE_ENABLED: true */ - /* {"version":1,"sourceType":"raw","mappings":[{"from":"externalId","to":"externalId","asType":"STRING"},{"from":"sourceExternalId","to":"startNode","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"},{"to":"endNode","asType":"STRUCT<`space`:STRING, `externalId`:STRING>","from":"targetExternalId"}],"sourceLevel1":"{{apm_simple_raw_db}}","sourceLevel2":"workorder2assets"} */ + /* {"version":1,"sourceType":"raw","mappings":[{"from":"externalId","to":"externalId","asType":"STRING"},{"from":"sourceExternalId","to":"startNode","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"},{"to":"endNode","asType":"STRUCT<`space`:STRING, `externalId`:STRING>","from":"targetExternalId"}],"sourceLevel1":"{{raw_db}}","sourceLevel2":"workorder2assets"} */ select cast(`externalId` as STRING) as externalId, - node_reference('{{apm_simple_space}}', `sourceExternalId`) as startNode, - node_reference('{{apm_simple_space}}', `targetExternalId`) as endNode + node_reference('{{space}}', `sourceExternalId`) as startNode, + node_reference('{{space}}', `targetExternalId`) as endNode from - `{{apm_simple_raw_db}}`.`workorder2assets`; + `{{raw_db}}`.`workorder2assets`; destination: dataModel: - space: {{apm_simple_space}} - externalId: {{apm_simple_datamodel}} + space: {{space}} + externalId: {{datamodel}} version: "1" destinationType: WorkOrder destinationRelationshipFromType: linkedAssets - instanceSpace: {{apm_simple_space}} + instanceSpace: {{space}} type: instances ignoreNullFields: true shared: true diff --git a/modules/cdf_apm_simple/transformations/timeseries2assets.yaml b/modules/cdf_apm_simple/transformations/timeseries2assets.yaml index 8a21677f4..56658591c 100644 --- a/modules/cdf_apm_simple/transformations/timeseries2assets.yaml +++ b/modules/cdf_apm_simple/transformations/timeseries2assets.yaml @@ -17,20 +17,20 @@ query: >- {"from":"","to":"tag","asType":"STRING"}, {"from":"","to":"areaId","asType":"INT"}, {"from":"","to":"isActive","asType":"BOOLEAN"}], - "sourceLevel1":"{{apm_simple_raw_db}}","sourceLevel2":"timeseries2assets"} */ + "sourceLevel1":"{{raw_db}}","sourceLevel2":"timeseries2assets"} */ select cast(`asset` as STRING) as externalId, array(timeseries) as metrics from - `{{apm_simple_raw_db}}`.`timeseries2assets`; + `{{raw_db}}`.`timeseries2assets`; destination: dataModel: - space: {{apm_simple_space}} - externalId: {{apm_simple_datamodel}} + space: {{space}} + externalId: {{datamodel}} version: '1' destinationType: Asset - instanceSpace: {{apm_simple_space}} + instanceSpace: {{space}} type: instances ignoreNullFields: true shared: true diff --git a/modules/cdf_apm_simple/transformations/tutorial-load-asset2children.json b/modules/cdf_apm_simple/transformations/tutorial-load-asset2children.json index 032f24c81..6d53d155b 100644 --- a/modules/cdf_apm_simple/transformations/tutorial-load-asset2children.json +++ b/modules/cdf_apm_simple/transformations/tutorial-load-asset2children.json @@ -4,13 +4,13 @@ "destination": { "type": "instances", "data_model": { - "space": "{{apm_simple_space}}", - "external_id": "{{apm_simple_datamodel}}", + "space": "{{space}}", + "external_id": "{{datamodel}}", "version": "1", "destination_type": "Asset", "destination_relationship_from_type": "children" }, - "instance_space": "{{apm_simple_space}}" + "instance_space": "{{space}}" }, "conflict_mode": "upsert", "is_public": true, diff --git a/modules/cdf_apm_simple/transformations/tutorial-load-assets.json b/modules/cdf_apm_simple/transformations/tutorial-load-assets.json index 5b711a0f9..a57bb2cbf 100644 --- a/modules/cdf_apm_simple/transformations/tutorial-load-assets.json +++ b/modules/cdf_apm_simple/transformations/tutorial-load-assets.json @@ -4,13 +4,13 @@ "destination": { "type": "instances", "data_model": { - "space": "{{apm_simple_space}}", - "external_id": "{{apm_simple_datamodel}}", + "space": "{{space}}", + "external_id": "{{datamodel}}", "version": "1", "destination_type": "Asset", "destination_relationship_from_type": null }, - "instance_space": "{{apm_simple_space}}" + "instance_space": "{{space}}" }, "conflict_mode": "upsert", "is_public": true, diff --git a/modules/cdf_apm_simple/transformations/tutorial-load-timeseries2assets.json b/modules/cdf_apm_simple/transformations/tutorial-load-timeseries2assets.json index fb72f6d36..e55a8dfac 100644 --- a/modules/cdf_apm_simple/transformations/tutorial-load-timeseries2assets.json +++ b/modules/cdf_apm_simple/transformations/tutorial-load-timeseries2assets.json @@ -4,13 +4,13 @@ "destination": { "type": "instances", "data_model": { - "space": "{{apm_simple_space}}", - "external_id": "{{apm_simple_datamodel}}", + "space": "{{space}}", + "external_id": "{{datamodel}}", "version": "1", "destination_type": "Asset", "destination_relationship_from_type": null }, - "instance_space": "{{apm_simple_space}}" + "instance_space": "{{space}}" }, "conflict_mode": "upsert", "is_public": true, diff --git a/modules/cdf_apm_simple/transformations/tutorial-load-workitems.json b/modules/cdf_apm_simple/transformations/tutorial-load-workitems.json index 22c65faa6..43a4e1563 100644 --- a/modules/cdf_apm_simple/transformations/tutorial-load-workitems.json +++ b/modules/cdf_apm_simple/transformations/tutorial-load-workitems.json @@ -4,13 +4,13 @@ "destination": { "type": "instances", "data_model": { - "space": "{{apm_simple_space}}", - "external_id": "{{apm_simple_datamodel}}", + "space": "{{space}}", + "external_id": "{{datamodel}}", "version": "1", "destination_type": "WorkItem", "destination_relationship_from_type": null }, - "instance_space": "{{apm_simple_space}}" + "instance_space": "{{space}}" }, "conflict_mode": "upsert", "is_public": true, diff --git a/modules/cdf_apm_simple/transformations/tutorial-load-workitems2assets.json b/modules/cdf_apm_simple/transformations/tutorial-load-workitems2assets.json index 210eed9f9..a440f0c1c 100644 --- a/modules/cdf_apm_simple/transformations/tutorial-load-workitems2assets.json +++ b/modules/cdf_apm_simple/transformations/tutorial-load-workitems2assets.json @@ -4,13 +4,13 @@ "destination": { "type": "instances", "data_model": { - "space": "{{apm_simple_space}}", - "external_id": "{{apm_simple_datamodel}}", + "space": "{{space}}", + "external_id": "{{datamodel}}", "version": "1", "destination_type": "WorkItem", "destination_relationship_from_type": "linkedAssets" }, - "instance_space": "{{apm_simple_space}}" + "instance_space": "{{space}}" }, "conflict_mode": "upsert", "is_public": true, diff --git a/modules/cdf_apm_simple/transformations/tutorial-load-workitems2workorders.json b/modules/cdf_apm_simple/transformations/tutorial-load-workitems2workorders.json index a7cc26c36..a087a7bad 100644 --- a/modules/cdf_apm_simple/transformations/tutorial-load-workitems2workorders.json +++ b/modules/cdf_apm_simple/transformations/tutorial-load-workitems2workorders.json @@ -4,13 +4,13 @@ "destination": { "type": "instances", "data_model": { - "space": "{{apm_simple_space}}", - "external_id": "{{apm_simple_datamodel}}", + "space": "{{space}}", + "external_id": "{{datamodel}}", "version": "1", "destination_type": "WorkOrder", "destination_relationship_from_type": "workItems" }, - "instance_space": "{{apm_simple_space}}" + "instance_space": "{{space}}" }, "conflict_mode": "upsert", "is_public": true, diff --git a/modules/cdf_apm_simple/transformations/tutorial-load-workorders.json b/modules/cdf_apm_simple/transformations/tutorial-load-workorders.json index b58851619..f2cf084c6 100644 --- a/modules/cdf_apm_simple/transformations/tutorial-load-workorders.json +++ b/modules/cdf_apm_simple/transformations/tutorial-load-workorders.json @@ -4,13 +4,13 @@ "destination": { "type": "instances", "data_model": { - "space": "{{apm_simple_space}}", - "external_id": "{{apm_simple_datamodel}}", + "space": "{{space}}", + "external_id": "{{datamodel}}", "version": "1", "destination_type": "WorkOrder", "destination_relationship_from_type": null }, - "instance_space": "{{apm_simple_space}}" + "instance_space": "{{space}}" }, "conflict_mode": "upsert", "is_public": true, diff --git a/modules/cdf_apm_simple/transformations/tutorial-load-workorders2assets.json b/modules/cdf_apm_simple/transformations/tutorial-load-workorders2assets.json index d2e38d205..17d3d185f 100644 --- a/modules/cdf_apm_simple/transformations/tutorial-load-workorders2assets.json +++ b/modules/cdf_apm_simple/transformations/tutorial-load-workorders2assets.json @@ -4,13 +4,13 @@ "destination": { "type": "instances", "data_model": { - "space": "{{apm_simple_space}}", - "external_id": "{{apm_simple_datamodel}}", + "space": "{{space}}", + "external_id": "{{datamodel}}", "version": "1", "destination_type": "WorkOrder", "destination_relationship_from_type": "linkedAssets" }, - "instance_space": "{{apm_simple_space}}" + "instance_space": "{{space}}" }, "conflict_mode": "upsert", "is_public": true, diff --git a/modules/cdf_apm_simple/transformations/workitems.yaml b/modules/cdf_apm_simple/transformations/workitems.yaml index d36a22471..8751a8cf9 100644 --- a/modules/cdf_apm_simple/transformations/workitems.yaml +++ b/modules/cdf_apm_simple/transformations/workitems.yaml @@ -13,11 +13,11 @@ query: >- {"from":"criticality","to":"criticality","asType":"STRING"}, {"from":"method","to":"method","asType":"STRING"}, {"from":"isCompleted","to":"isCompleted","asType":"BOOLEAN"}], - "sourceLevel1":"{{apm_simple_raw_db}}","sourceLevel2":"workitems"} */ + "sourceLevel1":"{{raw_db}}","sourceLevel2":"workitems"} */ select cast(`externalId` as STRING) as externalId, cast(`description` as STRING) as description, - node_reference('{{apm_simple_space}}', `workOrder`) as workOrder, + node_reference('{{space}}', `workOrder`) as workOrder, cast(`toBeDone` as BOOLEAN) as toBeDone, cast(`itemInfo` as STRING) as itemInfo, cast(`itemName` as STRING) as itemName, @@ -26,14 +26,14 @@ query: >- cast(`method` as STRING) as method, cast(`isCompleted` as BOOLEAN) as isCompleted from - `{{apm_simple_raw_db}}`.`workitems`; + `{{raw_db}}`.`workitems`; destination: dataModel: - space: {{apm_simple_space}} - externalId: {{apm_simple_datamodel}} + space: {{space}} + externalId: {{datamodel}} version: "1" destinationType: WorkItem - instanceSpace: {{apm_simple_space}} + instanceSpace: {{space}} type: instances ignoreNullFields: true shared: true diff --git a/modules/cdf_apm_simple/transformations/workitems2assets.yaml b/modules/cdf_apm_simple/transformations/workitems2assets.yaml index 1cece05b3..c343e459c 100644 --- a/modules/cdf_apm_simple/transformations/workitems2assets.yaml +++ b/modules/cdf_apm_simple/transformations/workitems2assets.yaml @@ -6,21 +6,21 @@ query: >- {"from":"externalId","to":"externalId","asType":"STRING"}, {"from":"sourceExternalId","to":"startNode","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"}, {"from":"targetExternalId","to":"endNode","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"}], - "sourceLevel1":"{{apm_simple_raw_db}}","sourceLevel2":"workitem2assets"} */ + "sourceLevel1":"{{raw_db}}","sourceLevel2":"workitem2assets"} */ select cast(`externalId` as STRING) as externalId, - node_reference('{{apm_simple_space}}', `sourceExternalId`) as startNode, - node_reference('{{apm_simple_space}}', `targetExternalId`) as endNode + node_reference('{{space}}', `sourceExternalId`) as startNode, + node_reference('{{space}}', `targetExternalId`) as endNode from - `{{apm_simple_raw_db}}`.`workitem2assets`; + `{{raw_db}}`.`workitem2assets`; destination: dataModel: - space: {{apm_simple_space}} - externalId: {{apm_simple_datamodel}} + space: {{space}} + externalId: {{datamodel}} version: "1" destinationType: WorkItem destinationRelationshipFromType: linkedAssets - instanceSpace: {{apm_simple_space}} + instanceSpace: {{space}} type: instances ignoreNullFields: true shared: true diff --git a/modules/cdf_apm_simple/transformations/workitems2workorders.yaml b/modules/cdf_apm_simple/transformations/workitems2workorders.yaml index 75c338e01..21bcb16cf 100644 --- a/modules/cdf_apm_simple/transformations/workitems2workorders.yaml +++ b/modules/cdf_apm_simple/transformations/workitems2workorders.yaml @@ -6,21 +6,21 @@ query: >- {"from":"externalId","to":"externalId","asType":"STRING"}, {"from":"sourceExternalId","to":"startNode","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"}, {"from":"targetExternalId","to":"endNode","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"}], - "sourceLevel1":"{{apm_simple_raw_db}}","sourceLevel2":"workorder2items"} */ + "sourceLevel1":"{{raw_db}}","sourceLevel2":"workorder2items"} */ select cast(`externalId` as STRING) as externalId, - node_reference('{{apm_simple_space}}', `sourceExternalId`) as startNode, - node_reference('{{apm_simple_space}}', `targetExternalId`) as endNode + node_reference('{{space}}', `sourceExternalId`) as startNode, + node_reference('{{space}}', `targetExternalId`) as endNode from - `{{apm_simple_raw_db}}`.`workorder2items`; + `{{raw_db}}`.`workorder2items`; destination: dataModel: - space: {{apm_simple_space}} - externalId: {{apm_simple_datamodel}} + space: {{space}} + externalId: {{datamodel}} version: "1" destinationType: WorkOrder destinationRelationshipFromType: workItems - instanceSpace: {{apm_simple_space}} + instanceSpace: {{space}} type: instances ignoreNullFields: true shared: true diff --git a/modules/cdf_apm_simple/transformations/workorders.yaml b/modules/cdf_apm_simple/transformations/workorders.yaml index 97266f567..10301b521 100644 --- a/modules/cdf_apm_simple/transformations/workorders.yaml +++ b/modules/cdf_apm_simple/transformations/workorders.yaml @@ -23,7 +23,7 @@ query: >- {"from":"dueDate","to":"dueDate","asType":"TIMESTAMP"}, {"from":"createdDate","to":"createdDate","asType":"TIMESTAMP"}, {"from":"programNumber","to":"programNumber","asType":"STRING"}], - "sourceLevel1":"{{apm_simple_raw_db}}","sourceLevel2":"workorders"} */ + "sourceLevel1":"{{raw_db}}","sourceLevel2":"workorders"} */ select cast(`externalId` as STRING) as externalId, cast(`isCompleted` as BOOLEAN) as isCompleted, @@ -46,14 +46,14 @@ query: >- cast(`createdDate` as TIMESTAMP) as createdDate, cast(`programNumber` as STRING) as programNumber from - `{{apm_simple_raw_db}}`.`workorders`; + `{{raw_db}}`.`workorders`; destination: dataModel: - space: {{apm_simple_space}} - externalId: {{apm_simple_datamodel}} + space: {{space}} + externalId: {{datamodel}} version: "1" destinationType: WorkOrder - instanceSpace: {{apm_simple_space}} + instanceSpace: {{space}} type: instances ignoreNullFields: true shared: true diff --git a/scripts/datamodel.py b/scripts/datamodel.py index 97f04d41e..44bcd1b6d 100644 --- a/scripts/datamodel.py +++ b/scripts/datamodel.py @@ -11,7 +11,18 @@ import yaml from cognite.client.data_classes._base import CogniteResource -from cognite.client.data_classes.data_modeling import View, ViewApply, SpaceApply, ContainerApply, DataModel, DataModelList, DataModelApply, ViewId, DirectRelationReference, DirectRelation +from cognite.client.data_classes.data_modeling import ( + View, + ViewApply, + SpaceApply, + ContainerApply, + DataModel, + DataModelList, + DataModelApply, + ViewId, + DirectRelationReference, + DirectRelation, +) from cognite.client.exceptions import CogniteAPIError from .delete import delete_datamodel @@ -27,9 +38,16 @@ class Difference: unchanged: list[CogniteResource] -def load_datamodel(ToolGlobals: CDFToolConfig, drop: bool, directory=None) -> None: - if directory is None: - directory = f"./examples/{ToolGlobals.example}" +def load_datamodel( + ToolGlobals: CDFToolConfig, + space_name: str = None, + model_name: str = None, + drop: bool = False, + directory=None, +) -> None: + """Load a graphql datamode from file.""" + if space_name is None or model_name is None or directory is None: + raise ValueError("space_name, model_name, and directory must be supplied.") with open(f"{directory}/datamodel.graphql", "rt") as file: # Read directly into a string. datamodel = file.read() @@ -43,18 +61,16 @@ def load_datamodel(ToolGlobals: CDFToolConfig, drop: bool, directory=None) -> No "dataModelInstancesAcl": ["READ", "WRITE"], } ) - space_name = ToolGlobals.config("model_space") - model_name = ToolGlobals.config("data_model") try: client.data_modeling.spaces.apply( SpaceApply( space=space_name, name=space_name, - description=f"Space for {ToolGlobals.example} example", + description=f"Space for {model_name}", ) ) except Exception as e: - print(f"Failed to write space {space_name} for example {ToolGlobals.example}.") + print(f"Failed to write space {space_name}.") print(e) ToolGlobals.failed = True return @@ -64,12 +80,10 @@ def load_datamodel(ToolGlobals: CDFToolConfig, drop: bool, directory=None) -> No (space_name, model_name, "1"), dml=datamodel, name=model_name, - description=f"Data model for {ToolGlobals.example} example", + description=f"Data model for {model_name}", ) except Exception as e: - print( - f"Failed to write data model {model_name} to space {space_name} for example {ToolGlobals.example}." - ) + print(f"Failed to write data model {model_name} to space {space_name}.") print(e) ToolGlobals.failed = True return @@ -207,12 +221,12 @@ def clean_out_datamodels( def load_datamodel_dump( ToolGlobals: CDFToolConfig, drop: bool, - directory: Path | None =None, - dry_run: bool =False, - only_drop: bool=False, + directory: Path | None = None, + dry_run: bool = False, + only_drop: bool = False, ) -> None: if directory is None: - directory = Path(f"./examples/{ToolGlobals.example}/data_model") + raise ValueError("directory must be supplied.") model_files_by_type: dict[str, list[Path]] = defaultdict(list) models_pattern = re.compile(r"^(\w+\.)?(container|view|datamodel)\.yaml$") for file in directory.glob("**/*.yaml"): @@ -222,7 +236,9 @@ def load_datamodel_dump( for type_, files in model_files_by_type.items(): print(f"Found {len(files)} {type_}s in {directory}.") - cognite_resources_by_type: dict[str, list[Union[ContainerApply, ViewApply, DataModelApply, SpaceApply]]] = defaultdict(list) + cognite_resources_by_type: dict[ + str, list[Union[ContainerApply, ViewApply, DataModelApply, SpaceApply]] + ] = defaultdict(list) for type_, files in model_files_by_type.items(): resource_cls = { "container": ContainerApply, @@ -237,10 +253,18 @@ def load_datamodel_dump( for type_, resources in cognite_resources_by_type.items(): print(f" {type_}: {len(resources)}") - space_list = list({r.space for _, resources in cognite_resources_by_type.items() for r in resources}) + space_list = list( + { + r.space + for _, resources in cognite_resources_by_type.items() + for r in resources + } + ) print(f"Found {len(space_list)} spaces") - cognite_resources_by_type["space"] = [SpaceApply(space=s, name=s, description="Imported space") for s in space_list] + cognite_resources_by_type["space"] = [ + SpaceApply(space=s, name=s, description="Imported space") for s in space_list + ] # Clear any delete errors ToolGlobals.failed = False @@ -251,7 +275,9 @@ def load_datamodel_dump( } ) - existing_resources_by_type: dict[str, list[Union[ContainerApply, ViewApply, DataModelApply, SpaceApply]]] = defaultdict(list) + existing_resources_by_type: dict[ + str, list[Union[ContainerApply, ViewApply, DataModelApply, SpaceApply]] + ] = defaultdict(list) resource_api_by_type = { "container": client.data_modeling.containers, "view": client.data_modeling.views, @@ -259,7 +285,9 @@ def load_datamodel_dump( "space": client.data_modeling.spaces, } for type_, resources in cognite_resources_by_type.items(): - existing_resources_by_type[type_] = resource_api_by_type[type_].retrieve([r.as_id() for r in resources]) + existing_resources_by_type[type_] = resource_api_by_type[type_].retrieve( + [r.as_id() for r in resources] + ) differences: dict[str, Difference] = {} for type_, resources in cognite_resources_by_type.items(): @@ -267,11 +295,13 @@ def load_datamodel_dump( existing_by_id = {r.as_id(): r for r in existing_resources_by_type[type_]} added = [r for r in resources if r.as_id() not in existing_by_id] - removed = [r for r in existing_resources_by_type[type_] if r.as_id() not in new_by_id] + removed = [ + r for r in existing_resources_by_type[type_] if r.as_id() not in new_by_id + ] changed = [] unchanged = [] - for existing_id in (set(new_by_id.keys()) & set(existing_by_id.keys())): + for existing_id in set(new_by_id.keys()) & set(existing_by_id.keys()): if new_by_id[existing_id] == existing_by_id[existing_id]: unchanged.append(new_by_id[existing_id]) else: diff --git a/scripts/delete.py b/scripts/delete.py index 63a0fe537..657f2f645 100644 --- a/scripts/delete.py +++ b/scripts/delete.py @@ -20,18 +20,13 @@ from .utils import CDFToolConfig -def delete_raw(ToolGlobals: CDFToolConfig, dry_run=False) -> None: - """Delete raw data from CDF raw based om csv files in the example""" +def delete_raw(ToolGlobals: CDFToolConfig, raw_db: str = None, dry_run=False) -> None: + """Delete raw data from CDF raw based om csv files""" client = ToolGlobals.verify_client(capabilities={"rawAcl": ["READ", "WRITE"]}) # The name of the raw database to create is picked up from the inventory.py file, which # again is templated with cookiecutter based on the user's input. - raw_db = ToolGlobals.config("raw_db") - if raw_db == "": - print( - f"Could not find RAW db {raw_db} in inventory.py for example {ToolGlobals.example}." - ) - ToolGlobals.failed = True - return + if raw_db is None or len(raw_db) == 0: + raise ValueError("raw_db must be specified") try: tables = client.raw.tables.list(raw_db) if len(tables) > 0: @@ -40,22 +35,20 @@ def delete_raw(ToolGlobals: CDFToolConfig, dry_run=False) -> None: client.raw.tables.delete(raw_db, table.name) if not dry_run: client.raw.databases.delete(raw_db) - print(f"Deleted RAW db {raw_db} for example {ToolGlobals.example}.") + print(f"Deleted RAW db {raw_db}.") except: - print( - f"Failed to delete RAW db {raw_db} for example {ToolGlobals.example}. It may not exist." - ) + print(f"Failed to delete RAW db {raw_db}. It may not exist.") def delete_files(ToolGlobals: CDFToolConfig, dry_run=False, directory=None) -> None: if directory is None: - directory = f"./examples/{ToolGlobals.example}/data/files" + raise ValueError("directory must be specified") client = ToolGlobals.verify_client(capabilities={"filesAcl": ["READ", "WRITE"]}) files = [] - # Pick up all the files in the files folder of the example. + # Pick up all the files in the files folder. for _, _, filenames in os.walk(directory): for f in filenames: - files.append(ToolGlobals.example + "_" + f) + files.append(f) if len(files) == 0: return count = 0 @@ -67,25 +60,23 @@ def delete_files(ToolGlobals: CDFToolConfig, dry_run=False, directory=None) -> N except Exception as e: pass if count > 0: - print(f"Deleted {count} files for example {ToolGlobals.example}") + print(f"Deleted {count} files") return - print( - f"Failed to delete files for example {ToolGlobals.example}. They may not exist." - ) + print(f"Failed to delete files. They may not exist.") def delete_timeseries( ToolGlobals: CDFToolConfig, dry_run=False, directory=None ) -> None: - """Delete timeseries from CDF based on json files in the example""" + """Delete timeseries from CDF based on json files""" if directory is None: - directory = f"./examples/{ToolGlobals.example}/data/timeseries" + raise ValueError("directory must be specified") client = ToolGlobals.verify_client( capabilities={"timeseriesAcl": ["READ", "WRITE"]} ) files = [] - # Pick up all the .json files in the data folder of the example. + # Pick up all the .json files in the data folder. for _, _, filenames in os.walk(directory): for f in filenames: if ".json" in f: @@ -114,18 +105,16 @@ def delete_timeseries( except Exception as e: pass if count > 0: - print(f"Deleted {count} timeseries for example {ToolGlobals.example}.") + print(f"Deleted {count} timeseries.") else: - print( - f"Failed to delete timeseries for example {ToolGlobals.example}. They may not exist." - ) + print(f"Failed to delete timeseries. They may not exist.") def delete_transformations( ToolGlobals: CDFToolConfig, dry_run=False, directory=None ) -> None: if directory is None: - directory = f"./examples/{ToolGlobals.example}/transformations" + raise ValueError("directory must be specified") client = ToolGlobals.verify_client( capabilities={"transformationsAcl": ["READ", "WRITE"]} ) @@ -134,20 +123,20 @@ def delete_transformations( try: if not dry_run: client.transformations.delete(external_id=transformations_ext_ids) - print( - f"Deleted {len(transformations_ext_ids)} transformations for example {ToolGlobals.example}." - ) + print(f"Deleted {len(transformations_ext_ids)} transformations.") except Exception as e: - print( - f"Failed to delete transformations for example {ToolGlobals.example}. They may not exist." - ) + print(f"Failed to delete transformations. They may not exist.") return def delete_datamodel( - ToolGlobals: CDFToolConfig, instances_only=True, dry_run=False + ToolGlobals: CDFToolConfig, + space_name: str = None, + model_name: str = None, + instances_only=True, + dry_run=False, ) -> None: - """Delete data model from CDF based on the data model in the example + """Delete data model from CDF based on the data model Note that deleting the data model does not delete the views it consists of or the instances stored across one or more containers. Hence, the clean up data, you @@ -155,29 +144,24 @@ def delete_datamodel( delete these, and then delete the containers and views, before finally deleting the data model itself. """ - + if space_name is None or model_name is None: + raise ValueError("space_name and model_name must be specified") client = ToolGlobals.verify_client( capabilities={ "dataModelsAcl": ["READ", "WRITE"], "dataModelInstancesAcl": ["READ", "WRITE"], } ) - space_name = ToolGlobals.config("model_space") - model_name = ToolGlobals.config("data_model") try: data_model = client.data_modeling.data_models.retrieve( (space_name, model_name, "1") ) except Exception as e: - print( - f"Failed to retrieve data model {model_name} for example {ToolGlobals.example}" - ) + print(f"Failed to retrieve data model {model_name}") print(e) return if len(data_model) == 0: - print( - f"Failed to retrieve data model {model_name} for example {ToolGlobals.example}" - ) + print(f"Failed to retrieve data model {model_name}") view_list = [] else: view_list = [ @@ -225,18 +209,14 @@ def delete_datamodel( f"Found {node_count} nodes and deleted {node_delete} nodes from {id} in {model_name}." ) try: - containers = client.data_modeling.containers.list( - space=ToolGlobals.config("model_space"), limit=None - ) + containers = client.data_modeling.containers.list(space=space_name, limit=None) except Exception as e: - print(f"Failed to retrieve containers for example {ToolGlobals.example}") + print(f"Failed to retrieve containers") print(e) ToolGlobals.failed = True return container_list = [(space_name, c.external_id) for c in containers.data] - print( - f"Found {len(container_list)} containers in the space {space_name} for example {ToolGlobals.example}" - ) + print(f"Found {len(container_list)} containers in the space {space_name}") # Find any remaining nodes in the space node_count = 0 node_delete = 0 @@ -264,9 +244,7 @@ def delete_datamodel( f"Deleted {len(container_list)} containers in data model {model_name}." ) except Exception as e: - print( - f"Failed to delete containers in {space_name} for example {ToolGlobals.example}" - ) + print(f"Failed to delete containers in {space_name}") print(e) ToolGlobals.failed = True try: @@ -275,9 +253,7 @@ def delete_datamodel( client.data_modeling.views.delete(view_list) print(f"Deleted {len(view_list)} views in data model {model_name}.") except Exception as e: - print( - f"Failed to delete views in {space_name} for example {ToolGlobals.example}" - ) + print(f"Failed to delete views in {space_name}") print(e) ToolGlobals.failed = True try: @@ -286,9 +262,7 @@ def delete_datamodel( client.data_modeling.data_models.delete((space_name, model_name, "1")) print(f"Deleted the data model {model_name}.") except Exception as e: - print( - f"Failed to delete data model in {space_name} for example {ToolGlobals.example}" - ) + print(f"Failed to delete data model in {space_name}") print(e) ToolGlobals.failed = True try: @@ -298,6 +272,6 @@ def delete_datamodel( client.data_modeling.spaces.delete(space_name) print(f"Deleted the space {space_name}.") except Exception as e: - print(f"Failed to delete space {space_name} for example {ToolGlobals.example}") + print(f"Failed to delete space {space_name}") print(e) ToolGlobals.failed = True diff --git a/scripts/load.py b/scripts/load.py index 77b66eb5a..a617906c0 100644 --- a/scripts/load.py +++ b/scripts/load.py @@ -29,23 +29,27 @@ from cognite.client.exceptions import CogniteNotFoundError -def load_raw(ToolGlobals: CDFToolConfig, file: str, drop: bool, directory=None) -> None: +def load_raw( + ToolGlobals: CDFToolConfig, + file: str, + raw_db: str = None, + drop: bool = False, + directory=None, +) -> None: """Load raw data from csv files into CDF Raw Args: file: name of file to load, if empty load all files drop: whether to drop existing data """ - if directory is None: - directory = f"./examples/{ToolGlobals.example}/data/raw" + if directory is None or raw_db is None: + raise ValueError("directory and raw_db must be specified") client = ToolGlobals.verify_client(capabilities={"rawAcl": ["READ", "WRITE"]}) # The name of the raw database to create is picked up from the inventory.py file, which # again is templated with cookiecutter based on the user's input. - raw_db = ToolGlobals.config("raw_db") + if raw_db == "": - print( - f"Could not find raw_db in inventory.py for example {ToolGlobals.example}." - ) + print(f"Could not find raw_db in inventory.py.") ToolGlobals.failed = True return try: @@ -55,11 +59,9 @@ def load_raw(ToolGlobals: CDFToolConfig, file: str, drop: bool, directory=None) for table in tables: client.raw.tables.delete(raw_db, table.name) client.raw.databases.delete(raw_db) - print(f"Deleted {raw_db} for example {ToolGlobals.example}.") + print(f"Deleted {raw_db}.") except: - print( - f"Failed to delete {raw_db} for example {ToolGlobals.example}. It may not exist." - ) + print(f"Failed to delete {raw_db}. It may not exist.") try: # Creating the raw database and tables is actually not necessary as # the SDK will create them automatically when inserting data with insert_dataframe() @@ -67,9 +69,7 @@ def load_raw(ToolGlobals: CDFToolConfig, file: str, drop: bool, directory=None) # However, it is included to show how you can use the SDK. client.raw.databases.create(raw_db) except Exception as e: - print( - f"Failed to create {raw_db} for example {ToolGlobals.example}: {e.message}" - ) + print(f"Failed to create {raw_db}: {e.message}") ToolGlobals.failed = True return files = [] @@ -77,7 +77,7 @@ def load_raw(ToolGlobals: CDFToolConfig, file: str, drop: bool, directory=None) # Only load the supplied filename. files.append(file) else: - # Pick up all the .csv files in the data folder of the example. + # Pick up all the .csv files in the data folder. for _, _, filenames in os.walk(directory): for f in filenames: if ".csv" in f: @@ -97,7 +97,7 @@ def load_raw(ToolGlobals: CDFToolConfig, file: str, drop: bool, directory=None) ensure_parent=True, ) except Exception as e: - print(f"Failed to upload {f} for example {ToolGlobals.example}") + print(f"Failed to upload {f}") print(e) ToolGlobals.failed = True return @@ -105,17 +105,21 @@ def load_raw(ToolGlobals: CDFToolConfig, file: str, drop: bool, directory=None) def load_files( - ToolGlobals: CDFToolConfig, file: str, drop: bool, directory=None + ToolGlobals: CDFToolConfig, + id_prefix: str = "example", + file: str = None, + drop: bool = False, + directory=None, ) -> None: if directory is None: - directory = f"./examples/{ToolGlobals.example}/data/files" + raise ValueError("directory must be specified") try: client = ToolGlobals.verify_client(capabilities={"filesAcl": ["READ", "WRITE"]}) files = [] if file is not None and len(file) > 0: files.append(file) else: - # Pick up all the files in the files folder of the example. + # Pick up all the files in the files folder. for _, _, filenames in os.walk(directory): for f in filenames: files.append(f) @@ -127,14 +131,12 @@ def load_files( path=f"{directory}/{f}", data_set_id=ToolGlobals.data_set_id, name=f, - external_id=ToolGlobals.example + "_" + f, + external_id=id_prefix + "_" + f, overwrite=drop, ) - print( - f"Uploaded successfully {len(files)} files/documents from example {ToolGlobals.example}" - ) + print(f"Uploaded successfully {len(files)} files/documents.") except Exception as e: - print(f"Failed to upload files for example {ToolGlobals.example}") + print(f"Failed to upload files") print(e) ToolGlobals.failed = True return @@ -153,7 +155,7 @@ def load_timeseries_metadata( ToolGlobals: CDFToolConfig, file: str, drop: bool, directory=None ) -> None: if directory is None: - directory = f"./examples/{ToolGlobals.example}/data/timeseries" + raise ValueError("directory must be specified") client = ToolGlobals.verify_client( capabilities={"timeseriesAcl": ["READ", "WRITE"]} ) @@ -162,7 +164,7 @@ def load_timeseries_metadata( # Only load the supplied filename. files.append(file) else: - # Pick up all the .json files in the data folder of the example. + # Pick up all the .json files in the data folder. for _, _, filenames in os.walk(directory): for f in filenames: if ".json" in f: @@ -188,17 +190,13 @@ def load_timeseries_metadata( try: if drop: client.time_series.delete(external_id=drop_ts, ignore_unknown_ids=True) - print( - f"Deleted {len(drop_ts)} timeseries for example {ToolGlobals.example}." - ) + print(f"Deleted {len(drop_ts)} timeseries.") except Exception as e: - print( - f"Failed to delete {t.external_id} for example {ToolGlobals.example}. It may not exist." - ) + print(f"Failed to delete {t.external_id}. It may not exist.") try: client.time_series.create(timeseries) except Exception as e: - print(f"Failed to upload timeseries for example {ToolGlobals.example}.") + print(f"Failed to upload timeseries.") print(e) ToolGlobals.failed = True return @@ -209,7 +207,7 @@ def load_timeseries_datapoints( ToolGlobals: CDFToolConfig, file: str, directory=None ) -> None: if directory is None: - directory = f"./examples/{ToolGlobals.example}/data/timeseries/datapoints" + raise ValueError("directory must be specified") client = ToolGlobals.verify_client( capabilities={"timeseriesAcl": ["READ", "WRITE"]} ) @@ -218,7 +216,7 @@ def load_timeseries_datapoints( # Only load the supplied filename. files.append(file) else: - # Pick up all the .csv files in the data folder of the example. + # Pick up all the .csv files in the data folder. for _, _, filenames in os.walk(directory): for f in filenames: if ".csv" in f: @@ -234,7 +232,7 @@ def load_timeseries_datapoints( client.time_series.data.insert_dataframe(dataframe) print(f"Uploaded {len(files)} .csv file(s) as datapoints to CDF timeseries.") except Exception as e: - print(f"Failed to upload datapoints for example {ToolGlobals.example}.") + print(f"Failed to upload datapoints.") print(e) ToolGlobals.failed = True return @@ -244,7 +242,7 @@ def load_transformations( ToolGlobals: CDFToolConfig, file: str, drop: bool, directory=None ) -> None: if directory is None: - directory = f"./examples/{ToolGlobals.example}/transformations" + raise ValueError("directory must be specified") client = ToolGlobals.verify_client( capabilities={"transformationsAcl": ["READ", "WRITE"]} ) @@ -282,16 +280,12 @@ def load_transformations( new_transformation_ext_ids, ) except Exception as e: - print(f"Failed to upsert transformations for example {ToolGlobals.example}.") + print(f"Failed to upsert transformations.") print(e) ToolGlobals.failed = True return - print( - f"Updated {len(updated_transformations)} transformations for example {ToolGlobals.example}." - ) - print( - f"Created {len(created_transformations)} transformations for example {ToolGlobals.example}." - ) + print(f"Updated {len(updated_transformations)} transformations.") + print(f"Created {len(created_transformations)} transformations.") def load_readwrite_group( diff --git a/scripts/transformations.py b/scripts/transformations.py index e1006c41b..02716af91 100644 --- a/scripts/transformations.py +++ b/scripts/transformations.py @@ -32,7 +32,7 @@ def run_transformations(ToolGlobals: CDFToolConfig, directory: str = None): if directory is None: - directory = f"./examples/{ToolGlobals.example}/transformations" + raise ValueError("directory must be specified") ToolGlobals.failed = False client = ToolGlobals.verify_client( capabilities={"transformationsAcl": ["READ", "WRITE"]} @@ -42,13 +42,9 @@ def run_transformations(ToolGlobals: CDFToolConfig, directory: str = None): try: for t in transformations_ext_ids: client.transformations.run(transformation_external_id=t, wait=False) - print( - f"Started {len(transformations_ext_ids)} transformation jobs for example {ToolGlobals.example}." - ) + print(f"Started {len(transformations_ext_ids)} transformation jobs.") except Exception as e: - print( - f"Failed to start transformation jobs for example {ToolGlobals.example}. They may not exist." - ) + print(f"Failed to start transformation jobs. They may not exist.") print(e) ToolGlobals.failed = True @@ -120,7 +116,7 @@ def load_transformations_dump( necessary config. Schedules, authentication, etc is not supported. """ if directory is None: - directory = f"./examples/{ToolGlobals.example}/transformations/dump" + raise ValueError("directory must be specified") client = ToolGlobals.verify_client( capabilities={"transformationsAcl": ["READ", "WRITE"]} ) @@ -129,7 +125,7 @@ def load_transformations_dump( # Only load the supplied filename. files.append(file) else: - # Pick up all the .json files in the data folder of the example. + # Pick up all the .json files in the data folder. for _, _, filenames in os.walk(directory): for f in filenames: if ".json" in f: diff --git a/scripts/utils.py b/scripts/utils.py index fc21f1f08..679c3e87f 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -25,55 +25,28 @@ logger = logging.getLogger(__name__) -# To add a new example, add a new entry here with the same name as the folder -# These values are used by the python scripts. class CDFToolConfig: """Configurations for how to store data in CDF Properties: client: active CogniteClient - example: name of the example folder you want to use Functions: - config: configuration for the example (.get("config_name")) verify_client: verify that the client has correct credentials and specified access capabilities verify_dataset: verify that the data set exists and that the client has access to it - To add a new example, add a new entry here with the same name as the folder. - These values are used by the python scripts. """ def __init__( self, - client_name: str = "Generic Cognite examples library", - config: dict | None = None, + client_name: str = "Generic Cognite config deploy tool", token: str = None, ) -> None: self._data_set_id: int = 0 - self._example = None + self._data_set = None self._failed = False self._environ = {} if token is not None: self._environ["CDF_TOKEN"] = token - if config is not None: - self._config = config - else: - try: - with open(f"./inventory.json", "rt") as file: - self._config = json.load(file) - except Exception as e: - logger.info( - "Not loading configuration from inventory.json file, using 'default' as values for all attributes." - ) - self._config = { - "default": { - "raw_db": "default", - "data_set": "default", - "data_set_desc": "-", - "model_space": "default", - "data_model": "default", - } - } - self._example = "default" if ( self.environ("CDF_URL", default=None, fail=False) is None and self.environ("CDF_CLUSTER", default=None, fail=False) is None @@ -111,10 +84,12 @@ def __init__( # We can infer scopes and audience from the cluster value. # However, the URL to use to retrieve the token, as well as # the client id and secret, must be set as environment variables. - self._scopes = [self.environ( - "IDP_SCOPES", - f"https://{self._cluster}.cognitedata.com/.default", - )] + self._scopes = [ + self.environ( + "IDP_SCOPES", + f"https://{self._cluster}.cognitedata.com/.default", + ) + ] self._audience = self.environ( "IDP_AUDIENCE", f"https://{self._cluster}.cognitedata.com" ) @@ -156,31 +131,12 @@ def client(self) -> CogniteClient: @property def data_set_id(self) -> int: - return self._data_set_id + return self._data_set_id if self._data_set_id > 0 else None # Use this to ignore the data set when verifying the client's access capabilities - # Set the example property to configure the data set and verify it def clear_dataset(self): self._data_set_id = 0 - - def config(self, attr: str) -> str: - """Helper function to get configuration for an example (from inventory.json). - - This function uses the example property in this class as a key to get the configuration, - so example must be set before calling the function. - - Args: - attr: name of configuration variable - - Yields: - Value of the configuration variable for example - Raises ValueError if configuration variable is not set - """ - if attr not in self._config.get(self._example, {}): - raise ValueError( - f"{attr} property must be set in CDFToolConfig()/inventory.json." - ) - return self._config.get(self._example, {}).get(attr, "") + self._data_set = None def environ( self, attr: str, default: str | List[str] = None, fail: bool = True @@ -214,20 +170,20 @@ def environ( return self._environ[attr] @property - def example(self) -> str: - return self._example - - @example.setter - def example(self, value: str): - if value is None or value not in self._config: - raise ValueError( - "example must be set to one of the values in the inventory.json file used by CDFToolConfig()." - ) - self._example = value + def data_set(self) -> str: + return self._data_set + + @data_set.setter + def data_set(self, value: str): + if value is None: + raise ValueError("Please provide an externalId of a dataset.") + self._data_set = value # Since we now have a new configuration, check the dataset and set the id - self._data_set_id = self.verify_dataset() + self._data_set_id = self.verify_dataset(data_set_name=value) - def verify_client(self, capabilities: dict[str, list[str]] | None = None) -> CogniteClient: + def verify_client( + self, capabilities: dict[str, list[str]] | None = None + ) -> CogniteClient: """Verify that the client has correct credentials and required access rights Supply requirement CDF ACLs to verify if you have correct access @@ -285,12 +241,13 @@ def verify_client(self, capabilities: dict[str, list[str]] | None = None) -> Cog continue return self._client - def verify_dataset(self, data_set_name: str = None) -> int | None: + def verify_dataset( + self, data_set_name: str = None, create: bool = True + ) -> int | None: """Verify that the configured data set exists and is accessible - This function can be used independent of example config by supplying the data set name. - It will then ignore the example config and use the supplied name. - Calling this function directly will not influence verify_client(). + If the data set does not exist, it will be created unless create=False. + If create=False and the data set does not exist, verify_dataset will return 0. Args: data_set_name (str, optional): name of the data set to verify @@ -299,31 +256,22 @@ def verify_dataset(self, data_set_name: str = None) -> int | None: Re-raises underlying SDK exception """ - def get_dataset_name() -> str: - """Helper function to get the dataset name from the inventory.json file""" - return ( - data_set_name - if data_set_name is not None and len(data_set_name) > 0 - else self.config("data_set") - ) - client = self.verify_client(capabilities={"datasetsAcl": ["READ", "WRITE"]}) try: - data_set = client.data_sets.retrieve(external_id=get_dataset_name()) + data_set = client.data_sets.retrieve(external_id=data_set_name) if data_set is not None: return data_set.id except Exception as e: raise CogniteAuthError( f"Don't have correct access rights. Need READ and WRITE on datasetsAcl." ) + if not create: + return 0 try: # name can be empty, but is useful for UI purposes data_set = DataSet( - external_id=get_dataset_name(), - name=get_dataset_name(), - description=self.config("data_set_desc") - if self.config("data_set_desc") != "" - else "Test data set for tutorials", + external_id=data_set_name, + name=data_set_name, ) data_set = client.data_sets.create(data_set) return data_set.id