savepoint

build.py

@@ -28,4 +28,4 @@ def run(build_dir: str) -> None:
         help="Where to write the config files",
     )
     args, unknown_args = parser.parse_known_args()
-    run(args.build_dir)
+    run(args.build_dir)

clean.py

@@ -37,7 +37,7 @@ def run(build_dir: str) -> None:
     print(
         "  The current utils/ delete tooling needs to be adapted to pick up configurations in"
     )
-    print("  ./build/ directory and not (as today) use hard-coded examples directory.")
+    print("  ./build/ directory.")
     if ToolGlobals.failed:
         print(f"Failure to load as expected.")
         exit(1)

config.yaml

@@ -1,10 +1 @@
-apm_simple_raw_db: tutorial_apm
-apm_simple_datamodel: tutorial_apm_simple
-apm_simple_space: tutorial_apm_simple
-apm_simple_data_set: Valhall_System_23
-apm_simple_data_set_desc: Valhall_System_23
-apm_simple_datamodel_version: "1"
-apm_simple_view_Asset_version: "1"
-apm_simple_view_WorkOrder_version: "1"
-apm_simple_view_WorkItem_version: "1"
 cdf_cluster: ${CDF_CLUSTER}

deploy.py

@@ -18,19 +18,16 @@
 # This is a convenience object that has a CDF client (.client) and
 # allows access to environment variables (.environ) using a consistent
 # naming scheme that is also aligned with recommendations externally.
-load_dotenv(".env")
+load_dotenv(".env", override=True)
 
-
-def run(build_dir: str) -> None:
+def run(build_dir: str, drop: bool = True, dry_run: bool = True) -> None:
     print(f"Deploying config files from {build_dir}...")
     # Configure a client and load credentials from environment
     build_path = Path(__file__).parent / build_dir
     if not build_path.is_dir():
         print(f"{build_dir} does not exists.")
         exit(1)
     ToolGlobals = CDFToolConfig(client_name="cdf-project-templates")
-    # TODO: #14 This is confusing heritage from data-model-examples. Refactor to use config.yaml and module structure.
-    ToolGlobals.example = "default"
     print("Using following configurations: ")
     print(ToolGlobals)
     # TODO: #6 This is a very limited support. Needs to be expanded to support configurable groups.
@@ -43,18 +40,34 @@ def run(build_dir: str) -> None:
         )
     if Path(f"{build_dir}/raw").is_dir():
         # TODO: #7 load_raw only loads one database as configured in ToolGlobals.config, needs more dynamic support
-        load_raw(ToolGlobals, drop=True, file=None, directory=f"f{build_dir}/raw")
+        # Now hardcoded to load into "default" database. This must be configurable in the module (config.yaml)
+        load_raw(
+            ToolGlobals,
+            raw_db="default",
+            drop=drop,
+            file=None,
+            directory=f"{build_dir}/raw",
+        )
+    # TODO: #21 Implement dry-run consistently across.
     if Path(f"{build_dir}/timeseries").is_dir():
         load_timeseries_metadata(
-            ToolGlobals, drop=True, file=None, directory=f"f{build_dir}/timeseries"
+            ToolGlobals, drop=drop, file=None, directory=f"f{build_dir}/timeseries"
         )
     if Path(f"{build_dir}/transformations").is_dir():
         load_transformations_dump(
-            ToolGlobals, file=None, drop=True, directory=f"{build_dir}/transformations"
+            ToolGlobals, file=None, drop=drop, directory=f"{build_dir}/transformations"
+        )
+    if (models_dir := Path(f"{build_dir}/source_models")).is_dir():
+        load_datamodel_dump(
+            ToolGlobals, drop=drop, directory=models_dir, dry_run=dry_run
+        )
+    if (models_dir := Path(f"{build_dir}/domain_models")).is_dir():
+        load_datamodel_dump(
+            ToolGlobals, drop=drop, directory=models_dir, dry_run=dry_run
         )
-    if (models_dir := Path(f"{build_dir}/data_models")).is_dir():
+    if (models_dir := Path(f"{build_dir}/solution_models")).is_dir():
         load_datamodel_dump(
-            ToolGlobals, drop=True, directory=models_dir, dry_run=True
+            ToolGlobals, drop=drop, directory=models_dir, dry_run=dry_run
         )
     if ToolGlobals.failed:
         print(f"Failure to load as expected.")
@@ -63,11 +76,21 @@ def run(build_dir: str) -> None:
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(epilog="Further functionality to be added")
+    parser.add_argument(
+        "--dry-run", help="whether to do a dry-run", type=bool, default=False
+    )
+    parser.add_argument(
+        "--drop", help="whether to drop existing data", type=bool, default=True
+    )
     parser.add_argument(
         "build_dir",
         default="./build",
         nargs="?",
         help="Where to pick up the config files to deploy",
     )
     args, unknown_args = parser.parse_known_args()
-    run(args.build_dir)
+    run(
+        build_dir=args.build_dir,
+        dry_run=args.dry_run,
+        drop=args.drop,
+    )

modules/cdf_apm_simple/config.yaml

@@ -1,4 +1,13 @@
 # Only valid for this module, loads template variables from environment
+raw_db: tutorial_apm
+datamodel: tutorial_apm_simple
+space: tutorial_apm_simple
+data_set: Valhall_System_23
+data_set_desc: Valhall_System_23
+datamodel_version: "1"
+view_Asset_version: "1"
+view_WorkOrder_version: "1"
+view_WorkItem_version: "1"
 clientId: ${TRANSFORMATIONS_CLIENT_ID}
 clientSecret: ${TRANSFORMATIONS_CLIENT_SECRET}
 tokenUrl: ${TRANSFORMATIONS_TOKEN_URL}

modules/cdf_apm_simple/data_models/domain_models/Asset.container.yaml

@@ -112,5 +112,5 @@ properties:
     type:
       list: false
       type: timestamp
-space: '{{apm_simple_space}}'
+space: '{{space}}'
 usedFor: node

modules/cdf_apm_simple/data_models/domain_models/WorkItem.container.yaml

@@ -88,5 +88,5 @@ properties:
     type:
       container: null
       type: direct
-space: '{{apm_simple_space}}'
+space: '{{space}}'
 usedFor: node

modules/cdf_apm_simple/data_models/domain_models/WorkOrder.container.yaml

@@ -179,5 +179,5 @@ properties:
       collation: ucs_basic
       list: false
       type: text
-space: '{{apm_simple_space}}'
+space: '{{space}}'
 usedFor: node

modules/cdf_apm_simple/data_models/solution_models/Asset.view.yaml

@@ -1,19 +1,19 @@
 externalId: Asset
 name: Asset
-space: '{{apm_simple_space}}'
-version: "{{apm_simple_view_Asset_version}}"
+space: '{{space}}'
+version: "{{view_Asset_version}}"
 properties:
   areaId:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: areaId
     name: areaId
   categoryId:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: categoryId
     name: categoryId
@@ -22,79 +22,79 @@ properties:
     name: children
     source:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: view
       version: 8069f1498c7f9a
     type:
       externalId: Asset.children
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
   createdDate:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: createdDate
     name: createdDate
   description:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: description
     name: description
   documents:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: documents
     name: documents
   isActive:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: isActive
     name: isActive
   isCriticalLine:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: isCriticalLine
     name: isCriticalLine
   metrics:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: metrics
     name: metrics
   parent:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: parent
     name: parent
   sourceDb:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: sourceDb
     name: sourceDb
   tag:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: tag
     name: tag
   updatedDate:
     container:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: updatedDate
     name: updatedDate

modules/cdf_apm_simple/data_models/solution_models/WorkItem.view.yaml

@@ -1,40 +1,40 @@
 externalId: WorkItem
 name: WorkItem
-space: '{{apm_simple_space}}'
-version: "{{apm_simple_view_WorkItem_version}}"
+space: '{{space}}'
+version: "{{view_WorkItem_version}}"
 properties:
   criticality:
     container:
       externalId: WorkItem
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: criticality
     name: criticality
   description:
     container:
       externalId: WorkItem
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: description
     name: description
   isCompleted:
     container:
       externalId: WorkItem
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: isCompleted
     name: isCompleted
   itemInfo:
     container:
       externalId: WorkItem
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: itemInfo
     name: itemInfo
   itemName:
     container:
       externalId: WorkItem
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: itemName
     name: itemName
@@ -43,37 +43,37 @@ properties:
     name: linkedAssets
     source:
       externalId: Asset
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: view
-      version: '{{apm_simple_view_Asset_version}}'
+      version: '{{view_Asset_version}}'
     type:
       externalId: WorkItem.linkedAssets
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
   method:
     container:
       externalId: WorkItem
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: method
     name: method
   title:
     container:
       externalId: WorkItem
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: title
     name: title
   toBeDone:
     container:
       externalId: WorkItem
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: toBeDone
     name: toBeDone
   workOrder:
     container:
       externalId: WorkItem
-      space: '{{apm_simple_space}}'
+      space: '{{space}}'
       type: container
     containerPropertyIdentifier: workOrder
     name: workOrder