diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index 073a26fd3..915b776bb 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -23,6 +23,7 @@ Changes are grouped as follows: ### Fixed +- Replacing variables in an inline SQL query no longer removes the quotes around the variable. - Running `cdf build` on an older module will no longer raise an `KeyError` if the `module.toml` does not have a `package` key. - [alpha feature] `cdf purge dataset` no longer deletes `LocationFilters` diff --git a/cognite_toolkit/_cdf_tk/data_classes/_build_variables.py b/cognite_toolkit/_cdf_tk/data_classes/_build_variables.py index 3f54d0a35..1f54c6e21 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/_build_variables.py +++ b/cognite_toolkit/_cdf_tk/data_classes/_build_variables.py @@ -154,16 +154,18 @@ def get_module_variables(self, module: ModuleLocation) -> list[BuildVariables]: def replace(self, content: str, file_suffix: str = ".yaml") -> str: for variable in self: replace = variable.value_variable - _core_patter = rf"{{{{\s*{variable.key}\s*}}}}" + _core_pattern = rf"{{{{\s*{variable.key}\s*}}}}" if file_suffix in {".yaml", ".yml", ".json"}: # Preserve data types + pattern = _core_pattern if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")): replace = f'"{replace}"' + pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"' elif replace is None: replace = "null" - content = re.sub(rf"'{_core_patter}'|{_core_patter}|" + rf'"{_core_patter}"', str(replace), content) + content = re.sub(pattern, str(replace), content) else: - content = re.sub(_core_patter, str(replace), content) + content = re.sub(_core_pattern, str(replace), content) return content diff --git a/tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py b/tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py index 70c9f339b..47dae93f9 100644 --- a/tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py +++ b/tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py @@ -73,6 +73,34 @@ def test_replace_not_preserve_type(self) -> None: assert result == "dataset_id('ds_external_id')" + def test_replace_inline_sql_preserve_double_quotes(self) -> None: + source_yaml = """externalId: some_id +name: Some Transformation +destination: + type: nodes + view: + space: cdf_cdm + externalId: CogniteTimeSeries + version: v1 + instanceSpace: my_instance_space +query: >- + select "fpso_{{location_id}}" as externalId, "{{location_ID}}" as uid, "{{location_ID}}" as description +""" + variables = BuildVariables.load_raw( + { + "location_id": "uny", + "location_ID": "UNY", + }, + available_modules=set(), + selected_modules=set(), + ) + + result = variables.replace(source_yaml, file_suffix=".yaml") + + loaded = yaml.safe_load(result) + + assert loaded["query"] == 'select "fpso_uny" as externalId, "UNY" as uid, "UNY" as description' + def test_get_module_variables_variable_preference_order(self) -> None: source_yaml = """ modules: diff --git a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_common.yaml b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_common.yaml index 24d742806..62f8445b5 100644 --- a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_common.yaml +++ b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_common.yaml @@ -36,8 +36,8 @@ ExtractionPipeline: name: Direct Relation Writer ExtractionPipelineConfig: - config: "state:\n rawDatabase: contextualizationState\n rawTable: diagramParsing\n\ - data:\n annotationSpace: springfield_instances\n directRelationMappings:\n \ - \ - startNodeView:\n space: cdf_cdm\n externalId: CogniteFile\n\ + data:\n annotationSpace: 'springfield_instances'\n directRelationMappings:\n\ + \ - startNodeView:\n space: cdf_cdm\n externalId: CogniteFile\n\ \ version: v1\n directRelationProperty: assets\n endNodeView:\n\ \ space: cdf_cdm\n externalId: CogniteAsset\n version: v1\n\ \ - startNodeView:\n type: view\n space: cdf_cdm\n externalId:\ diff --git a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_3d_valhall.yaml b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_3d_valhall.yaml index fd45628a8..f4da9b37d 100644 --- a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_3d_valhall.yaml +++ b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_3d_valhall.yaml @@ -80,7 +80,7 @@ ExtractionPipelineConfig: - config: "data:\n debug: False\n runAll: False\n rawdb: '3d_oid_fileshare'\n \ \ rawTableGood: 'contextualization_good'\n rawTableBad: 'contextualization_bad'\n\ \ rawTableManual: 'contextualization_manual_input'\n assetRootExtId: WMT:VAL\n\ - \ 3dDataSetExtId: 'ds_3d_oid'\n 3dModelName: Valhall\n matchThreshold: 0.75\n\ + \ 3dDataSetExtId: 'ds_3d_oid'\n 3dModelName: 'Valhall'\n matchThreshold: 0.75\n\ \ keepOldMapping: False\n" externalId: ep_ctx_3d_oid_fileshare_annotation - config: "logger:\n # Logging to console/terminal. Remove or comment out to disable\ diff --git a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_files_valhall.yaml b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_files_valhall.yaml index 121afa0c2..2ff93318e 100644 --- a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_files_valhall.yaml +++ b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_files_valhall.yaml @@ -34,8 +34,8 @@ ExtractionPipeline: \ documents, if False only run on document not updated since last annotation\n\ \ runAll: False\n # Number of document from each asset to annotate -1 = All, else\ \ number is used\n docLimit: -1\n # Data set to read documents / P&ID from + update\ - \ with annotated timestamp when done\n docDataSetExtId: ds_files_oid\n # In document\ - \ metadata, where is location of property for P&ID type documents\n docTypeMetaCol:\ + \ with annotated timestamp when done\n docDataSetExtId: \"ds_files_oid\"\n # In\ + \ document metadata, where is location of property for P&ID type documents\n docTypeMetaCol:\ \ \"doc_type\"\n # Document type for P&ID type documents\n pAndIdDocType: \"PNID\"\ \n # List of externalId for root assets to be used for annotation of documents.\ \ Documents are also extracted based on the root asset ID\n assetRootExtIds: [WMT:VAL]\n\ @@ -65,8 +65,8 @@ ExtractionPipeline: source: fileshare ExtractionPipelineConfig: - config: "data:\n debug: False\n runAll: False\n docLimit: -1\n docDataSetExtId:\ - \ ds_files_oid\n docTypeMetaCol: \"doc_type\"\n pAndIdDocType: \"PNID\"\n assetRootExtIds:\ - \ [WMT:VAL]\n matchThreshold: 0.85\n \n" + \ \"ds_files_oid\"\n docTypeMetaCol: \"doc_type\"\n pAndIdDocType: \"PNID\"\n\ + \ assetRootExtIds: [WMT:VAL]\n matchThreshold: 0.85\n \n" externalId: ep_ctx_files_oid_fileshare_pandid_annotation - config: "logger:\n # Logging to console/terminal. Remove or comment out to disable\ \ terminal\n # logging\n console:\n level: INFO\n\n # Logging to file. Include\ diff --git a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_timeseries_valhall.yaml b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_timeseries_valhall.yaml index 2e9f31616..c1a0ed830 100644 --- a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_timeseries_valhall.yaml +++ b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_timeseries_valhall.yaml @@ -117,7 +117,7 @@ ExtractionPipelineConfig: \ matchThreshold: 0.75\n" externalId: ep_ctx_timeseries_oid_opcua_asset - config: "# Version of the config schema\nversion: 1\n\nsource:\n # The URL of\ - \ the OPC-UA server to connect to\n endpoint-url: opc.tcp://DESKTOP-18JE0L8:53530/OPCUA/SimulationServer\n\ + \ the OPC-UA server to connect to\n endpoint-url: 'opc.tcp://DESKTOP-18JE0L8:53530/OPCUA/SimulationServer'\n\ \ force-restart: true\n browse-nodes-chunk: 100\n subscription-chunk:\ \ 1000\n keep-alive-interval: 50000\n sampling-interval: 0\n queue-length:\ \ 10\n browse-throttling:\n max-node-parallelism: 1000\n retries:\n\ @@ -133,9 +133,9 @@ ExtractionPipelineConfig: \ \"opc-ua:\"\n\n # Source node in the OPC-UA server. Leave empty to use the\ \ top level Objects node.\n # If root-nodes is set, this is added to the list\ \ of root nodes.\n root-node:\n # Full name of the namespace of the\ - \ root node.\n namespace-uri: http://www.prosysopc.com/OPCUA/SimulationNodes/\n\ + \ root node.\n namespace-uri: 'http://www.prosysopc.com/OPCUA/SimulationNodes/'\n\ \ # Id of the root node, on the form \"i=123\" or \"s=stringid\" etc.\n\ - \ node-id: s=85/0:Simulation\n\n # Time in minutes between each call\ + \ node-id: 's=85/0:Simulation'\n\n # Time in minutes between each call\ \ to browse the OPC-UA directory, then push new nodes to destinations.\n #\ \ Note that this is a heavy operation, so this number should not be set too low.\n\ \ # Alternatively, use N[timeunit] where timeunit is w, d, h, m, s or ms.\n\ diff --git a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_entity_matching.yaml b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_entity_matching.yaml index 2221013ac..bd77717a1 100644 --- a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_entity_matching.yaml +++ b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_entity_matching.yaml @@ -33,7 +33,7 @@ ExtractionPipelineConfig: - config: "parameters:\n autoApprovalThreshold: 0.85\n autoRejectThreshold: 0.25\n\ \ featureType: \"bigram\"\nsourceSystem:\n space: springfield_instances\n externalId:\ \ entity_matcher\nstate:\n rawDatabase: contextualizationState\n rawTable: entityMatching\n\ - data:\n annotationSpace: springfield_instances\n instanceSpaces:\n ['springfield_instances']\n\ + data:\n annotationSpace: 'springfield_instances'\n instanceSpaces:\n ['springfield_instances']\n\ \ matchingJobs:\n job1:\n sourceView:\n space: cdf_cdm\n \ \ externalId: CogniteTimeSeries\n version: v1\n properties:\n\ \ - name\n targetViews:\n - space: cdf_cdm\n externalId:\ diff --git a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_p_and_id_parser.yaml b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_p_and_id_parser.yaml index b22110bf2..aa89ba289 100644 --- a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_p_and_id_parser.yaml +++ b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_p_and_id_parser.yaml @@ -28,7 +28,7 @@ ExtractionPipeline: ExtractionPipelineConfig: - config: "parameters:\n autoApprovalThreshold: 0.85\n autoRejectThreshold: 0.25\n\ sourceSystem:\n space: springfield_instances\n externalId: p_and_id_parser\n\ - data:\n annotationSpace: springfield_instances\n instanceSpaces:\n ['springfield_instances']\n\ + data:\n annotationSpace: 'springfield_instances'\n instanceSpaces:\n ['springfield_instances']\n\ \ annotationJobs:\n - fileView:\n space: cdf_cdm\n externalId:\ \ CogniteFile\n version: v1\n entityViews:\n - space: cdf_cdm\n\ \ externalId: CogniteEquipment\n version: v1\n searchProperty:\