[CDF-23462] 🥲 Missing quotes (#1278)

* tests: added failing test

* fix: variable replacement

* build: changelog

* tests: regen

* refactor: renaming

CHANGELOG.cdf-tk.md

@@ -23,6 +23,7 @@ Changes are grouped as follows:
 
 ### Fixed
 
+- Replacing variables in an inline SQL query no longer removes the quotes around the variable.
 - Running `cdf build` on an older module will no longer raise an `KeyError` if the `module.toml` does
   not have a `package` key.
 - [alpha feature] `cdf purge dataset` no longer deletes `LocationFilters`

cognite_toolkit/_cdf_tk/data_classes/_build_variables.py

@@ -154,16 +154,18 @@ def get_module_variables(self, module: ModuleLocation) -> list[BuildVariables]:
     def replace(self, content: str, file_suffix: str = ".yaml") -> str:
         for variable in self:
             replace = variable.value_variable
-            _core_patter = rf"{{{{\s*{variable.key}\s*}}}}"
+            _core_pattern = rf"{{{{\s*{variable.key}\s*}}}}"
             if file_suffix in {".yaml", ".yml", ".json"}:
                 # Preserve data types
+                pattern = _core_pattern
                 if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")):
                     replace = f'"{replace}"'
+                    pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"'
                 elif replace is None:
                     replace = "null"
-                content = re.sub(rf"'{_core_patter}'|{_core_patter}|" + rf'"{_core_patter}"', str(replace), content)
+                content = re.sub(pattern, str(replace), content)
             else:
-                content = re.sub(_core_patter, str(replace), content)
+                content = re.sub(_core_pattern, str(replace), content)
 
         return content
 

tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py

@@ -73,6 +73,34 @@ def test_replace_not_preserve_type(self) -> None:
 
         assert result == "dataset_id('ds_external_id')"
 
+    def test_replace_inline_sql_preserve_double_quotes(self) -> None:
+        source_yaml = """externalId: some_id
+name: Some Transformation
+destination:
+  type: nodes
+  view:
+    space: cdf_cdm
+    externalId: CogniteTimeSeries
+    version: v1
+  instanceSpace: my_instance_space
+query: >-
+  select "fpso_{{location_id}}" as externalId, "{{location_ID}}" as uid, "{{location_ID}}" as description
+"""
+        variables = BuildVariables.load_raw(
+            {
+                "location_id": "uny",
+                "location_ID": "UNY",
+            },
+            available_modules=set(),
+            selected_modules=set(),
+        )
+
+        result = variables.replace(source_yaml, file_suffix=".yaml")
+
+        loaded = yaml.safe_load(result)
+
+        assert loaded["query"] == 'select "fpso_uny" as externalId, "UNY" as uid, "UNY" as description'
+
     def test_get_module_variables_variable_preference_order(self) -> None:
         source_yaml = """
 modules:

tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_common.yaml

@@ -36,8 +36,8 @@ ExtractionPipeline:
   name: Direct Relation Writer
 ExtractionPipelineConfig:
 - config: "state:\n  rawDatabase: contextualizationState\n  rawTable: diagramParsing\n\
-    data:\n  annotationSpace: springfield_instances\n  directRelationMappings:\n \
-    \   - startNodeView:\n        space: cdf_cdm\n        externalId: CogniteFile\n\
+    data:\n  annotationSpace: 'springfield_instances'\n  directRelationMappings:\n\
+    \    - startNodeView:\n        space: cdf_cdm\n        externalId: CogniteFile\n\
     \        version: v1\n        directRelationProperty: assets\n      endNodeView:\n\
     \        space: cdf_cdm\n        externalId: CogniteAsset\n        version: v1\n\
     \    - startNodeView:\n        type: view\n        space: cdf_cdm\n        externalId:\

tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_3d_valhall.yaml

@@ -80,7 +80,7 @@ ExtractionPipelineConfig:
 - config: "data:\n  debug: False\n  runAll: False\n  rawdb: '3d_oid_fileshare'\n \
     \ rawTableGood: 'contextualization_good'\n  rawTableBad: 'contextualization_bad'\n\
     \  rawTableManual: 'contextualization_manual_input'\n  assetRootExtId: WMT:VAL\n\
-    \  3dDataSetExtId: 'ds_3d_oid'\n  3dModelName: Valhall\n  matchThreshold: 0.75\n\
+    \  3dDataSetExtId: 'ds_3d_oid'\n  3dModelName: 'Valhall'\n  matchThreshold: 0.75\n\
     \  keepOldMapping: False\n"
   externalId: ep_ctx_3d_oid_fileshare_annotation
 - config: "logger:\n  # Logging to console/terminal. Remove or comment out to disable\

tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_files_valhall.yaml

@@ -34,8 +34,8 @@ ExtractionPipeline:
     \ documents, if False only run on document not updated since last  annotation\n\
     \ runAll: False\n # Number of document from each asset to annotate -1 = All, else\
     \ number is used\n docLimit: -1\n # Data set to read documents / P&ID from + update\
-    \ with annotated timestamp when done\n docDataSetExtId: ds_files_oid\n # In document\
-    \ metadata, where is location of property for P&ID type documents\n docTypeMetaCol:\
+    \ with annotated timestamp when done\n docDataSetExtId: \"ds_files_oid\"\n # In\
+    \ document metadata, where is location of property for P&ID type documents\n docTypeMetaCol:\
     \ \"doc_type\"\n # Document type for P&ID type documents\n pAndIdDocType: \"PNID\"\
     \n # List of externalId for root assets to be used for annotation of documents.\
     \ Documents are also extracted based on the root asset ID\n assetRootExtIds: [WMT:VAL]\n\
@@ -65,8 +65,8 @@ ExtractionPipeline:
   source: fileshare
 ExtractionPipelineConfig:
 - config: "data:\n  debug: False\n  runAll: False\n  docLimit: -1\n  docDataSetExtId:\
-    \ ds_files_oid\n  docTypeMetaCol: \"doc_type\"\n  pAndIdDocType: \"PNID\"\n  assetRootExtIds:\
-    \ [WMT:VAL]\n  matchThreshold: 0.85\n \n"
+    \ \"ds_files_oid\"\n  docTypeMetaCol: \"doc_type\"\n  pAndIdDocType: \"PNID\"\n\
+    \  assetRootExtIds: [WMT:VAL]\n  matchThreshold: 0.85\n \n"
   externalId: ep_ctx_files_oid_fileshare_pandid_annotation
 - config: "logger:\n  # Logging to console/terminal. Remove or comment out to disable\
     \ terminal\n  # logging\n  console:\n    level: INFO\n\n  # Logging to file. Include\

tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_data_pipeline_timeseries_valhall.yaml

@@ -117,7 +117,7 @@ ExtractionPipelineConfig:
     \  matchThreshold: 0.75\n"
   externalId: ep_ctx_timeseries_oid_opcua_asset
 - config: "# Version of the config schema\nversion: 1\n\nsource:\n    # The URL of\
-    \ the OPC-UA server to connect to\n    endpoint-url: opc.tcp://DESKTOP-18JE0L8:53530/OPCUA/SimulationServer\n\
+    \ the OPC-UA server to connect to\n    endpoint-url: 'opc.tcp://DESKTOP-18JE0L8:53530/OPCUA/SimulationServer'\n\
     \    force-restart: true\n    browse-nodes-chunk: 100\n    subscription-chunk:\
     \ 1000\n    keep-alive-interval: 50000\n    sampling-interval: 0\n    queue-length:\
     \ 10\n    browse-throttling:\n        max-node-parallelism: 1000\n    retries:\n\
@@ -133,9 +133,9 @@ ExtractionPipelineConfig:
     \ \"opc-ua:\"\n\n    # Source node in the OPC-UA server. Leave empty to use the\
     \ top level Objects node.\n    # If root-nodes is set, this is added to the list\
     \ of root nodes.\n    root-node:\n        # Full name of the namespace of the\
-    \ root node.\n        namespace-uri: http://www.prosysopc.com/OPCUA/SimulationNodes/\n\
+    \ root node.\n        namespace-uri: 'http://www.prosysopc.com/OPCUA/SimulationNodes/'\n\
     \        # Id of the root node, on the form \"i=123\" or \"s=stringid\" etc.\n\
-    \        node-id: s=85/0:Simulation\n\n    # Time in minutes between each call\
+    \        node-id: 's=85/0:Simulation'\n\n    # Time in minutes between each call\
     \ to browse the OPC-UA directory, then push new nodes to destinations.\n    #\
     \ Note that this is a heavy operation, so this number should not be set too low.\n\
     \    # Alternatively, use N[timeunit] where timeunit is w, d, h, m, s or ms.\n\

tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_entity_matching.yaml

@@ -33,7 +33,7 @@ ExtractionPipelineConfig:
 - config: "parameters:\n  autoApprovalThreshold: 0.85\n  autoRejectThreshold: 0.25\n\
     \  featureType: \"bigram\"\nsourceSystem:\n  space: springfield_instances\n  externalId:\
     \ entity_matcher\nstate:\n  rawDatabase: contextualizationState\n  rawTable: entityMatching\n\
-    data:\n  annotationSpace: springfield_instances\n  instanceSpaces:\n    ['springfield_instances']\n\
+    data:\n  annotationSpace: 'springfield_instances'\n  instanceSpaces:\n    ['springfield_instances']\n\
     \  matchingJobs:\n    job1:\n      sourceView:\n        space: cdf_cdm\n     \
     \   externalId: CogniteTimeSeries\n        version: v1\n        properties:\n\
     \          - name\n      targetViews:\n        - space: cdf_cdm\n          externalId:\

tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_p_and_id_parser.yaml

@@ -28,7 +28,7 @@ ExtractionPipeline:
 ExtractionPipelineConfig:
 - config: "parameters:\n  autoApprovalThreshold: 0.85\n  autoRejectThreshold: 0.25\n\
     sourceSystem:\n  space: springfield_instances\n  externalId: p_and_id_parser\n\
-    data:\n  annotationSpace: springfield_instances\n  instanceSpaces:\n    ['springfield_instances']\n\
+    data:\n  annotationSpace: 'springfield_instances'\n  instanceSpaces:\n    ['springfield_instances']\n\
     \  annotationJobs:\n    - fileView:\n        space: cdf_cdm\n        externalId:\
     \ CogniteFile\n        version: v1\n      entityViews:\n        - space: cdf_cdm\n\
     \          externalId: CogniteEquipment\n          version: v1\n          searchProperty:\