NEXT-37536 - Added column conversion via column_type mapping option

CHANGELOG.md

@@ -1,4 +1,5 @@
 # NEXT-RELEASE
+- NEXT-37536 - Added the profile mapping option `column_type` with which the column type can be specified if the inferred one leads to conversion t issues. Valid options are `string`, `number` and `boolean`.
 - NEXT-37310 - Added single row import strategy when encountering an error that cannot be handled automatically during a chunk import.
 
 # v0.8.0

README.md

@@ -128,6 +128,8 @@ mappings:
     entity_path: "name"
   - file_column: "product number"
     entity_path: "productNumber"
+    # column type defines the data type for the internal processing of the column data
+    colum_type: "string"
   - file_column: "stock"
     entity_path: "stock"
   - file_column: "tax id"

profiles/default_customer.yaml

@@ -16,7 +16,8 @@ mappings:
   - file_column: "account type"
     entity_path: "accountType"
   - file_column: "customer number"
-    key: "customer_number"
+    entity_path: "customerNumber"
+    column_type: "string"
   - file_column: "first name"
     entity_path: "firstName"
   - file_column: "last name"
@@ -52,7 +53,8 @@ mappings:
   - file_column: "default billing address street"
     entity_path: "defaultBillingAddress?.street"
   - file_column: "default billing address zip code"
-    key: "default_billing_address_zip_code"
+    entity_path: "defaultBillingAddress?.zipcode"
+    column_type: "string"
   - file_column: "default billing address city"
     entity_path: "defaultBillingAddress?.city"
   - file_column: "default billing address country id"
@@ -74,7 +76,8 @@ mappings:
   - file_column: "default shipping address street"
     entity_path: "defaultShippingAddress?.street"
   - file_column: "default shipping address zip code"
-    key: "default_shipping_address_zip_code"
+    entity_path: "defaultShippingAddress?.zipcode"
+    column_type: "string"
   - file_column: "default shipping address city"
     entity_path: "defaultShippingAddress?.city"
   - file_column: "default shipping address country id"
@@ -90,9 +93,6 @@ mappings:
 
 serialize_script: |
   row = #{
-    customer_number: entity.customerNumber,
-    default_billing_address_zip_code: entity.defaultBillingAddress?.zipCode,
-    default_shipping_address_zip_code: entity.defaultShippingAddress?.zipCode,
     default_group_name: entity.group.translated?.name,
     default_sales_channel_name: entity.salesChannel.translated?.name,
     default_payment_method_name: entity.defaultPaymentMethod.translated?.name
@@ -102,13 +102,6 @@ deserialize_script: |
   let default_language = get_default("LANGUAGE_SYSTEM");
 
   entity = #{
-    customerNumber: `${row.customer_number}`, // TODO: fix conversion (NEXT-37536)
-    defaultBillingAddress: #{
-      zipCode: `${row.default_billing_address_zip_code}`, // TODO: fix conversion (NEXT-37536)
-    },
-    defaultShippingAddress: #{
-      zipCode: `${row.default_shipping_address_zip_code}`, // TODO: fix conversion (NEXT-37536)
-    },
     group: #{
       translations: [#{
         languageId: default_language,

profiles/default_newsletter_recipient.yaml

@@ -26,10 +26,5 @@ mappings:
   - file_column: "hash"
     entity_path: "hash"
   - file_column: "zipCode"
-    key: "zip_code"
-
-serialize_script: |
-  row.zip_code = entity.zipCode;
-
-deserialize_script: |
-  entity.zipCode = `${row.zip_code}`; // TODO: fix conversion (NEXT-37536)
+    entity_path: "zipCode"
+    column_type: "string"

profiles/default_order.yaml

@@ -11,10 +11,9 @@ mappings:
     entity_path: "languageId"
   - file_column: "sales channel id"
     entity_path: "salesChannelId"
-  - file_column: "state id"
-    entity_path: "stateId"
   - file_column: "order number"
-    key: "order_number"
+    entity_path: "orderNumber"
+    column_type: "string"
   - file_column: "currency factor"
     entity_path: "currencyFactor"
   - file_column: "order date time"
@@ -38,7 +37,8 @@ mappings:
   - file_column: "billing address street"
     entity_path: "billingAddress?.street"
   - file_column: "billing address zip code"
-    key: "billing_address_zipcode"
+    entity_path: "billingAddress?.zipcode"
+    column_type: "string"
   - file_column: "billing address company"
     entity_path: "billingAddress?.company"
   - file_column: "billing address city"
@@ -73,14 +73,19 @@ mappings:
     key: "price_position_price"
   - file_column: "price tax rates"
     key: "price_tax_rates"
+    column_type: "string"
   - file_column: "price tax percentages"
     key: "price_tax_percentages"
+    column_type: "string"
   - file_column: "price calculated taxes"
     key: "price_calculated_taxes"
+    column_type: "string"
   - file_column: "price calculated tax prices"
     key: "price_calculated_tax_prices"
+    column_type: "string"
   - file_column: "price calculated tax rates"
     key: "price_calculated_tax_rates"
+    column_type: "string"
   - file_column: "shipping cost quantity"
     key: "shipping_cost_quantity"
   - file_column: "shipping cost unit price"
@@ -89,14 +94,19 @@ mappings:
     key: "shipping_cost_total_price"
   - file_column: "shipping cost tax rates"
     key: "shipping_cost_tax_rates"
+    column_type: "string"
   - file_column: "shipping cost tax percentages"
     key: "shipping_cost_tax_percentages"
+    column_type: "string"
   - file_column: "shipping cost calculated taxes"
     key: "shipping_cost_calculated_taxes"
+    column_type: "string"
   - file_column: "shipping cost calculated tax prices"
     key: "shipping_cost_calculated_tax_prices"
+    column_type: "string"
   - file_column: "shipping cost calculated tax rates"
     key: "shipping_cost_calculated_tax_rates"
+    column_type: "string"
 
 serialize_script: |
   fn encode_values(arr, value_name) {
@@ -114,8 +124,6 @@ serialize_script: |
   }
 
   row = #{
-    order_number: entity.orderNumber,
-    billing_address_zipcode: entity.billingAddress?.zipcode,
     item_rounding_decimals: entity.itemRounding?.decimals,
     item_rounding_interval: entity.itemRounding?.interval,
     item_rounding_round_for_net: entity.itemRounding?.roundForNet,
@@ -214,10 +222,6 @@ deserialize_script: |
   }
 
   entity = #{
-    orderNumber: `${row.order_number}`, // TODO: fix conversion (NEXT-37536)
-    billingAddress: #{
-      zipcode: `${row.billing_address_zipcode}`, // TODO: fix conversion (NEXT-37536)
-    },
     itemRounding: #{
       decimals: row.item_rounding_decimals,
       interval: row.item_rounding_interval,

profiles/default_product.yaml

@@ -23,6 +23,7 @@ mappings:
     entity_path: "cover"
   - file_column: "product number"
     entity_path: "productNumber"
+    column_type: "string"
   - file_column: "active"
     entity_path: "active"
   - file_column: "stock"

profiles/default_product_variants.yaml

@@ -26,6 +26,7 @@ mappings:
     entity_path: "cover"
   - file_column: "product number"
     entity_path: "productNumber"
+    column_type: "string"
   - file_column: "active"
     entity_path: "active"
   - file_column: "stock"

profiles/default_promotion_discount.yaml

@@ -18,14 +18,9 @@ mappings:
   - file_column: "sorter key"
     entity_path: "sorterKey"
   - file_column: "applier key"
-    key: "applier_key"
+    entity_path: "applierKey"
+    column_type: "string"
   - file_column: "usage key"
     entity_path: "usageKey"
   - file_column: "picker key"
     entity_path: "pickerKey"
-
-serialize_script: |
-  row.applier_key = entity.applierKey;
-
-deserialize_script: |
-  entity.applierKey = `${row.applier_key}`; // TODO: fix conversion (NEXT-37536)

src/config_file.rs

@@ -132,13 +132,23 @@ impl Mapping {
 pub struct EntityPathMapping {
     pub file_column: String,
     pub entity_path: String,
+    pub column_type: Option<ColumnType>,
 }
 
 #[derive(Debug, Clone, Default, Eq, PartialEq, Deserialize)]
 pub struct EntityScriptMapping {
     pub file_column: String,
     /// used as an identifier inside the script
     pub key: String,
+    pub column_type: Option<ColumnType>,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum ColumnType {
+    String,
+    Number,
+    Boolean,
 }
 
 #[cfg(test)]

src/data/import.rs

@@ -4,7 +4,7 @@ use crate::api::filter::Criteria;
 use crate::api::{Entity, SwApiError, SwError, SwErrorBody, SyncAction};
 use crate::data::transform::deserialize_row;
 use crate::SyncContext;
-use anyhow::anyhow;
+use anyhow::{anyhow, Context};
 use csv::StringRecord;
 use itertools::Itertools;
 use std::sync::Arc;
@@ -55,10 +55,10 @@ fn process_file_chunk(
             let headers = &headers;
             s.spawn_fifo(move |_| {
                 println!("sync chunk {first_index}..={last_index} (size={chunk_length}) is now being deserialized");
-                let entity_chunk = match deserialize_chunk(headers, records_chunk, &context_clone) {
+                let entity_chunk = match deserialize_chunk(headers, first_index, records_chunk, &context_clone) {
                     Ok(chunk) => chunk,
                     Err(e) => {
-                        println!("sync chunk {first_index}..={last_index} (size={chunk_length}) failed to deserialize:\n{e}");
+                        println!("sync chunk {first_index}..={last_index} (size={chunk_length}) failed to deserialize:\n{e:#}");
                         return;
                     }
                 };
@@ -76,24 +76,21 @@ fn process_file_chunk(
 
 fn deserialize_chunk(
     headers: &StringRecord,
+    first_index: usize,
     records_chunk: Vec<Result<StringRecord, csv::Error>>,
     context: &Arc<SyncContext>,
 ) -> anyhow::Result<Vec<Entity>> {
     let mut entities: Vec<Entity> = Vec::with_capacity(Criteria::MAX_LIMIT);
-    for record in records_chunk {
+    for (record_counter, record) in records_chunk.into_iter().enumerate() {
         let record = record?; // fail on first CSV read failure
 
-        let entity = match deserialize_row(
+        let entity = deserialize_row(
             headers,
             &record,
             &context.profile,
             &context.scripting_environment,
-        ) {
-            Ok(e) => e,
-            Err(e) => {
-                return Err(e);
-            }
-        };
+        )
+        .with_context(|| format!("error in row {}", record_counter + first_index))?;
 
         entities.push(entity);
     }