From 741d46fac4321522baa0658acd222a2de916be2b Mon Sep 17 00:00:00 2001 From: Jerry Westberg Date: Mon, 17 Dec 2018 17:35:43 -0700 Subject: [PATCH] General cleanup + incremental support --- ...rchicalDataConverter.AutomatedTests.csproj | 38 ++--- .../ConfigReaderTests.cs | 4 +- .../GrandparentParentChildTest.cs | 24 +-- .../packages.config | 2 +- .../Catalyst.HierarchicalDataConverter.csproj | 38 ++--- .../HierarchicalDataTransformer.cs | 140 ++++++++++++------ .../packages.config | 2 +- 7 files changed, 150 insertions(+), 98 deletions(-) diff --git a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/Catalyst.HierarchicalDataConverter.AutomatedTests.csproj b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/Catalyst.HierarchicalDataConverter.AutomatedTests.csproj index e7b6c35..7e7c839 100644 --- a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/Catalyst.HierarchicalDataConverter.AutomatedTests.csproj +++ b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/Catalyst.HierarchicalDataConverter.AutomatedTests.csproj @@ -57,59 +57,59 @@ ..\packages\Unity.5.8.13\lib\net46\CommonServiceLocator.dll - - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Client.dll + + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Client.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Config.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Config.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Domain.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Domain.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ElasticSearch.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ElasticSearch.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Http.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Http.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Interfaces.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Interfaces.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Json.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Json.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.PipelineRunner.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.PipelineRunner.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.PipelineSteps.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.PipelineSteps.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ProgressMonitors.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ProgressMonitors.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.QueueItems.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.QueueItems.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Schema.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Schema.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Shared.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Shared.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.SqlGenerator.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.SqlGenerator.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ZipCodeToGeoCode.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ZipCodeToGeoCode.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.ReliableHttp.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.ReliableHttp.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.ReliableSql.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.ReliableSql.dll diff --git a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/ConfigReaderTests.cs b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/ConfigReaderTests.cs index 9f25d26..39a577e 100644 --- a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/ConfigReaderTests.cs +++ b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/ConfigReaderTests.cs @@ -10,6 +10,7 @@ namespace Catalyst.HierarchicalDataConverter.AutomatedTests using System.Reflection; using Catalyst.DataProcessing.Shared.Utilities.Client; + using Catalyst.DataProcessing.Shared.Utilities.Context; using DataConverter; @@ -45,7 +46,8 @@ public void TestReadingTheConfig() configName); File.Copy(configFileLocation, configFileNewLocation, true); var serviceClientMock = new Mock(); - var converter = new HierarchicalDataTransformer(serviceClientMock.Object); + var processingContextWrapperFactoryMock = new Mock(); + var converter = new HierarchicalDataTransformer(serviceClientMock.Object, processingContextWrapperFactoryMock.Object); var privateMethodRunner = new PrivateObject(converter); var args = new object[1] { configName }; var config = (HierarchicalConfiguration)privateMethodRunner.Invoke("GetConfigurationFromJsonFile", args); diff --git a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/GrandparentParentChildTest.cs b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/GrandparentParentChildTest.cs index e1babdb..2245286 100644 --- a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/GrandparentParentChildTest.cs +++ b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/GrandparentParentChildTest.cs @@ -1,20 +1,21 @@ -using System; -using Microsoft.VisualStudio.TestTools.UnitTesting; - -namespace Catalyst.HierarchicalDataConverter.AutomatedTests +namespace Catalyst.HierarchicalDataConverter.AutomatedTests { - using System.Collections.Generic; + using System; using System.Linq; using System.Threading.Tasks; + using Catalyst.DataProcessing.Shared.Models.DataProcessing; using Catalyst.DataProcessing.Shared.Models.Enums; using Catalyst.DataProcessing.Shared.Models.Metadata; using Catalyst.DataProcessing.Shared.Utilities.Client; + using Catalyst.DataProcessing.Shared.Utilities.Context; using DataConverter; using Fabric.Databus.Config; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Moq; [TestClass] @@ -27,7 +28,7 @@ public class GrandparentParentChildTest [TestMethod] public void GrandparentParentChild() { - Binding[] bindings = new Binding[3] + Binding[] bindings = new Binding[] { this.GetNestedBindingLevel0Source(), this.GetNestedBindingLevel1Source(), @@ -48,9 +49,14 @@ public void GrandparentParentChild() serviceClientMock.Setup(mock => mock.GetEntityAsync(1)).Returns(Task.FromResult(this.GetLevel1SourceEntity())); serviceClientMock.Setup(mock => mock.GetEntityAsync(2)).Returns(Task.FromResult(this.GetLevel2SourceEntity())); - var converter = new HierarchicalDataTransformer(serviceClientMock.Object); + var processingContextWrapperFactoryMock = new Mock(); + var processingContextWrapperMock = new Mock(); + processingContextWrapperFactoryMock.Setup(mock => mock.CreateProcessingContextWrapper()).Returns(processingContextWrapperMock.Object); + processingContextWrapperMock.Setup(mock => mock.GetIncrementalValue(It.IsAny())).Returns(new IncrementalValue { LastMaxIncrementalDate = DateTime.Now }); + + var converter = new HierarchicalDataTransformer(serviceClientMock.Object, processingContextWrapperFactoryMock.Object); var privateMethodRunner = new PrivateObject(converter); - object[] args = new object[2] { this.GetNestedBindingLevel0Source(), this.GetNestedDestinationEntity() }; + object[] args = new object[] { this.GetNestedBindingLevel0Source(), new BindingExecution(), this.GetNestedDestinationEntity() }; var jobData = ((Task)privateMethodRunner.Invoke("GetJobData", args)).Result; @@ -89,7 +95,6 @@ public void GrandparentParentChild() Assert.AreEqual("[MyDatabaseName].[Level1TableName].[Level1Entity]", thirdSource.Relationships.Last().Source.Entity); Assert.AreEqual("'id1'", thirdSource.Relationships.Last().Source.Key); - Assert.AreEqual(1, secondSource.Relationships.Count()); Assert.AreEqual("[MyDatabaseName].[Level1TableName].[Level1Entity]", secondSource.Relationships.First().Destination.Entity); Assert.AreEqual("'id0'", secondSource.Relationships.First().Destination.Key); @@ -511,6 +516,5 @@ private Entity GetLevel2SourceEntity() return entity; } - } } diff --git a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/packages.config b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/packages.config index 634f16a..ef3131f 100644 --- a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/packages.config +++ b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.AutomatedTests/packages.config @@ -3,7 +3,7 @@ - + diff --git a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.csproj b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.csproj index 055e98d..e0556e2 100644 --- a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.csproj +++ b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/Catalyst.HierarchicalDataConverter.csproj @@ -46,59 +46,59 @@ ..\packages\Unity.5.8.13\lib\net46\CommonServiceLocator.dll - - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Client.dll + + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Client.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Config.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Config.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Domain.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Domain.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ElasticSearch.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ElasticSearch.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Http.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Http.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Interfaces.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Interfaces.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Json.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Json.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.PipelineRunner.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.PipelineRunner.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.PipelineSteps.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.PipelineSteps.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ProgressMonitors.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ProgressMonitors.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.QueueItems.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.QueueItems.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Schema.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Schema.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Shared.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.Shared.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.SqlGenerator.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.SqlGenerator.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ZipCodeToGeoCode.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Databus.ZipCodeToGeoCode.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.ReliableHttp.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.ReliableHttp.dll - ..\packages\Catalyst.Fabric.Databus.Client.1.1.18350.14\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.ReliableSql.dll + ..\packages\Catalyst.Fabric.Databus.Client.1.1.18351.2\lib\netstandard2.0\bin\Release\netstandard2.0\Fabric.Shared.ReliableSql.dll ..\packages\log4net.2.0.8\lib\net45-full\log4net.dll diff --git a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/HierarchicalDataTransformer.cs b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/HierarchicalDataTransformer.cs index 6ab7091..14ea0fc 100644 --- a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/HierarchicalDataTransformer.cs +++ b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/HierarchicalDataTransformer.cs @@ -22,10 +22,10 @@ namespace DataConverter using Catalyst.DataProcessing.Shared.Models.Enums; using Catalyst.DataProcessing.Shared.Models.Metadata; using Catalyst.DataProcessing.Shared.Utilities.Client; + using Catalyst.DataProcessing.Shared.Utilities.Context; using Fabric.Databus.Client; using Fabric.Databus.Config; - using Fabric.Databus.Interfaces.Http; using Fabric.Shared.ReliableHttp.Interfaces; using Newtonsoft.Json; @@ -38,30 +38,26 @@ namespace DataConverter public class HierarchicalDataTransformer : IDataTransformer { private const string NestedBindingTypeName = "Nested"; - private const string PluginFolderName = "HierarchicalDataConverter"; // Plugin must be placed in this folder within the Plugins folder private const string SourceEntitySourceColumnSeparator = "__"; - /// - /// The helper. - /// private readonly IMetadataServiceClient metadataServiceClient; + private readonly IProcessingContextWrapperFactory processingContextWrapperFactory; private readonly DatabusRunner runner; - /// /// Initializes a new instance of the class. /// - /// - /// The metadata Service Client. - /// - public HierarchicalDataTransformer(IMetadataServiceClient metadataServiceClient) + /// + /// + public HierarchicalDataTransformer(IMetadataServiceClient metadataServiceClient, IProcessingContextWrapperFactory processingContextWrapperFactory) { this.metadataServiceClient = metadataServiceClient ?? throw new ArgumentException("metadataServiceClient cannot be null."); + this.processingContextWrapperFactory = processingContextWrapperFactory ?? throw new ArgumentException("ProcessingContextWrapperFactory cannot be null."); this.runner = new DatabusRunner(); - LoggingHelper2.Debug("We Got Here: HierarchicalDataTransformer!"); + LoggingHelper2.Debug("Created instance of HierarchicalDataTransformer"); } /// @@ -91,10 +87,10 @@ public async Task TransformDataAsync( try { LoggingHelper2.Debug("In TransformDataAsync()"); - var config = this.GetConfigurationFromJsonFile(); + HierarchicalConfiguration config = this.GetConfigurationFromJsonFile(); LoggingHelper2.Debug($"Configuration: {JsonConvert.SerializeObject(config)}"); - var jobData = await this.GetJobData(binding, entity); + JobData jobData = await this.GetJobData(binding, bindingExecution, entity); LoggingHelper2.Debug($"JobData: {JsonConvert.SerializeObject(jobData)}"); this.RunDatabus(config, jobData); @@ -116,8 +112,6 @@ public async Task TransformDataAsync( /// public bool CanHandle(BindingExecution bindingExecution, Binding binding, Entity destinationEntity) { - var guid2 = Guid.NewGuid(); - Binding topMost; try { @@ -126,7 +120,7 @@ public bool CanHandle(BindingExecution bindingExecution, Binding binding, Entity } catch (Exception e) { - LoggingHelper2.Debug($"Threw exception ({guid2.ToString().Substring(0, 10)}): {e}"); + LoggingHelper2.Debug($"Threw exception: {e}"); throw; } @@ -155,7 +149,7 @@ private HierarchicalConfiguration GetConfigurationFromJsonFile(string filePath = throw new InvalidOperationException("Could not find plugin configuration file base path."); } - string fullPath = Path.Combine(directoryName, "Plugins", PluginFolderName, filePath); + string fullPath = Path.Combine(directoryName, filePath); string json = File.ReadAllText(fullPath); dynamic deserialized = JsonConvert.DeserializeObject(json); dynamic databusConfiguration = deserialized.DatabusConfiguration; @@ -192,7 +186,7 @@ private HierarchicalConfiguration GetConfigurationFromJsonFile(string filePath = return hierarchicalConfig; } - private async Task GetJobData(Binding binding, Entity destinationEntity) + private async Task GetJobData(Binding binding, BindingExecution bindingExecution, Entity destinationEntity) { var jobData = new JobData(); @@ -202,7 +196,7 @@ private async Task GetJobData(Binding binding, Entity destinationEntity List dataSources = new List(); - await this.GenerateDataSources(binding, allBindings, destinationEntity, dataSources, null, "$", isFirst: true); + await this.GenerateDataSources(binding, bindingExecution, allBindings, destinationEntity, dataSources, null, "$", isFirst: true); var jobDataTopLevelDataSource = dataSources.First(); jobData.TopLevelDataSource = jobDataTopLevelDataSource as TopLevelDataSource; @@ -226,11 +220,11 @@ private void RunDatabus(HierarchicalConfiguration config, JobData jobData) }; try { - var upmcSpecificConfig = (UpmcSpecificConfig)config.ClientSpecificConfiguration; + UpmcSpecificConfig upmcSpecificConfig = (UpmcSpecificConfig)config.ClientSpecificConfiguration; var container = new UnityContainer(); container.RegisterInstance(new HmacAuthorizationRequestInterceptor( - upmcSpecificConfig.AppId, - upmcSpecificConfig.AppSecret, + upmcSpecificConfig.AppId, + upmcSpecificConfig.AppSecret, upmcSpecificConfig.TenantId, upmcSpecificConfig.TenantSecret)); @@ -239,6 +233,7 @@ private void RunDatabus(HierarchicalConfiguration config, JobData jobData) catch (Exception e) { LoggingHelper2.Debug($"Exception thrown by Databus: {e}"); + throw; } LoggingHelper2.Debug("Finished executing Databus"); @@ -266,6 +261,7 @@ private void ValidateHierarchicalBinding(Binding binding, Binding[] allBindings) private async Task GenerateDataSources( Binding rootBinding, + BindingExecution bindingExecution, Binding[] allBindings, Entity destinationEntity, List dataSources, @@ -273,21 +269,22 @@ private async Task GenerateDataSources( string path, bool isFirst) { - var sourceEntity = await this.GetEntityFromBinding(rootBinding); + Entity sourceEntity = await this.GetEntityFromBinding(rootBinding); + Field[] sourceEntityFields = await this.metadataServiceClient.GetEntityFieldsAsync(sourceEntity); - // LoggingHelper2.Debug(this.guid, $"GenerateDataSources -- sourceEntity: {JsonConvert.SerializeObject(sourceEntity)}"); if (isFirst) { dataSources.Add( new TopLevelDataSource { Path = path, - Key = await this.GetKeyColumnsAsCsv(sourceEntity), + Key = this.GetKeyColumnsAsCsv(sourceEntity, sourceEntityFields), TableOrView = this.GetFullyQualifiedTableName(sourceEntity), MySqlEntityColumnMappings = await this.GetColumnsFromEntity(sourceEntity, destinationEntity, rootBinding.SourcedByEntities.First().SourceAliasName), PropertyType = null, - MyRelationships = new List() + MyRelationships = new List(), + MyIncrementalColumns = this.GetIncrementalConfigurations(rootBinding, bindingExecution, sourceEntityFields) }); } else @@ -304,20 +301,21 @@ await this.GetColumnsFromEntity(sourceEntity, destinationEntity, rootBinding.Sou }); } - var childObjectRelationships = this.GetChildObjectRelationships(rootBinding); - var hasChildren = childObjectRelationships.Count > 0; + List childObjectRelationships = this.GetChildObjectRelationships(rootBinding); + bool hasChildren = childObjectRelationships.Count > 0; if (!hasChildren) { return; } - foreach (var childObjectRelationship in childObjectRelationships) + foreach (ObjectReference childObjectRelationship in childObjectRelationships) { if (childObjectRelationship != null) { - var childBinding = this.GetMatchingChild(allBindings, childObjectRelationship.ChildObjectId); + Binding childBinding = this.GetMatchingChild(allBindings, childObjectRelationship.ChildObjectId); await this.GenerateDataSources( childBinding, + bindingExecution, allBindings, destinationEntity, dataSources, @@ -328,10 +326,61 @@ await this.GenerateDataSources( } } - private async Task GetKeyColumnsAsCsv(Entity sourceEntity) + private List GetIncrementalConfigurations(Binding binding, BindingExecution bindingExecution, Field[] sourceEntityFields) { - Field[] sourceEntityFields = await this.metadataServiceClient.GetEntityFieldsAsync(sourceEntity); + LoggingHelper2.Debug($"IncrementalConfigurations: {JsonConvert.SerializeObject(binding.IncrementalConfigurations)}"); + LoggingHelper2.Debug($"MaxObservedIncrementalDate: {JsonConvert.SerializeObject(bindingExecution.MaxObservedIncrementalDate)}"); + var incrementalColumns = new List(); + if (binding.IncrementalConfigurations.Count == 0) + { + return incrementalColumns; + } + + foreach (IncrementalConfiguration incrementalConfiguration in binding.IncrementalConfigurations) + { + LoggingHelper2.Debug($"IncrementalStartDateTime: {JsonConvert.SerializeObject(bindingExecution.BatchExecution.IncrementalStartDateTime)}"); + + IncrementalValue incrementalValue; + if (bindingExecution.BatchExecution.IncrementalStartDateTime.HasValue) + { + incrementalValue = new IncrementalValue + { + DestinationBindingId = bindingExecution.BindingId, + LastMaxIncrementalDate = bindingExecution.BatchExecution.IncrementalStartDateTime + }; + } + else + { + using (IProcessingContextWrapper processingContextWrapper = this.processingContextWrapperFactory.CreateProcessingContextWrapper()) + { + incrementalValue = processingContextWrapper.GetIncrementalValue(incrementalConfiguration); + } + } + + LoggingHelper2.Debug($"incrementalValue: {JsonConvert.SerializeObject(incrementalValue)}"); + + if (incrementalValue?.LastMaxIncrementalDate == null) + { + continue; + } + + incrementalColumns.Add(new IncrementalColumn + { + Name = incrementalConfiguration.IncrementalColumnName, + Operator = IncrementalOperator.GreaterThanOrEqualTo, + Type = sourceEntityFields.First(f => f.FieldName == incrementalConfiguration.IncrementalColumnName).DataType, + Value = JsonConvert.SerializeObject(incrementalValue.LastMaxIncrementalDate.Value).Replace("\"", string.Empty) + }); + } + + LoggingHelper2.Debug($"incrementalColumns: {JsonConvert.SerializeObject(incrementalColumns)}"); + + return incrementalColumns; + } + + private string GetKeyColumnsAsCsv(Entity sourceEntity, Field[] sourceEntityFields) + { List list = sourceEntityFields.Where(field => field.IsPrimaryKey).Select(field => field.FieldName).ToList(); if (!list.Any()) @@ -356,8 +405,6 @@ private async Task> GetDatabusRelationships(Binding bindin { Entity ancestorEntity = await this.GetEntityFromBinding(allBindings.First(b => b.Id == ancestorRelationship.ParentObjectId)); - // LoggingHelper2.Debug(this.guid, $"SourceEntity: {JsonConvert.SerializeObject(sourceEntity)}"); - // LoggingHelper2.Debug(this.guid, $"ancestorEntity: {JsonConvert.SerializeObject(ancestorEntity)}"); sqlRelationships.Add( new SqlRelationship { @@ -366,13 +413,13 @@ private async Task> GetDatabusRelationships(Binding bindin Entity = this.GetFullyQualifiedTableName(ancestorEntity), Key = this.CleanJson( ancestorRelationship.AttributeValues.GetAttributeTextValue(AttributeName.ParentKeyFields)) - }, // TODO - databus doesn't currently handle comma separated lists here + }, MyDestination = new SqlRelationshipEntity { Entity = this.GetFullyQualifiedTableName(sourceEntity), Key = this.CleanJson(ancestorRelationship.AttributeValues.GetAttributeTextValue(AttributeName.ChildKeyFields)) - } // TODO - databus doesn't currently handle comma separated lists here + } }); } @@ -381,7 +428,7 @@ private async Task> GetDatabusRelationships(Binding bindin private async Task GetBindingsForEntityAsync(Entity entity) { - var bindingsForDataMart = await this.metadataServiceClient.GetBindingsForDataMartAsync(entity.DataMartId); + Binding[] bindingsForDataMart = await this.metadataServiceClient.GetBindingsForDataMartAsync(entity.DataMartId); return bindingsForDataMart.Where(binding => binding.DestinationEntityId == entity.Id).ToArray(); } @@ -389,14 +436,12 @@ private async Task GetBindingsForEntityAsync(Entity entity) private string GetCardinalityFromObjectReference(ObjectReference objectReference) { LoggingHelper2.Debug("Entering GetCardinalityFromObjectReference(...)"); - // LoggingHelper2.Debug(this.guid, $"objectReference: {JsonConvert.SerializeObject(objectReference)}"); return this.GetAttributeValueFromObjectReference(objectReference, AttributeName.Cardinality).Equals("array", StringComparison.CurrentCultureIgnoreCase) ? "array" : "object"; } private string GetAttributeValueFromObjectReference(ObjectReference objectReference, string attributeName) { LoggingHelper2.Debug("Entering GetAttributeValueFromObjectReference(...)"); - // LoggingHelper2.Debug(this.guid, $"objectReference: {JsonConvert.SerializeObject(objectReference)}"); LoggingHelper2.Debug($"attributeName: {attributeName}"); return objectReference.AttributeValues.Where(x => x.AttributeName == attributeName) @@ -412,13 +457,12 @@ private Binding GetMatchingChild(Binding[] bindings, int childBindingId) private List GetChildObjectRelationships(Binding binding) { LoggingHelper2.Debug("Entering GetChildObjectRelationships(...)"); - var childRelationships = binding.ObjectRelationships.Where( + List childRelationships = binding.ObjectRelationships.Where( or => or.ChildObjectType == MetadataObjectType.Binding && or.AttributeValues.First(attr => attr.AttributeName == AttributeName.GenerationGap).ValueToInt() == 1) .ToList(); - // LoggingHelper2.Debug(this.guid, $"Found the following childRelationships for binding with id = {binding.Id}: \n{JsonConvert.SerializeObject(childRelationships)}"); return childRelationships; } @@ -426,7 +470,7 @@ private List GetAncestorObjectRelationships(Binding binding, B { LoggingHelper2.Debug("Entering GetAncestorObjectRelationships(...)"); var parentRelationships = new List(); - foreach (var otherBinding in allBindings.Where(b => b.Id != binding.Id)) + foreach (Binding otherBinding in allBindings.Where(b => b.Id != binding.Id)) { parentRelationships.AddRange( otherBinding.ObjectRelationships.Where( @@ -440,8 +484,6 @@ private List GetAncestorObjectRelationships(Binding binding, B })); } - // LoggingHelper2.Debug(this.guid, $"Found the following parentRelationships for binding with id = {binding.Id}: \n{JsonConvert.SerializeObject(parentRelationships)}"); - return parentRelationships; } @@ -449,14 +491,13 @@ private async Task GetEntityFromBinding(Binding binding) { LoggingHelper2.Debug("Entering GetEntityFromBinding(...)"); - // LoggingHelper2.Debug(this.guid, "binding: " + JsonConvert.SerializeObject(binding)); if (binding == null || !binding.SourcedByEntities.Any() || binding.SourcedByEntities.FirstOrDefault() == null) { return null; } - var entityReference = binding.SourcedByEntities.First(); - var entity = await this.metadataServiceClient.GetEntityAsync(entityReference.SourceEntityId); + SourceEntityReference entityReference = binding.SourcedByEntities.First(); + Entity entity = await this.metadataServiceClient.GetEntityAsync(entityReference.SourceEntityId); LoggingHelper2.Debug($"Found source destinationEntity ({entity.EntityName}) for binding (id = {binding.Id})"); return entity; } @@ -505,5 +546,10 @@ private static class AttributeName public const string ChildKeyFields = "ChildKeyFields"; public const string GenerationGap = "GenerationGap"; } + + private static class IncrementalOperator + { + public const string GreaterThanOrEqualTo = "GreaterThanOrEqualTo"; + } } } diff --git a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/packages.config b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/packages.config index 2321c97..41ed6a2 100644 --- a/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/packages.config +++ b/HierarchicalDataConverter/Catalyst.HierarchicalDataConverter/packages.config @@ -2,7 +2,7 @@ - +