diff --git a/mastering-carol/sql-pipelines/pipe-demo.csql b/mastering-carol/sql-pipelines/pipe-demo.csql new file mode 100644 index 0000000..00269bc --- /dev/null +++ b/mastering-carol/sql-pipelines/pipe-demo.csql @@ -0,0 +1,29 @@ +WITH organization_organization AS ( + SELECT + null AS uuid, + null AS name, + null AS alias, + null AS federalid + --metadata-- + FROM ( + SELECT * EXCEPT(ranking) + FROM ( + SELECT ROW_NUMBER() OVER (partition BY mdmId ORDER BY mdmCounterForEntity DESC) ranking, * + FROM stg_protheus_carol_organization + --timestamp-- WHERE mdmCounterForEntity__DATETIME__ > SAFE.DATETIME(TIMESTAMP_MICROS(SAFE_CAST({{start_from}} AS INT64))) + ) + WHERE ranking = 1 + ) AS stg +), +combinedSources AS ( + SELECT * FROM organization_organization +), +processedData AS ( + SELECT * + FROM combinedSources + -- WHERE + -- rejection rules +) + +SELECT * FROM processedData + diff --git a/mastering-carol/sql-pipelines/pipelines.json b/mastering-carol/sql-pipelines/pipelines.json index e43f6f4..3bdea69 100644 --- a/mastering-carol/sql-pipelines/pipelines.json +++ b/mastering-carol/sql-pipelines/pipelines.json @@ -27,6 +27,29 @@ ] } }, + { + "useBatchNotification": true, + "pipelineName": "pipedemo", + "pipelineDescription": "Pipeline de teste criado durante demo Carol.", + "outputDataModelName": "organization", + "overlapDeltaMinutes": 60, + "saveToCds": false, + "saveToUnified": true, + "sendToSubscriptions": true, + "cronExpressions": ["0 0 * * * ?"], + "timeZone": "America/Sao_Paulo", + "prepareScripts": [""], + "processScript": "pipe-demo.csql", + "sourceEntities": { + "dataModels": [], + "stagings": [ + { + "connectorName": "protheus_carol", + "stagingName": "sa2" + } + ] + } + }, { "useBatchNotification": true, "pipelineName": "organization_pipeline",