feat(specs): add ga4BigqueryExport, streaming and append options to t…

…ask inputs (#2858)

specs/ingestion/common/schemas/source.yml

@@ -94,7 +94,7 @@ SourceSearch:
 
 SourceType:
   type: string
-  enum: ['bigcommerce', 'commercetools', 'json', 'csv', 'bigquery', 'docker']
+  enum: ['bigcommerce', 'commercetools', 'json', 'csv', 'bigquery', 'docker', 'ga4BigqueryExport']
 
 SourceCommercetools:
   type: object
@@ -261,6 +261,28 @@ SourceCSV:
   required:
     - url
 
+SourceGA4BigQueryExport:
+  type: object
+  additionalProperties: false
+  properties:
+    projectID:
+      type: string
+      description: GCP project ID that the BigQuery Export writes to.
+    datasetID:
+      type: string
+      description: BigQuery dataset ID that the BigQuery Export writes to.
+    tablePrefix:
+      type: string
+      description: Prefix of the tables that the BigQuery Export writes to (i.e. events_intraday_ for streaming, events_ for daily).
+  required:
+    - projectID
+    - datasetID
+    - tablePrefix
+  x-discriminator-fields:
+    - projectID
+    - datasetID
+    - tablePrefix
+
 SourceBigQuery:
   type: object
   additionalProperties: false
@@ -356,6 +378,7 @@ SourceInput:
     - $ref: '#/SourceJSON'
     - $ref: '#/SourceCSV'
     - $ref: '#/SourceBigQuery'
+    - $ref: '#/SourceGA4BigQueryExport'
     - $ref: '#/SourceDocker'
 
 SourceUpdateCommercetools:
@@ -388,6 +411,7 @@ SourceUpdateInput:
     - $ref: '#/SourceJSON'
     - $ref: '#/SourceCSV'
     - $ref: '#/SourceBigQuery'
+    - $ref: '#/SourceGA4BigQueryExport'
     - $ref: '#/SourceUpdateDocker'
 
 DockerSourceStream:

specs/ingestion/common/schemas/task.yml

@@ -41,6 +41,7 @@ Trigger:
     - $ref: '#/OnDemandTrigger'
     - $ref: '#/ScheduleTrigger'
     - $ref: '#/SubscriptionTrigger'
+    - $ref: '#/StreamingTrigger'
 
 TaskCreate:
   type: object
@@ -76,6 +77,7 @@ TaskCreateTrigger:
     - $ref: '#/OnDemandTriggerInput'
     - $ref: '#/ScheduleTriggerInput'
     - $ref: '#/SubscriptionTrigger'
+    - $ref: '#/StreamingTrigger'
 
 TaskCreateResponse:
   type: object
@@ -134,12 +136,13 @@ TaskSearch:
 ActionType:
   type: string
   description: The action to perform on the Algolia index.
-  enum: ['replace', 'save', 'partial']
+  enum: ['replace', 'save', 'partial', 'append']
 
 TaskInput:
   oneOf:
     - $ref: '#/OnDemandDateUtilsInput'
     - $ref: '#/ScheduleDateUtilsInput'
+    - $ref: '#/StreamingUtilsInput'
 
 TriggerInput:
   type: object
@@ -168,7 +171,8 @@ TriggerType:
       - onDemand: a task that runs manually
       - schedule: a task that runs regularly, following a given cron expression
       - subscription: a task that runs after a subscription event is received from an integration (e.g. Webhook).
-  enum: ['onDemand', 'schedule', 'subscription']
+      - streaming: a task that runs continuously.
+  enum: ['onDemand', 'schedule', 'subscription', 'streaming']
 
 # schedule trigger
 
@@ -264,6 +268,23 @@ SubscriptionTrigger:
   required:
     - type
 
+# streaming trigger
+
+StreamingTriggerType:
+  type: string
+  description: A task that runs continuously.
+  enum: ['streaming']
+
+StreamingTrigger:
+  type: object
+  additionalProperties: false
+  description: The trigger input for a task of type 'streaming'.
+  properties:
+    type:
+      $ref: '#/StreamingTriggerType'
+  required:
+    - type
+
 # Task input
 
 MappingFieldDirective:
@@ -353,6 +374,16 @@ ScheduleDateUtilsInput:
   required:
     - timeframe
 
+StreamingUtilsInput:
+  type: object
+  additionalProperties: false
+  description: The input for a `streaming` task whose source is of type `ga4BigqueryExport` and for which extracted data is continuously streamed.
+  properties:
+    mapping:
+      $ref: '#/MappingInput'
+  required:
+    - mapping
+
 failureThreshold:
   type: integer
   minimum: 0

specs/ingestion/common/taskParameters.yml

@@ -18,12 +18,12 @@ action:
     description: An array of actions to perform on the Algolia index.
     items:
       $ref: './schemas/task.yml#/ActionType'
-    example: save,replace
+    example: save,replace,partial,append
 
 sourceID:
   name: sourceID
   in: query
-  description: The sourceIDs of the tasks to retrive.
+  description: The sourceIDs of the tasks to retrieve.
   required: false
   style: form
   explode: false
@@ -35,7 +35,7 @@ sourceID:
 destinationID:
   name: destinationID
   in: query
-  description: The destinationIDs of the tasks to retrive.
+  description: The destinationIDs of the tasks to retrieve.
   required: false
   style: form
   explode: false
@@ -55,7 +55,7 @@ triggerType:
     type: array
     items:
       $ref: './schemas/task.yml#/TriggerType'
-    example: onDemand,schedule
+    example: onDemand,schedule,subscription,streaming
 
 sort:
   name: sort