-
Notifications
You must be signed in to change notification settings - Fork 3
/
cloud-function.py
43 lines (33 loc) · 1.46 KB
/
cloud-function.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
import os
import requests
import json
def dataprep_job_gcs_trigger(event, context):
"""Background Cloud Function to be triggered by Cloud Storage.
Args:
event (dict): The Cloud Functions event payload.
context (google.cloud.functions.Context): Metadata of triggering event."""
head_tail = os.path.split(event['name'])
newfilename = head_tail[1]
newfilepath = head_tail[0]
datataprep_auth_token = 'xxxxxxxxxxxxxxx'
dataprep_jobid = 99999999
if context.event_type == 'google.storage.object.finalize' and newfilepath == 'landingzone':
print('Run Dataprep job on new file: {}'.format(newfilename))
dataprep_runjob_endpoint = 'https://api.clouddataprep.com/v4/jobGroups'
datataprep_job_param = {
"wrangledDataset": {"id": dataprep_jobid},
"runParameters": {"overrides": {"data": [{"key": "FileName","value": newfilename}]}}
}
print('Run Dataprep job param: {}'.format(datataprep_job_param))
dataprep_headers = {
"Content-Type":"application/json",
"Authorization": "Bearer "+datataprep_auth_token
}
resp = requests.post(
url=dataprep_runjob_endpoint,
headers=dataprep_headers,
data=json.dumps(datataprep_job_param)
)
print('Status Code : {}'.format(resp.status_code))
print('Result : {}'.format(resp.json()))
return 'End File event'.format(newfilename)