Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Datadog Pipeline changes #37

Merged
merged 8 commits into from
Sep 11, 2023
Merged

Datadog Pipeline changes #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f1ae0e7
Removing time_key from fluentd config
MahithaB Jul 21, 2023
7a74f89
Adding parse json
MahithaB Jul 21, 2023
af3b7ff
Adding dd_source, service for rt, xray
MahithaB Jul 21, 2023
3c11008
Removing frontend service log
MahithaB Jul 21, 2023
80ed161
Changing type to json
MahithaB Jul 21, 2023
7ff124e
Changing remote_address name
MahithaB Aug 3, 2023
66a440a
Renaming remote_address back to old address
MahithaB Aug 3, 2023
22f879f
Replacing partnership fluentd image with releases image
MahithaB Aug 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 4 additions & 38 deletions fluent.conf.rt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,16 +116,6 @@
@type none
</parse>
</source>
<source>
@type tail
@id frontend_service_tail
path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log"
pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log.pos"
tag jfrog.rt.frontend.service
<parse>
@type none
</parse>
</source>
<source>
@type tail
@id metadata_service_tail
Expand Down Expand Up @@ -157,8 +147,6 @@
@type multiline
format_firstline /\d{4}-\d{1,2}-\d{1,2}/
format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</source>
## REQUEST LOGS
Expand All @@ -171,8 +159,6 @@
<parse>
@type regexp
expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
types response_content_length:integer, request_content_length:integer, return_status:integer
</parse>
</source>
Expand Down Expand Up @@ -213,9 +199,7 @@
pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos"
tag jfrog.rt.router.request
<parse>
@type json
time_key time
time_format %Y-%m-%dT%H:%M:%S%:z
@type none
</parse>
</source>
<source>
Expand All @@ -224,11 +208,10 @@
path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request-out.log"
pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request-out.log.pos"
tag jfrog.rt.request.out
read_from_head true
<parse>
@type regexp
expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_repo_name>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<remote_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)$
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</source>
## ACCESS LOG
Expand All @@ -241,8 +224,6 @@
<parse>
@type regexp
expression /^(?<log_timestamp>[^ ]*) \[(?<trace_id>[^\]]*)\] \[(?<action_response>[^\]]*)\] (?<repo_path>.*) for client : (?<username>.+)\/(?<ip>\s*\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})\.(?<message>.+)?$/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</source>
## AUDIT LOG
Expand All @@ -255,8 +236,6 @@
<parse>
@type regexp
expression /^(?<log_timestamp>[^ ]*)\|(?<token_id>[^ ]*)\|(?<user_ip>[^ ]*)\|(?<user>[^ ]*)\|(?<logged_principal>[^ ]*)\|(?<entity_name>[^ ]*)\|(?<event_type>[^ ]*)\|(?<event>[^ ]*)\|(?<data_changed>.*)/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</source>

Expand Down Expand Up @@ -290,8 +269,6 @@
@type multiline
format_firstline /\d{4}-\d{1,2}-\d{1,2}/
format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</filter>
<filter jfrog.rt.artifactory.service>
Expand All @@ -301,8 +278,6 @@
@type multiline
format_firstline /\d{4}-\d{1,2}-\d{1,2}/
format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</filter>
<filter jfrog.rt.metadata.service>
Expand All @@ -312,8 +287,6 @@
@type multiline
format_firstline /\d{4}-\d{1,2}-\d{1,2}/
format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</filter>
<filter jfrog.rt.router.service>
Expand All @@ -323,8 +296,6 @@
@type multiline
format_firstline /\d{4}-\d{1,2}-\d{1,2}/
format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</filter>
## REQUEST LOGS
Expand All @@ -334,8 +305,6 @@
<parse>
@type regexp
expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</filter>
<filter jfrog.rt.artifactory.request>
Expand All @@ -360,8 +329,6 @@
<parse>
@type regexp
expression ^(?<log_timestamp>[^\|]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</filter>
<filter jfrog.rt.metadata.request>
Expand All @@ -370,8 +337,6 @@
<parse>
@type regexp
expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</filter>
## ACCESS LOGS
Expand Down Expand Up @@ -403,7 +368,8 @@
api_key "#{ENV['DATADOG_API_KEY']}"
#optional
include_tag_key true
dd_source jfrog_artifactory
dd_source jfrog_platform
service jfrog_artifactory
host http-intake.logs.datadoghq.com
<buffer>
flush_interval 1s
Expand Down
24 changes: 5 additions & 19 deletions fluent.conf.xray
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,8 @@
api_key "#{ENV['DATADOG_API_KEY']}"
#optional
include_tag_key true
dd_source jfrog_xray
dd_source jfrog_platform
service jfrog_xray
host http-intake.logs.datadoghq.com
<buffer>
flush_interval 1s
Expand Down Expand Up @@ -208,8 +209,6 @@
<parse>
@type regexp
expression ^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.+)$
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</source>
# REQUEST LOGS
Expand All @@ -220,9 +219,7 @@
pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos"
tag jfrog.xray.router.request
<parse>
@type json
time_key time
time_format %Y-%m-%dT%H:%M:%S%:z
@type none
</parse>
</source>
<source>
Expand All @@ -234,8 +231,6 @@
<parse>
@type regexp
expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^ ]*)\|(?<remote_address>[^|]++)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>.*)$
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</source>

Expand Down Expand Up @@ -269,8 +264,6 @@
@type multiline
format_firstline /\d{4}-\d{1,2}-\d{1,2}/
format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
emit_invalid_record_to_error false
</filter>
Expand All @@ -281,8 +274,6 @@
@type multiline
format_firstline /\d{4}-\d{1,2}-\d{1,2}/
format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
emit_invalid_record_to_error false
</filter>
Expand All @@ -293,8 +284,6 @@
@type multiline
format_firstline /\d{4}-\d{1,2}-\d{1,2}/
format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</filter>
<filter jfrog.xray.router.service>
Expand All @@ -304,8 +293,6 @@
@type multiline
format_firstline /\d{4}-\d{1,2}-\d{1,2}/
format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</filter>
<filter jfrog.xray.server.service>
Expand All @@ -315,8 +302,6 @@
@type multiline
format_firstline /\d{4}-\d{1,2}-\d{1,2}/
format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
time_key log_timestamp
time_format %Y-%m-%dT%H:%M:%S.%LZ
</parse>
</filter>
## ALL LOGS
Expand All @@ -335,7 +320,8 @@
api_key "#{ENV['DATADOG_API_KEY']}"
#optional
include_tag_key true
dd_source jfrog_xray
dd_source jfrog_platform
service jfrog_xray
host http-intake.logs.datadoghq.com
<buffer>
flush_interval 1s
Expand Down
4 changes: 2 additions & 2 deletions helm/artifactory-ha-values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@ artifactory:
- '-c'
- >
mkdir -p {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/;
curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/master/fluent.conf.rt -o {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf
curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/Pipeline_changes/fluent.conf.rt -o {{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf
volumeMounts:
- mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
name: volume
customSidecarContainers: |
- name: "artifactory-fluentd-sidecar"
image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.27.0"
image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.0"
imagePullPolicy: "IfNotPresent"
volumeMounts:
- mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
Expand Down
2 changes: 1 addition & 1 deletion helm/artifactory-values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ artifactory:
name: artifactory-volume
customSidecarContainers: |
- name: "artifactory-fluentd-sidecar"
image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.27.0"
image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.0"
imagePullPolicy: "IfNotPresent"
volumeMounts:
- mountPath: "{{ .Values.artifactory.persistence.mountPath }}"
Expand Down
4 changes: 2 additions & 2 deletions helm/xray-values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@ common:
- '-c'
- >
mkdir -p {{ .Values.xray.persistence.mountPath }}/etc/fluentd/;
curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/master/fluent.conf.xray -o {{ .Values.xray.persistence.mountPath }}/etc/fluentd/fluentd.conf;
curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/Pipeline_changes/fluent.conf.xray -o {{ .Values.xray.persistence.mountPath }}/etc/fluentd/fluentd.conf;
volumeMounts:
- mountPath: "{{ .Values.xray.persistence.mountPath }}"
name: data-volume
customSidecarContainers: |
- name: "xray-platform-fluentd-sidecar"
image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.27.0"
image: "releases-pts-observability-fluentd.jfrog.io/fluentd:2.0"
imagePullPolicy: "IfNotPresent"
volumeMounts:
- mountPath: "{{ .Values.xray.persistence.mountPath }}"
Expand Down
Loading