YARN and HDFS log persistence (#2244)

* rm log persist * change log dir to host * persist nm log to host * resolve conflict * persist namenode log * persist data node log * add comments * move log path to common pai storage
microsoft · Mar 6, 2019 · 83b9114 · 83b9114
1 parent 8cb09a1
commit 83b9114
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 0 deletions.
diff --git a/src/hadoop-data-node/deploy/hadoop-data-node.yaml.template b/src/hadoop-data-node/deploy/hadoop-data-node.yaml.template
@@ -48,6 +48,8 @@ spec:
           name: host-confg-volume
         - mountPath: /var/lib/hadoopdata
           name: hadoop-tmp-storage
+        - mountPath: /var/log/hadoop
+          name: log-dir
         readinessProbe:
           exec:
             command:
@@ -77,6 +79,11 @@ spec:
           value: datanode-start-service.sh
         - name: HADOOP_DATANODE_DATA_DIR
           value: {{ mount_points|join(",") }}
+        # Rolling File Appender, by default it keeps at most 256M*20=5G logs.
+        - name: HADOOP_ROOT_LOGGER
+          value: INFO,console,RFA
+        - name: HADOOP_LOG_DIR
+          value: /var/log/hadoop
         - name: POD_IP
           valueFrom:
             fieldRef:
@@ -98,6 +105,9 @@ spec:
       - name: host-confg-volume
         configMap:
           name: host-configuration
+      - name: log-dir
+        hostPath:
+          path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/pai-service-log/data-node
       tolerations:
       - key: node.kubernetes.io/memory-pressure
         operator: "Exists"

diff --git a/src/hadoop-name-node/deploy/hadoop-name-node.yaml.template b/src/hadoop-name-node/deploy/hadoop-name-node.yaml.template
@@ -41,6 +41,8 @@ spec:
           name: hadoop-name-node-config-volume
         - mountPath: /var/lib/hadoopdata
           name: hadoop-tmp-storage
+        - mountPath: /var/log/hadoop
+          name: log-dir
         readinessProbe:
           exec:
             command:
@@ -57,6 +59,11 @@ spec:
           value: namenode-generate-script.sh
         - name: START_SERVICE
           value: namenode-start-service.sh
+        # Rolling File Appender, by default it keeps at most 256M*20=5G logs.
+        - name: HADOOP_ROOT_LOGGER
+          value: INFO,console,RFA
+        - name: HADOOP_LOG_DIR
+          value: /var/log/hadoop
         {%- if cluster_cfg['cluster']['common']['qos-switch'] == "true" %}
         resources:
           limits:
@@ -74,3 +81,6 @@ spec:
       - name: hadoop-tmp-storage
         hostPath:
           path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/hadooptmp/namenode
+      - name: log-dir
+        hostPath:
+          path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/pai-service-log/name-node
diff --git a/src/hadoop-node-manager/deploy/hadoop-node-manager.yaml.template b/src/hadoop-node-manager/deploy/hadoop-node-manager.yaml.template
@@ -66,6 +66,8 @@ spec:
           name: host-confg-volume
         - mountPath: /var/lib/hadoopdata
           name: hadoop-tmp-storage
+        - mountPath: /var/log/hadoop
+          name: log-dir
         readinessProbe:
           exec:
             command:
@@ -107,6 +109,11 @@ spec:
           value: "3072"
         - name: NV_DRIVER
           value: /var/drivers/nvidia/current
+        # Rolling File Appender, by default it keeps at most 256M*20=5G logs.
+        - name: YARN_ROOT_LOGGER
+          value: INFO,console,RFA
+        - name: YARN_LOG_DIR
+          value: /var/log/hadoop
         - name: POD_IP
           valueFrom:
             fieldRef:
@@ -144,6 +151,9 @@ spec:
       - name: hadoop-tmp-storage
         hostPath:
           path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/hadooptmp/nodemanager
+      - name: log-dir
+        hostPath:
+          path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/pai-service-log/node-manager
       tolerations:
       - key: node.kubernetes.io/memory-pressure
         operator: "Exists"

diff --git a/src/hadoop-resource-manager/deploy/hadoop-resource-manager.yaml.template b/src/hadoop-resource-manager/deploy/hadoop-resource-manager.yaml.template
@@ -52,6 +52,8 @@ spec:
           name: yarn-resourcemanager-storage
         - mountPath: /var/lib/hadoopdata
           name: hadoop-tmp-storage
+        - mountPath: /var/log/hadoop
+          name: log-dir
         - mountPath: /exclude-path
           name: hadoop-resource-manager-exclude-nodes
         readinessProbe:
@@ -78,6 +80,11 @@ spec:
           value: resourcemanager-generate-script.sh
         - name: START_SERVICE
           value: resourcemanager-start-service.sh
+        # Rolling File Appender, by default it keeps at most 256M*20=5G logs.
+        - name: YARN_ROOT_LOGGER
+          value: INFO,console,RFA
+        - name: YARN_LOG_DIR
+          value: /var/log/hadoop
         {%- if cluster_cfg['cluster']['common']['qos-switch'] == "true" %}
         resources:
           limits:
@@ -115,6 +122,9 @@ spec:
       - name: hadoop-tmp-storage
         hostPath:
           path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/hadooptmp/resourcemanager
+      - name: log-dir
+        hostPath:
+          path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/pai-service-log/resource-manager
       - name: hadoop-resource-manager-exclude-nodes
         configMap:
           name: exclude-file