Expose log retain time (#2034)

make log retain time configurable extend default time to 30 days. add this PR: #1870, otherwise paictl will try to render the host_ip in advance of generate_script
microsoft · Jan 21, 2019 · 4d2cb13 · 4d2cb13
1 parent e9a4e88
commit 4d2cb13
Show file tree

Hide file tree

Showing 16 changed files with 36 additions and 10 deletions.
diff --git a/deployment/quick-start/services-configuration.yaml.template b/deployment/quick-start/services-configuration.yaml.template
@@ -46,6 +46,8 @@ cluster:
 
 #Uncomment following lines if you want to customize yarn
 #hadoop-resource-manager:
+#  # job log retain time
+#  yarn_log_retain_seconds: 2592000
 #  # port for yarn exporter
 #  yarn_exporter_port: 9459
 #  # Step 1 of 4 to set up Hadoop queues.

diff --git a/examples/cluster-configuration/services-configuration.yaml b/examples/cluster-configuration/services-configuration.yaml
@@ -52,6 +52,8 @@
 
 #Uncomment following lines if you want to customize yarn
 #hadoop-resource-manager:
+#  # job log retain time
+#  yarn_log_retain_seconds: 2592000
 #  # port for yarn exporter
 #  yarn_exporter_port: 9459
 #  # Step 1 of 4 to set up Hadoop queues.

diff --git a/src/hadoop-data-node/deploy/hadoop-data-node-configuration/datanode-generate-script.sh b/src/hadoop-data-node/deploy/hadoop-data-node-configuration/datanode-generate-script.sh
@@ -23,8 +23,8 @@ cp  /hadoop-configuration/hadoop-env.sh $HADOOP_CONF_DIR/hadoop-env.sh
 cp  /hadoop-configuration/yarn-env.sh $HADOOP_CONF_DIR/yarn-env.sh
 cp  /hadoop-configuration/mapred-site.xml $HADOOP_CONF_DIR/mapred-site.xml
 
-HOST_NAME=`hostname`
-/usr/local/host-configure.py -c /host-configuration/host-configuration.yaml -f $HADOOP_CONF_DIR/hdfs-site.xml -n $HOST_NAME
 
-sed -i "s/{HDFS_ADDRESS}/${HDFS_ADDRESS}/g" $HADOOP_CONF_DIR/core-site.xml
+
 sed -i "s#{HADOOP_DATANODE_DATA_DIR}#${HADOOP_DATANODE_DATA_DIR}#g" $HADOOP_CONF_DIR/hdfs-site.xml
+sed -i "s/{POD_IP}/${POD_IP}/g" $HADOOP_CONF_DIR/hdfs-site.xml
+sed -i "s/{HDFS_ADDRESS}/${HDFS_ADDRESS}/g" $HADOOP_CONF_DIR/core-site.xml
diff --git a/src/hadoop-data-node/deploy/hadoop-data-node-configuration/hdfs-site.xml b/src/hadoop-data-node/deploy/hadoop-data-node-configuration/hdfs-site.xml
@@ -244,7 +244,7 @@
           rejected. It is recommended that this setting be left on to prevent accidental
           registration of datanodes listed by hostname in the excludes file during a DNS
           outage. Only set this to false in environments where there is no infrastructure
-          to support reverse DNS lookup.  
+          to support reverse DNS lookup.
   </description>
 </property>
 
@@ -268,7 +268,7 @@
 
 <property>
    <name>dfs.datanode.hostname</name>
-   <value>{{ host_config['ip'] }}</value>
+   <value>{POD_IP}</value>
 </property>
 
 <property>

diff --git a/src/hadoop-data-node/deploy/hadoop-data-node.yaml.template b/src/hadoop-data-node/deploy/hadoop-data-node.yaml.template
@@ -73,6 +73,10 @@ spec:
           value: datanode-start-service.sh
         - name: HADOOP_DATANODE_DATA_DIR
           value: {{ mount_points|join(",") }}
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
       imagePullSecrets:
       - name: {{ cluster_cfg["cluster"]["docker-registry"]["secret-name"] }}
       volumes:

diff --git a/...op-jobhistory-configuration/yarn-site.xml → ...tory-configuration/yarn-site.xml.template b/...op-jobhistory-configuration/yarn-site.xml → ...tory-configuration/yarn-site.xml.template
@@ -70,7 +70,7 @@
 
     <property>
       <name>yarn.log-aggregation.retain-seconds</name>
-       <value>604800</value>
+       <value>{{ cluster_cfg[ "hadoop-resource-manager" ][ "yarn_log_retain_seconds" ] }}</value>
     </property>
 
     <property>

diff --git a/src/hadoop-jobhistory/deploy/service.yaml b/src/hadoop-jobhistory/deploy/service.yaml
@@ -27,6 +27,7 @@ prerequisite:
 template-list:
   - hadoop-jobhistory.yaml
   - delete.yaml
+  - hadoop-jobhistory-configuration/yarn-site.xml
 
 start-script: start.sh
 stop-script: stop.sh

diff --git a/...doop-node-manager/deploy/hadoop-node-manager-configuration/nodemanager-generate-script.sh b/...doop-node-manager/deploy/hadoop-node-manager-configuration/nodemanager-generate-script.sh
@@ -53,6 +53,7 @@ sed  -i "s/{TIMELINE_SERVER_ADDRESS}/${TIMELINE_SERVER_ADDRESS}/g" $HADOOP_CONF_
 sed  -i "s#{HOST_YARN_NODEMANAGER_STORAGE}#${HOST_YARN_NODEMANAGER_STORAGE}#g" $HADOOP_CONF_DIR/yarn-site.xml
 sed  -i "s#{HOST_HADOOP_TMP_STORAGE}#${HOST_HADOOP_TMP_STORAGE}#g" $HADOOP_CONF_DIR/yarn-site.xml
 sed  -i "s#{CURRENT_IMAGE_NAME}#${CURRENT_IMAGE_NAME}#g" $HADOOP_CONF_DIR/yarn-site.xml
+sed  -i "s/{POD_IP}/${POD_IP}/g" $HADOOP_CONF_DIR/yarn-site.xml
 
 sed  -i "s/{HDFS_ADDRESS}/${HDFS_ADDRESS}/g" $HADOOP_CONF_DIR/core-site.xml
 

diff --git a/...-node-manager-configuration/yarn-site.xml → ...ager-configuration/yarn-site.xml.template b/...-node-manager-configuration/yarn-site.xml → ...ager-configuration/yarn-site.xml.template
@@ -76,7 +76,7 @@
 
     <property>
       <name>yarn.log-aggregation.retain-seconds</name>
-      <value>604800</value>
+      <value>{{ cluster_cfg[ "hadoop-resource-manager" ][ "yarn_log_retain_seconds" ] }}</value>
     </property>
 
     <property>
@@ -92,7 +92,7 @@
 
     <property>
       <name>yarn.nodemanager.hostname</name>
-      <value>{{ host_config['ip'] }}</value>
+      <value>{POD_IP}</value>
     </property>  
 
     <property>  
@@ -137,7 +137,7 @@
 
     <property>
       <name>yarn.nodemanager.address</name>
-      <value>{{ host_config['ip'] }}:8041</value>
+      <value>{POD_IP}:8041</value>
     </property>
 
     <property>

diff --git a/src/hadoop-node-manager/deploy/hadoop-node-manager.yaml.template b/src/hadoop-node-manager/deploy/hadoop-node-manager.yaml.template
@@ -103,6 +103,10 @@ spec:
           value: "3072"
         - name: NV_DRIVER
           value: /var/drivers/nvidia/current
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
       imagePullSecrets:
       - name: {{ cluster_cfg["cluster"]["docker-registry"]["secret-name"] }}
       volumes:

diff --git a/src/hadoop-node-manager/deploy/service.yaml b/src/hadoop-node-manager/deploy/service.yaml
@@ -26,6 +26,7 @@ prerequisite:
 template-list:
   - hadoop-node-manager.yaml
   - delete.yaml
+  - hadoop-node-manager-configuration/yarn-site.xml
 
 start-script: start.sh
 stop-script: stop.sh

diff --git a/src/hadoop-resource-manager/config/hadoop-resource-manager.md b/src/hadoop-resource-manager/config/hadoop-resource-manager.md
@@ -33,6 +33,7 @@ hadoop-resource-manager:
         capacity: 100
 
     yarn_exporter_port: 9459
+    yarn_log_retain_seconds: 2592000
 ```
 
 
@@ -69,4 +70,10 @@ hadoop-resource-manager:
     <td>cluster_cfg["hadoop-resource-manager"]["yarn_exporter_port"]</td>
     <td>Int</td>
 </tr>
+<tr>
+    <td>hadoop-resource-manager.yarn_log_retain_seconds</td>
+    <td>com["hadoop-resource-manager"]["yarn_log_retain_seconds"]</td>
+    <td>cluster_cfg["hadoop-resource-manager"]["yarn_log_retain_seconds"]</td>
+    <td>Int</td>
+</tr>
 </table>
diff --git a/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml b/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml
@@ -21,3 +21,5 @@ virtualClusters:
     capacity: 100
 
 yarn_exporter_port: 9459
+
+yarn_log_retain_seconds: 2592000
diff --git a/src/hadoop-resource-manager/config/hadoop_resource_manager.py b/src/hadoop-resource-manager/config/hadoop_resource_manager.py
@@ -59,6 +59,7 @@ def run(self):
         com = {}
 
         com["yarn_exporter_port"] = self.service_configuration["yarn_exporter_port"]
+        com["yarn_log_retain_seconds"] = self.service_configuration["yarn_log_retain_seconds"]
 
         for host_config in self.cluster_configuration["machine-list"]:
             if "pai-master" in host_config and host_config["pai-master"] == "true":

diff --git a/...ource-manager-configuration/yarn-site.xml → ...ager-configuration/yarn-site.xml.template b/...ource-manager-configuration/yarn-site.xml → ...ager-configuration/yarn-site.xml.template
@@ -104,7 +104,7 @@
 
     <property>
       <name>yarn.log-aggregation.retain-seconds</name>
-       <value>604800</value>
+       <value>{{ cluster_cfg[ "hadoop-resource-manager" ][ "yarn_log_retain_seconds" ] }}</value>
     </property>
 
     <property>

diff --git a/src/hadoop-resource-manager/deploy/service.yaml b/src/hadoop-resource-manager/deploy/service.yaml
@@ -26,6 +26,7 @@ template-list:
   - hadoop-resource-manager.yaml
   - delete.yaml
   - hadoop-resource-manager-configuration/capacity-scheduler.xml
+  - hadoop-resource-manager-configuration/yarn-site.xml
 
 start-script: start.sh
 stop-script: stop.sh
Original file line number	Diff line number	Diff line change
Expand Up		@@ -21,3 +21,5 @@ virtualClusters:
		capacity: 100

		yarn_exporter_port: 9459

		yarn_log_retain_seconds: 2592000