diff --git a/deployment/quick-start/services-configuration.yaml.template b/deployment/quick-start/services-configuration.yaml.template index 3b48544f11..9e7af66cad 100644 --- a/deployment/quick-start/services-configuration.yaml.template +++ b/deployment/quick-start/services-configuration.yaml.template @@ -46,6 +46,8 @@ cluster: #Uncomment following lines if you want to customize yarn #hadoop-resource-manager: +# # job log retain time +# yarn_log_retain_seconds: 2592000 # # port for yarn exporter # yarn_exporter_port: 9459 # # Step 1 of 4 to set up Hadoop queues. diff --git a/examples/cluster-configuration/services-configuration.yaml b/examples/cluster-configuration/services-configuration.yaml index 91e5224bd7..873633ef64 100644 --- a/examples/cluster-configuration/services-configuration.yaml +++ b/examples/cluster-configuration/services-configuration.yaml @@ -52,6 +52,8 @@ #Uncomment following lines if you want to customize yarn #hadoop-resource-manager: +# # job log retain time +# yarn_log_retain_seconds: 2592000 # # port for yarn exporter # yarn_exporter_port: 9459 # # Step 1 of 4 to set up Hadoop queues. diff --git a/src/hadoop-data-node/deploy/hadoop-data-node-configuration/datanode-generate-script.sh b/src/hadoop-data-node/deploy/hadoop-data-node-configuration/datanode-generate-script.sh index 35221c24ca..b62466292c 100644 --- a/src/hadoop-data-node/deploy/hadoop-data-node-configuration/datanode-generate-script.sh +++ b/src/hadoop-data-node/deploy/hadoop-data-node-configuration/datanode-generate-script.sh @@ -23,8 +23,8 @@ cp /hadoop-configuration/hadoop-env.sh $HADOOP_CONF_DIR/hadoop-env.sh cp /hadoop-configuration/yarn-env.sh $HADOOP_CONF_DIR/yarn-env.sh cp /hadoop-configuration/mapred-site.xml $HADOOP_CONF_DIR/mapred-site.xml -HOST_NAME=`hostname` -/usr/local/host-configure.py -c /host-configuration/host-configuration.yaml -f $HADOOP_CONF_DIR/hdfs-site.xml -n $HOST_NAME -sed -i "s/{HDFS_ADDRESS}/${HDFS_ADDRESS}/g" $HADOOP_CONF_DIR/core-site.xml + sed -i "s#{HADOOP_DATANODE_DATA_DIR}#${HADOOP_DATANODE_DATA_DIR}#g" $HADOOP_CONF_DIR/hdfs-site.xml +sed -i "s/{POD_IP}/${POD_IP}/g" $HADOOP_CONF_DIR/hdfs-site.xml +sed -i "s/{HDFS_ADDRESS}/${HDFS_ADDRESS}/g" $HADOOP_CONF_DIR/core-site.xml diff --git a/src/hadoop-data-node/deploy/hadoop-data-node-configuration/hdfs-site.xml b/src/hadoop-data-node/deploy/hadoop-data-node-configuration/hdfs-site.xml index c763b97556..7a92142d10 100644 --- a/src/hadoop-data-node/deploy/hadoop-data-node-configuration/hdfs-site.xml +++ b/src/hadoop-data-node/deploy/hadoop-data-node-configuration/hdfs-site.xml @@ -244,7 +244,7 @@ rejected. It is recommended that this setting be left on to prevent accidental registration of datanodes listed by hostname in the excludes file during a DNS outage. Only set this to false in environments where there is no infrastructure - to support reverse DNS lookup. + to support reverse DNS lookup. @@ -268,7 +268,7 @@ dfs.datanode.hostname - {{ host_config['ip'] }} + {POD_IP} diff --git a/src/hadoop-data-node/deploy/hadoop-data-node.yaml.template b/src/hadoop-data-node/deploy/hadoop-data-node.yaml.template index 8198f8c6a6..6bbecebb04 100644 --- a/src/hadoop-data-node/deploy/hadoop-data-node.yaml.template +++ b/src/hadoop-data-node/deploy/hadoop-data-node.yaml.template @@ -73,6 +73,10 @@ spec: value: datanode-start-service.sh - name: HADOOP_DATANODE_DATA_DIR value: {{ mount_points|join(",") }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP imagePullSecrets: - name: {{ cluster_cfg["cluster"]["docker-registry"]["secret-name"] }} volumes: diff --git a/src/hadoop-jobhistory/deploy/hadoop-jobhistory-configuration/yarn-site.xml b/src/hadoop-jobhistory/deploy/hadoop-jobhistory-configuration/yarn-site.xml.template similarity index 98% rename from src/hadoop-jobhistory/deploy/hadoop-jobhistory-configuration/yarn-site.xml rename to src/hadoop-jobhistory/deploy/hadoop-jobhistory-configuration/yarn-site.xml.template index 7b1edb5d92..8165893db1 100644 --- a/src/hadoop-jobhistory/deploy/hadoop-jobhistory-configuration/yarn-site.xml +++ b/src/hadoop-jobhistory/deploy/hadoop-jobhistory-configuration/yarn-site.xml.template @@ -70,7 +70,7 @@ yarn.log-aggregation.retain-seconds - 604800 + {{ cluster_cfg[ "hadoop-resource-manager" ][ "yarn_log_retain_seconds" ] }} diff --git a/src/hadoop-jobhistory/deploy/service.yaml b/src/hadoop-jobhistory/deploy/service.yaml index 07897923b3..11df6d0fe7 100644 --- a/src/hadoop-jobhistory/deploy/service.yaml +++ b/src/hadoop-jobhistory/deploy/service.yaml @@ -27,6 +27,7 @@ prerequisite: template-list: - hadoop-jobhistory.yaml - delete.yaml + - hadoop-jobhistory-configuration/yarn-site.xml start-script: start.sh stop-script: stop.sh diff --git a/src/hadoop-node-manager/deploy/hadoop-node-manager-configuration/nodemanager-generate-script.sh b/src/hadoop-node-manager/deploy/hadoop-node-manager-configuration/nodemanager-generate-script.sh index 286443dfab..858dd19dd9 100644 --- a/src/hadoop-node-manager/deploy/hadoop-node-manager-configuration/nodemanager-generate-script.sh +++ b/src/hadoop-node-manager/deploy/hadoop-node-manager-configuration/nodemanager-generate-script.sh @@ -53,6 +53,7 @@ sed -i "s/{TIMELINE_SERVER_ADDRESS}/${TIMELINE_SERVER_ADDRESS}/g" $HADOOP_CONF_ sed -i "s#{HOST_YARN_NODEMANAGER_STORAGE}#${HOST_YARN_NODEMANAGER_STORAGE}#g" $HADOOP_CONF_DIR/yarn-site.xml sed -i "s#{HOST_HADOOP_TMP_STORAGE}#${HOST_HADOOP_TMP_STORAGE}#g" $HADOOP_CONF_DIR/yarn-site.xml sed -i "s#{CURRENT_IMAGE_NAME}#${CURRENT_IMAGE_NAME}#g" $HADOOP_CONF_DIR/yarn-site.xml +sed -i "s/{POD_IP}/${POD_IP}/g" $HADOOP_CONF_DIR/yarn-site.xml sed -i "s/{HDFS_ADDRESS}/${HDFS_ADDRESS}/g" $HADOOP_CONF_DIR/core-site.xml diff --git a/src/hadoop-node-manager/deploy/hadoop-node-manager-configuration/yarn-site.xml b/src/hadoop-node-manager/deploy/hadoop-node-manager-configuration/yarn-site.xml.template similarity index 98% rename from src/hadoop-node-manager/deploy/hadoop-node-manager-configuration/yarn-site.xml rename to src/hadoop-node-manager/deploy/hadoop-node-manager-configuration/yarn-site.xml.template index 133eccc32b..3e6b609101 100644 --- a/src/hadoop-node-manager/deploy/hadoop-node-manager-configuration/yarn-site.xml +++ b/src/hadoop-node-manager/deploy/hadoop-node-manager-configuration/yarn-site.xml.template @@ -76,7 +76,7 @@ yarn.log-aggregation.retain-seconds - 604800 + {{ cluster_cfg[ "hadoop-resource-manager" ][ "yarn_log_retain_seconds" ] }} @@ -92,7 +92,7 @@ yarn.nodemanager.hostname - {{ host_config['ip'] }} + {POD_IP} @@ -137,7 +137,7 @@ yarn.nodemanager.address - {{ host_config['ip'] }}:8041 + {POD_IP}:8041 diff --git a/src/hadoop-node-manager/deploy/hadoop-node-manager.yaml.template b/src/hadoop-node-manager/deploy/hadoop-node-manager.yaml.template index 417e59705d..2d3f4a1da5 100644 --- a/src/hadoop-node-manager/deploy/hadoop-node-manager.yaml.template +++ b/src/hadoop-node-manager/deploy/hadoop-node-manager.yaml.template @@ -103,6 +103,10 @@ spec: value: "3072" - name: NV_DRIVER value: /var/drivers/nvidia/current + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP imagePullSecrets: - name: {{ cluster_cfg["cluster"]["docker-registry"]["secret-name"] }} volumes: diff --git a/src/hadoop-node-manager/deploy/service.yaml b/src/hadoop-node-manager/deploy/service.yaml index 1087469322..43577f4d84 100644 --- a/src/hadoop-node-manager/deploy/service.yaml +++ b/src/hadoop-node-manager/deploy/service.yaml @@ -26,6 +26,7 @@ prerequisite: template-list: - hadoop-node-manager.yaml - delete.yaml + - hadoop-node-manager-configuration/yarn-site.xml start-script: start.sh stop-script: stop.sh diff --git a/src/hadoop-resource-manager/config/hadoop-resource-manager.md b/src/hadoop-resource-manager/config/hadoop-resource-manager.md index 373c7de5e3..28a8ce44a6 100644 --- a/src/hadoop-resource-manager/config/hadoop-resource-manager.md +++ b/src/hadoop-resource-manager/config/hadoop-resource-manager.md @@ -33,6 +33,7 @@ hadoop-resource-manager: capacity: 100 yarn_exporter_port: 9459 + yarn_log_retain_seconds: 2592000 ``` @@ -69,4 +70,10 @@ hadoop-resource-manager: cluster_cfg["hadoop-resource-manager"]["yarn_exporter_port"] Int + + hadoop-resource-manager.yarn_log_retain_seconds + com["hadoop-resource-manager"]["yarn_log_retain_seconds"] + cluster_cfg["hadoop-resource-manager"]["yarn_log_retain_seconds"] + Int + diff --git a/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml b/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml index c68367d550..2330e758e1 100644 --- a/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml +++ b/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml @@ -21,3 +21,5 @@ virtualClusters: capacity: 100 yarn_exporter_port: 9459 + +yarn_log_retain_seconds: 2592000 \ No newline at end of file diff --git a/src/hadoop-resource-manager/config/hadoop_resource_manager.py b/src/hadoop-resource-manager/config/hadoop_resource_manager.py index aec78d60d7..fa200ad09d 100644 --- a/src/hadoop-resource-manager/config/hadoop_resource_manager.py +++ b/src/hadoop-resource-manager/config/hadoop_resource_manager.py @@ -59,6 +59,7 @@ def run(self): com = {} com["yarn_exporter_port"] = self.service_configuration["yarn_exporter_port"] + com["yarn_log_retain_seconds"] = self.service_configuration["yarn_log_retain_seconds"] for host_config in self.cluster_configuration["machine-list"]: if "pai-master" in host_config and host_config["pai-master"] == "true": diff --git a/src/hadoop-resource-manager/deploy/hadoop-resource-manager-configuration/yarn-site.xml b/src/hadoop-resource-manager/deploy/hadoop-resource-manager-configuration/yarn-site.xml.template similarity index 98% rename from src/hadoop-resource-manager/deploy/hadoop-resource-manager-configuration/yarn-site.xml rename to src/hadoop-resource-manager/deploy/hadoop-resource-manager-configuration/yarn-site.xml.template index 6124ad2463..0af29a413a 100644 --- a/src/hadoop-resource-manager/deploy/hadoop-resource-manager-configuration/yarn-site.xml +++ b/src/hadoop-resource-manager/deploy/hadoop-resource-manager-configuration/yarn-site.xml.template @@ -104,7 +104,7 @@ yarn.log-aggregation.retain-seconds - 604800 + {{ cluster_cfg[ "hadoop-resource-manager" ][ "yarn_log_retain_seconds" ] }} diff --git a/src/hadoop-resource-manager/deploy/service.yaml b/src/hadoop-resource-manager/deploy/service.yaml index 91d028cb8c..46a780c7ec 100644 --- a/src/hadoop-resource-manager/deploy/service.yaml +++ b/src/hadoop-resource-manager/deploy/service.yaml @@ -26,6 +26,7 @@ template-list: - hadoop-resource-manager.yaml - delete.yaml - hadoop-resource-manager-configuration/capacity-scheduler.xml + - hadoop-resource-manager-configuration/yarn-site.xml start-script: start.sh stop-script: stop.sh