Merge master-2.x commits 2023/04/01~2023/04/30 into main

Merge missing commits from master-2.x to main. The commits in 2023/04/01~2023/04/30 from main...master-2.x will be included by this PR.

**Exceptions**
4435584 was skipped because it has been manually cherry-picked to main by f64314c

9a4e154 and fd19fb6 are in a wrong order (ported in the end)

[new checkpointing for embedded journal](8cbcbcd) is not in, need manual work

[rockdb thread safety fix](9f152c5) is not in, need manual work

bf60aef depends on the commit above

[fsadmin report proxy command](dbac084) is not in, need manual work

[fsadmin report jobservice show job master/worker versions](8da5953) is not in, need manual work

32f923e is not included because it has been manually cherry-picked by #17361

[graceful decommission worker](141ee0e) is not in, need manual work

[fsadmin report capacity command show worker version](26257e6) is not in, need manual work


			pr-link: #17454
			change-id: cid-1e0faae7f382429b4405a02d4a036d116757070e

bin/alluxio-common.sh

@@ -68,3 +68,13 @@ function get_ramdisk_array() {
   done
   IFS=$oldifs
 }
+
+# Compose the ssh command according to the hostname
+function ssh_command() {
+  local host=$1
+  local command=""
+  if [[ $host != "localhost" && $host != "127.0.0.1" ]]; then
+    command="ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${host}"
+  fi
+  echo "${command}"
+}

bin/alluxio-masters.sh

@@ -12,12 +12,7 @@
 
 set -o pipefail
 
-LAUNCHER=
-# If debugging is enabled propagate that through to sub-shells
-if [[ "$-" == *x* ]]; then
-  LAUNCHER="bash -x"
-fi
-BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd)
+. $(dirname "$0")/alluxio-common.sh
 
 USAGE="Usage: alluxio-masters.sh command..."
 
@@ -46,10 +41,10 @@ fi
 for master in ${HOSTLIST[@]}; do
   echo "[${master}] Connecting as ${USER}..." >> ${ALLUXIO_TASK_LOG}
   if [[ ${HA_ENABLED} == "true" || ${N} -eq 0 ]]; then
-    nohup ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${master} ${LAUNCHER} \
+    nohup $(ssh_command ${master}) ${LAUNCHER} \
       $"${@// /\\ }" 2>&1 | while read line; do echo "[$(date '+%F %T')][${master}] ${line}"; done >> ${ALLUXIO_TASK_LOG} &
   else
-    nohup ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${master} ${LAUNCHER} \
+    nohup $(ssh_command ${master}) ${LAUNCHER} \
       $"export ALLUXIO_MASTER_SECONDARY=true; ${@// /\\ }" 2>&1 | while read line; do echo "[$(date '+%F %T')][${master}] ${line}"; done >> ${ALLUXIO_TASK_LOG} &
   fi
   pids[${#pids[@]}]=$!

bin/alluxio-workers.sh

@@ -12,12 +12,7 @@
 
 set -o pipefail
 
-LAUNCHER=
-# If debugging is enabled propagate that through to sub-shells
-if [[ "$-" == *x* ]]; then
-  LAUNCHER="bash -x"
-fi
-BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd)
+. $(dirname "$0")/alluxio-common.sh
 
 USAGE="Usage: alluxio-workers.sh command..."
 
@@ -39,7 +34,7 @@ echo "Executing the following command on all worker nodes and logging to ${ALLUX
 
 for worker in ${HOSTLIST[@]}; do
   echo "[${worker}] Connecting as ${USER}..." >> ${ALLUXIO_TASK_LOG}
-  nohup ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -tt ${worker} ${LAUNCHER} \
+  nohup $(ssh_command ${worker}) ${LAUNCHER} \
     $"${@// /\\ }" 2>&1 | while read line; do echo "[$(date '+%F %T')][${worker}] ${line}"; done >> ${ALLUXIO_TASK_LOG} &
   pids[${#pids[@]}]=$!
 done

common/transport/src/main/proto/grpc/meta_master.proto

@@ -77,6 +77,13 @@ message MasterInfo {
  optional string clusterId = 11;
  optional bool raftJournal = 12;
  repeated string raftAddress = 13;
+ repeated MasterVersion masterVersions = 14;
+}
+
+message MasterVersion {
+  optional grpc.NetAddress addresses = 1;
+  optional string version = 2;
+  optional string state = 3;
 }
 
 enum MasterInfoField {
@@ -93,6 +100,7 @@ enum MasterInfoField {
   CLUSTER_ID = 10;
   RAFT_JOURNAL = 11;
   RAFT_ADDRESSES = 12;
+  MASTER_VERSION = 13;
 }
 
 message GetMasterInfoPOptions {

common/transport/src/main/proto/proto.lock

@@ -5969,6 +5969,10 @@
               {
                 "name": "RAFT_ADDRESSES",
                 "integer": 12
+              },
+              {
+                "name": "MASTER_VERSION",
+                "integer": 13
               }
             ]
           },
@@ -6235,6 +6239,32 @@
                 "name": "raftAddress",
                 "type": "string",
                 "is_repeated": true
+              },
+              {
+                "id": 14,
+                "name": "masterVersions",
+                "type": "MasterVersion",
+                "is_repeated": true
+              }
+            ]
+          },
+          {
+            "name": "MasterVersion",
+            "fields": [
+              {
+                "id": 1,
+                "name": "addresses",
+                "type": "grpc.NetAddress"
+              },
+              {
+                "id": 2,
+                "name": "version",
+                "type": "string"
+              },
+              {
+                "id": 3,
+                "name": "state",
+                "type": "string"
               }
             ]
           },

dev/scripts/src/alluxio.org/build-distribution/cmd/generate-tarball.go

@@ -220,6 +220,8 @@ func addAdditionalFiles(srcPath, dstPath string, hadoopVersion version, version
 			"integration/metrics/otel-agent-config-worker.yaml",
 			"integration/metrics/otel-collector-config.yaml",
 			"integration/metrics/prometheus.yaml",
+			"integration/tools/ratis-shell/install-ratis-shell.sh",
+			"integration/tools/ratis-shell/README.md",
 		)
 	}
 

docs/_data/table/common-configuration.csv

@@ -83,7 +83,7 @@ alluxio.network.host.resolution.timeout,"5sec"
 alluxio.network.ip.address.used,"false"
 alluxio.proxy.audit.logging.enabled,"false"
 alluxio.proxy.s3.bucket.naming.restrictions.enabled,"false"
-alluxio.proxy.s3.bucketpathcache.timeout,"1min"
+alluxio.proxy.s3.bucketpathcache.timeout,"0min"
 alluxio.proxy.s3.complete.multipart.upload.keepalive.enabled,"false"
 alluxio.proxy.s3.complete.multipart.upload.keepalive.time.interval,"30sec"
 alluxio.proxy.s3.complete.multipart.upload.min.part.size,"5MB"
@@ -118,7 +118,6 @@ alluxio.site.conf.dir,"${alluxio.conf.dir}/,${user.home}/.alluxio/,/etc/alluxio/
 alluxio.site.conf.rocks.block.file,""
 alluxio.site.conf.rocks.inode.file,""
 alluxio.standalone.fuse.jvm.monitor.enabled,"false"
-alluxio.standby.master.grpc.enabled,"false"
 alluxio.standby.master.metrics.sink.enabled,"false"
 alluxio.standby.master.web.enabled,"false"
 alluxio.table.catalog.path,"/catalog"

docs/_data/table/en/common-configuration.yml

@@ -167,7 +167,7 @@ alluxio.proxy.audit.logging.enabled:
 alluxio.proxy.s3.bucket.naming.restrictions.enabled:
   'Toggles whether or not the Alluxio S3 API will enforce AWS S3 bucket naming restrictions. See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html.'
 alluxio.proxy.s3.bucketpathcache.timeout:
-  'Expire bucket path statistics in cache for this time period. Set 0min to disable the cache.'
+  'Expire bucket path statistics in cache for this time period. Set 0min to disable the cache. If enabling the cache, be careful that Alluxio S3 API will behave differently from AWS S3 API if bucket path cache entries become stale.'
 alluxio.proxy.s3.complete.multipart.upload.keepalive.enabled:
   'Whether or not to enabled sending whitespace characters as a keepalive message during CompleteMultipartUpload. Enabling this will cause any errors to be silently ignored. However, the errors will appear in the Proxy logs.'
 alluxio.proxy.s3.complete.multipart.upload.keepalive.time.interval:
@@ -236,8 +236,6 @@ alluxio.site.conf.rocks.inode.file:
   'Path of file containing RocksDB inode store configuration. A template configuration cab be found at ${alluxio.conf.dir}/rocks-inode.ini.template. See https://github.com/facebook/rocksdb/blob/main/examples/rocksdb_option_file_example.ini for more information on RocksDB configuration files. If unset then a default configuration will be used.'
 alluxio.standalone.fuse.jvm.monitor.enabled:
   'Whether to enable start JVM monitor thread on the standalone fuse process. This will start a thread to detect JVM-wide pauses induced by GC or other reasons.'
-alluxio.standby.master.grpc.enabled:
-  'Whether a standby master runs a grpc server'
 alluxio.standby.master.metrics.sink.enabled:
   'Whether a standby master runs the metric sink'
 alluxio.standby.master.web.enabled:

docs/_data/table/en/master-configuration.yml

@@ -72,6 +72,10 @@ alluxio.master.embedded.journal.retry.cache.expiry.time:
   'The time for embedded journal server retry cache to expire. Setting a bigger value allows embedded journal server to cache the responses for a longer time in case of journal writer retries, but will take up more memory in master.'
 alluxio.master.embedded.journal.snapshot.replication.chunk.size:
   'The stream chunk size used by masters to replicate snapshots.'
+alluxio.master.embedded.journal.snapshot.replication.compression.level:
+  'The zip compression level of sending a snapshot from one master to another. Only applicable when alluxio.master.embedded.journal.snapshot.replication.compression.type is not NO_COMPRESSION. The zip format defines ten levels of compression, ranging from 0 (no compression, but very fast) to 9 (best compression, but slow). Or -1 for the system default compression level.'
+alluxio.master.embedded.journal.snapshot.replication.compression.type:
+  'The type of compression to use when transferring a snapshot from one master to another. Options are NO_COMPRESSION, GZIP, TAR_GZIP'
 alluxio.master.embedded.journal.transport.max.inbound.message.size:
   'The maximum size of a message that can be sent to the embedded journal server node.'
 alluxio.master.embedded.journal.transport.request.timeout.ms:
@@ -126,8 +130,6 @@ alluxio.master.journal.init.from.backup:
   'A uri for a backup to initialize the journal from. When the master becomes primary, if it sees that its journal is freshly formatted, it will restore its state from the backup. When running multiple masters, this property must be configured on all masters since it isn''t known during startup which master will become the first primary.'
 alluxio.master.journal.local.log.compaction:
   'Whether to employ a quorum level log compaction policy or a local (individual) log compaction policy.'
-alluxio.master.journal.log.concurrency.max:
-  'Max concurrency for notifyTermIndexUpdated method, be sure it''s enough'
 alluxio.master.journal.log.size.bytes.max:
   'If a log file is bigger than this value, it will rotate to next file.'
 alluxio.master.journal.request.data.timeout:
@@ -244,8 +246,8 @@ alluxio.master.metastore.rocks.block.meta.cache.size:
   'The capacity in bytes of the RocksDB block metadata table LRU  cache. If unset, the RocksDB default will be used. See https://github.com/facebook/rocksdb/wiki/Block-Cache'
 alluxio.master.metastore.rocks.block.meta.index:
   'The index type to be used in the RocksDB block metadata table. If unset, the RocksDB default will be used. See https://github.com/facebook/rocksdb/wiki/Index-Block-Format'
-alluxio.master.metastore.rocks.checkpoint.compression.level:
-  'The zip compression level of checkpointing rocksdb, the zip format defines ten levels of compression, ranging from 0 (no compression, but very fast) to 9 (best compression, but slow). Or -1 for the system default compression level.'
+alluxio.master.metastore.rocks.checkpoint.compression.type:
+  'The compression algorithm that RocksDB uses internally. One of {NO_COMPRESSION SNAPPY_COMPRESSION ZLIB_COMPRESSION BZLIB2_COMPRESSION LZ4_COMPRESSION LZ4HC_COMPRESSION XPRESS_COMPRESSION ZSTD_COMPRESSION DISABLE_COMPRESSION_OPTION}'
 alluxio.master.metastore.rocks.edge.block.index:
   'The block index type to be used in the RocksDB inode edge table. If unset, the RocksDB default will be used. See https://rocksdb.org/blog/2018/08/23/data-block-hash-index.html'
 alluxio.master.metastore.rocks.edge.bloom.filter:

docs/_data/table/en/master-metrics.yml

@@ -48,8 +48,32 @@ Master.EdgeCacheSize:
   'Total number of edges (inode metadata) cached. The edge cache is responsible for managing the mapping from (parentId, childName) to childId.'
 Master.EdgeLockPoolSize:
   'The size of master edge lock pool'
+Master.EmbeddedJournalLastSnapshotDownloadDiskSize:
+  'Describes the size on disk of the snapshot downloaded from other masters in the cluster the previous time the download occurred. Only valid when using the embedded journal.'
+Master.EmbeddedJournalLastSnapshotDownloadDurationMs:
+  'Describes the amount of time taken to download journal snapshots from other masters in the cluster the previous time the download occurred. Only valid when using the embedded journal.'
+Master.EmbeddedJournalLastSnapshotDownloadSize:
+  'Describes the size of the snapshot downloaded from other masters in the cluster the previous time the download occurred. Only valid when using the embedded journal.'
+Master.EmbeddedJournalLastSnapshotDurationMs:
+  'Describes the amount of time taken to generate the last local journal snapshots on this master. Only valid when using the embedded journal.'
+Master.EmbeddedJournalLastSnapshotEntriesCount:
+  'Describes the number of entries in the last local journal snapshots on this master. Only valid when using the embedded journal.'
+Master.EmbeddedJournalLastSnapshotReplayDurationMs:
+  'Represents the time the last restore from checkpoint operation took in milliseconds.'
+Master.EmbeddedJournalLastSnapshotReplayEntriesCount:
+  'Represents the time the last restore from checkpoint operation took in milliseconds.'
+Master.EmbeddedJournalLastSnapshotUploadDiskSize:
+  'Describes the size on disk of the snapshot uploaded to other masters in the cluster the previous time the download occurred. Only valid when using the embedded journal.'
+Master.EmbeddedJournalLastSnapshotUploadDurationMs:
+  'Describes the amount of time taken to upload journal snapshots to another master in the cluster the previous time the upload occurred. Only valid when using the embedded journal.'
+Master.EmbeddedJournalLastSnapshotUploadSize:
+  'Describes the size of the snapshot uploaded to other masters in the cluster the previous time the download occurred. Only valid when using the embedded journal.'
+Master.EmbeddedJournalSnapshotDownloadDiskHistogram:
+  'Describes the size on disk of the snapshot downloaded from another master in the cluster. Only valid when using the embedded journal. Long running average.'
 Master.EmbeddedJournalSnapshotDownloadGenerate:
-  'Describes the amount of time taken to download journal snapshots from other masters in the cluster. Only valid when using the embedded journal. Use this metric to determine if there are potential communication bottlenecks between Alluxio masters.'
+  'Describes the amount of time taken to download journal snapshots from other masters in the cluster. Only valid when using the embedded journal. Long running average.'
+Master.EmbeddedJournalSnapshotDownloadHistogram:
+  'Describes the size of the snapshot downloaded from another master in the cluster. Only valid when using the embedded journal. Long running average.'
 Master.EmbeddedJournalSnapshotGenerateTimer:
   'Describes the amount of time taken to generate local journal snapshots on this master. Only valid when using the embedded journal. Use this metric to measure the performance of Alluxio''s snapshot generation.'
 Master.EmbeddedJournalSnapshotInstallTimer:
@@ -58,6 +82,12 @@ Master.EmbeddedJournalSnapshotLastIndex:
   'Represents the latest journal index that was recorded by this master in the most recent local snapshot or from a snapshot downloaded from another master in the cluster. Only valid when using the embedded journal.'
 Master.EmbeddedJournalSnapshotReplayTimer:
   'Describes the amount of time taken to replay a journal snapshot onto the master''s state machine. Only valid only when using the embedded journal. Use this metric to determine the performance of Alluxio when replaying journal snapshot file. Higher numbers may indicate a slow disk or CPU contention'
+Master.EmbeddedJournalSnapshotUploadDiskHistogram:
+  'Describes the size on disk of the snapshot uploaded to another master in the cluster. Only valid when using the embedded journal. Long running average.'
+Master.EmbeddedJournalSnapshotUploadHistogram:
+  'Describes the size of the snapshot uploaded to another master in the cluster. Only valid when using the embedded journal. Long running average.'
+Master.EmbeddedJournalSnapshotUploadTimer:
+  'Describes the amount of time taken to upload journal snapshots to another master in the cluster. Only valid when using the embedded journal. long running average'
 Master.FileBlockInfosGot:
   'Total number of succeed GetFileBlockInfo operations'
 Master.FileInfosGot:

docs/_data/table/master-configuration.csv

@@ -36,6 +36,8 @@ alluxio.master.embedded.journal.raft.client.request.timeout,"60sec"
 alluxio.master.embedded.journal.ratis.config,""
 alluxio.master.embedded.journal.retry.cache.expiry.time,"60s"
 alluxio.master.embedded.journal.snapshot.replication.chunk.size,"4MB"
+alluxio.master.embedded.journal.snapshot.replication.compression.level,"1"
+alluxio.master.embedded.journal.snapshot.replication.compression.type,"NO_COMPRESSION"
 alluxio.master.embedded.journal.transport.max.inbound.message.size,"100MB"
 alluxio.master.embedded.journal.transport.request.timeout.ms,"5sec"
 alluxio.master.embedded.journal.unsafe.flush.enabled,"false"
@@ -63,10 +65,9 @@ alluxio.master.journal.gc.period,"2min"
 alluxio.master.journal.gc.threshold,"5min"
 alluxio.master.journal.init.from.backup,""
 alluxio.master.journal.local.log.compaction,"true"
-alluxio.master.journal.log.concurrency.max,"256"
 alluxio.master.journal.log.size.bytes.max,"10MB"
 alluxio.master.journal.request.data.timeout,"20000"
-alluxio.master.journal.request.info.timeout,"20000"
+alluxio.master.journal.request.info.timeout,"10000"
 alluxio.master.journal.retry.interval,"1sec"
 alluxio.master.journal.space.monitor.interval,"10min"
 alluxio.master.journal.space.monitor.percent.free.threshold,"10"
@@ -122,7 +123,7 @@ alluxio.master.metastore.rocks.block.meta.block.index,""
 alluxio.master.metastore.rocks.block.meta.bloom.filter,"false"
 alluxio.master.metastore.rocks.block.meta.cache.size,""
 alluxio.master.metastore.rocks.block.meta.index,""
-alluxio.master.metastore.rocks.checkpoint.compression.level,"1"
+alluxio.master.metastore.rocks.checkpoint.compression.type,"LZ4_COMPRESSION"
 alluxio.master.metastore.rocks.edge.block.index,""
 alluxio.master.metastore.rocks.edge.bloom.filter,"false"
 alluxio.master.metastore.rocks.edge.cache.size,""

docs/_data/table/master-metrics.csv

@@ -24,11 +24,26 @@ Master.EdgeCacheLoadTimes,GAUGE
 Master.EdgeCacheMisses,GAUGE
 Master.EdgeCacheSize,GAUGE
 Master.EdgeLockPoolSize,GAUGE
+Master.EmbeddedJournalLastSnapshotDownloadDiskSize,GAUGE
+Master.EmbeddedJournalLastSnapshotDownloadDurationMs,GAUGE
+Master.EmbeddedJournalLastSnapshotDownloadSize,GAUGE
+Master.EmbeddedJournalLastSnapshotDurationMs,GAUGE
+Master.EmbeddedJournalLastSnapshotEntriesCount,GAUGE
+Master.EmbeddedJournalLastSnapshotReplayDurationMs,GAUGE
+Master.EmbeddedJournalLastSnapshotReplayEntriesCount,GAUGE
+Master.EmbeddedJournalLastSnapshotUploadDiskSize,GAUGE
+Master.EmbeddedJournalLastSnapshotUploadDurationMs,GAUGE
+Master.EmbeddedJournalLastSnapshotUploadSize,GAUGE
+Master.EmbeddedJournalSnapshotDownloadDiskHistogram,HISTOGRAM
 Master.EmbeddedJournalSnapshotDownloadGenerate,TIMER
+Master.EmbeddedJournalSnapshotDownloadHistogram,HISTOGRAM
 Master.EmbeddedJournalSnapshotGenerateTimer,TIMER
 Master.EmbeddedJournalSnapshotInstallTimer,TIMER
 Master.EmbeddedJournalSnapshotLastIndex,GAUGE
 Master.EmbeddedJournalSnapshotReplayTimer,TIMER
+Master.EmbeddedJournalSnapshotUploadDiskHistogram,HISTOGRAM
+Master.EmbeddedJournalSnapshotUploadHistogram,HISTOGRAM
+Master.EmbeddedJournalSnapshotUploadTimer,TIMER
 Master.FileBlockInfosGot,COUNTER
 Master.FileInfosGot,COUNTER
 Master.FileSize,GAUGE

docs/cn/core-services/Caching.md

@@ -334,7 +334,7 @@ Alluxio支持命名空间中每个文件和目录的"生存时间(TTL)"设置。
 则TTL功能可用于明确刷新旧数据，从而为新文件释放缓存空间。
 
 Alluxio具有与每个文件或目录关联的TTL属性。这些属性将保存为
-日志的一部分，所以集群重新后也能持久保持。活跃master节点负责
+日志的一部分，所以集群重启后也能持久保持。活跃master节点负责
 当Alluxio提供服务时将元数据保存在内存中。在内部，master运行一个后台
 线程，该线程定期检查文件是否已达到其TTL到期时间。
 
@@ -478,8 +478,12 @@ Alluxio cluster summary:
     Started: 09-28-2018 12:52:09:486
     Uptime: 0 day(s), 0 hour(s), 0 minute(s), and 26 second(s)
     Version: 2.0.0
-    Safe Mode: true
+    Safe Mode: false
     Zookeeper Enabled: false
+    Raft-based Journal: true
+    Raft Journal Addresses: 
+        localhost:19200
+        localhost:19201
     Live Workers: 1
     Lost Workers: 0
     Total Capacity: 10.67GB

docs/en/core-services/Caching.md

@@ -657,8 +657,12 @@ Alluxio cluster summary:
     Started: 09-28-2018 12:52:09:486
     Uptime: 0 day(s), 0 hour(s), 0 minute(s), and 26 second(s)
     Version: 2.0.0
-    Safe Mode: true
+    Safe Mode: false
     Zookeeper Enabled: false
+    Raft-based Journal: true
+    Raft Journal Addresses: 
+        localhost:19200
+        localhost:19201
     Live Workers: 1
     Lost Workers: 0
     Total Capacity: 10.67GB