config: Add GoMemLimit config option and use with 10-node test

cmd/algod/main.go

@@ -22,6 +22,7 @@
 	"math/rand"
 	"os"
 	"path/filepath"
+	"runtime/debug"
 	"strconv"
 	"strings"
 	"time"
@@ -173,6 +174,11 @@
 		log.Fatalf("Cannot load config: %v", err)
 	}
 
+	// set soft memory limit, if configured
+	if cfg.GoMemLimit > 0 {
+		debug.SetMemoryLimit(int64(cfg.GoMemLimit))
+	}
+
 	_, err = cfg.ValidateDNSBootstrapArray(genesis.Network)
 	if err != nil {
 		// log is not setup yet, this will log to stderr

config/localTemplate.go

@@ -42,7 +42,7 @@ type Local struct {
 	// Version tracks the current version of the defaults so we can migrate old -> new
 	// This is specifically important whenever we decide to change the default value
 	// for an existing parameter. This field tag must be updated any time we add a new version.
-	Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28" version[29]:"29" version[30]:"30" version[31]:"31" version[32]:"32" version[33]:"33"`
+	Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28" version[29]:"29" version[30]:"30" version[31]:"31" version[32]:"32" version[33]:"33" version[34]:"34"`
 
 	// Archival nodes retain a full copy of the block history. Non-Archival nodes will delete old blocks and only retain what's need to properly validate blockchain messages (the precise number of recent blocks depends on the consensus parameters. Currently the last 1321 blocks are required). This means that non-Archival nodes require significantly less storage than Archival nodes.  If setting this to true for the first time, the existing ledger may need to be deleted to get the historical values stored as the setting only affects current blocks forward. To do this, shutdown the node and delete all .sqlite files within the data/testnet-version directory, except the crash.sqlite file. Restart the node and wait for the node to sync.
 	Archival bool `version[0]:"false"`
@@ -609,6 +609,10 @@ type Local struct {
 
 	// DisableAPIAuth turns off authentication for public (non-admin) API endpoints.
 	DisableAPIAuth bool `version[30]:"false"`
+
+	// GoMemLimit provides the Go runtime with a soft memory limit. The default behavior is no limit,
+	// unless the GOMEMLIMIT environment variable is set.
+	GoMemLimit uint64 `version[34]:"0"`
 }
 
 // DNSBootstrapArray returns an array of one or more DNS Bootstrap identifiers

config/local_defaults.go

@@ -20,7 +20,7 @@
 package config
 
 var defaultLocal = Local{
-	Version:                                    33,
+	Version:                                    34,
 	AccountUpdatesStatsInterval:                5000000000,
 	AccountsRebuildSynchronousMode:             1,
 	AgreementIncomingBundlesQueueLength:        15,
@@ -89,6 +89,7 @@ var defaultLocal = Local{
 	FallbackDNSResolverAddress:                 "",
 	ForceFetchTransactions:                     false,
 	ForceRelayMessages:                         false,
+	GoMemLimit:                                 0,
 	GossipFanout:                               4,
 	HeartbeatUpdateInterval:                    600,
 	HotDataDir:                                 "",

installer/config.json.example

@@ -1,5 +1,5 @@
 {
-    "Version": 33,
+    "Version": 34,
     "AccountUpdatesStatsInterval": 5000000000,
     "AccountsRebuildSynchronousMode": 1,
     "AgreementIncomingBundlesQueueLength": 15,
@@ -68,6 +68,7 @@
     "FallbackDNSResolverAddress": "",
     "ForceFetchTransactions": false,
     "ForceRelayMessages": false,
+    "GoMemLimit": 0,
     "GossipFanout": 4,
     "HeartbeatUpdateInterval": 600,
     "HotDataDir": "",

test/e2e-go/features/catchup/stateproofsCatchup_test.go

@@ -221,6 +221,12 @@ func TestSendSigsAfterCatchpointCatchup(t *testing.T) {
 	var fixture fixtures.RestClientFixture
 	fixture.SetConsensus(configurableConsensus)
 	fixture.SetupNoStart(t, filepath.Join("nettemplates", "ThreeNodesWithRichAcct.json"))
+	for _, nodeDir := range fixture.NodeDataDirs() {
+		cfg, err := config.LoadConfigFromDisk(nodeDir)
+		a.NoError(err)
+		cfg.GoMemLimit = 4 * 1024 * 1024 * 1024 // 4GB
+		cfg.SaveToDisk(nodeDir)
+	}
 
 	primaryNode, primaryNodeRestClient, primaryEC := startCatchpointGeneratingNode(a, &fixture, "Primary")
 	defer primaryEC.Print()

test/e2e-go/features/partitionRecovery/partitionRecovery_test.go

@@ -251,6 +251,7 @@ func TestPartitionHalfOffline(t *testing.T) {
 		a.NoError(err)
 		// adjust the refresh interval for one hour, so that we won't be reloading the participation key during this test.
 		cfg.ParticipationKeysRefreshInterval = time.Hour
+		cfg.GoMemLimit = 1 * 1024 * 1024 * 1024 // 1GB
 		cfg.SaveToDisk(nodeDir)
 	}
 	fixture.Start()

test/testdata/configs/config-v34.json

@@ -0,0 +1,139 @@
+{
+    "Version": 34,
+    "AccountUpdatesStatsInterval": 5000000000,
+    "AccountsRebuildSynchronousMode": 1,
+    "AgreementIncomingBundlesQueueLength": 15,
+    "AgreementIncomingProposalsQueueLength": 50,
+    "AgreementIncomingVotesQueueLength": 20000,
+    "AnnounceParticipationKey": true,
+    "Archival": false,
+    "BaseLoggerDebugLevel": 4,
+    "BlockDBDir": "",
+    "BlockServiceCustomFallbackEndpoints": "",
+    "BlockServiceMemCap": 500000000,
+    "BroadcastConnectionsLimit": -1,
+    "CadaverDirectory": "",
+    "CadaverSizeTarget": 0,
+    "CatchpointDir": "",
+    "CatchpointFileHistoryLength": 365,
+    "CatchpointInterval": 10000,
+    "CatchpointTracking": 0,
+    "CatchupBlockDownloadRetryAttempts": 1000,
+    "CatchupBlockValidateMode": 0,
+    "CatchupFailurePeerRefreshRate": 10,
+    "CatchupGossipBlockFetchTimeoutSec": 4,
+    "CatchupHTTPBlockFetchTimeoutSec": 4,
+    "CatchupLedgerDownloadRetryAttempts": 50,
+    "CatchupParallelBlocks": 16,
+    "ColdDataDir": "",
+    "ConnectionsRateLimitingCount": 60,
+    "ConnectionsRateLimitingWindowSeconds": 1,
+    "CrashDBDir": "",
+    "DNSBootstrapID": "<network>.algorand.network?backup=<network>.algorand.net&dedup=<name>.algorand-<network>.(network|net)",
+    "DNSSecurityFlags": 1,
+    "DeadlockDetection": 0,
+    "DeadlockDetectionThreshold": 30,
+    "DisableAPIAuth": false,
+    "DisableLedgerLRUCache": false,
+    "DisableLocalhostConnectionRateLimit": true,
+    "DisableNetworking": false,
+    "DisableOutgoingConnectionThrottling": false,
+    "EnableAccountUpdatesStats": false,
+    "EnableAgreementReporting": false,
+    "EnableAgreementTimeMetrics": false,
+    "EnableAssembleStats": false,
+    "EnableBlockService": false,
+    "EnableDeveloperAPI": false,
+    "EnableExperimentalAPI": false,
+    "EnableFollowMode": false,
+    "EnableGossipBlockService": true,
+    "EnableGossipService": true,
+    "EnableIncomingMessageFilter": false,
+    "EnableLedgerService": false,
+    "EnableMetricReporting": false,
+    "EnableOutgoingNetworkMessageFiltering": true,
+    "EnableP2P": false,
+    "EnablePingHandler": true,
+    "EnableProcessBlockStats": false,
+    "EnableProfiler": false,
+    "EnableRequestLogger": false,
+    "EnableRuntimeMetrics": false,
+    "EnableTopAccountsReporting": false,
+    "EnableTxBacklogAppRateLimiting": true,
+    "EnableTxBacklogRateLimiting": true,
+    "EnableTxnEvalTracer": false,
+    "EnableUsageLog": false,
+    "EnableVerbosedTransactionSyncLogging": false,
+    "EndpointAddress": "127.0.0.1:0",
+    "FallbackDNSResolverAddress": "",
+    "ForceFetchTransactions": false,
+    "ForceRelayMessages": false,
+    "GoMemLimit": 0,
+    "GossipFanout": 4,
+    "HeartbeatUpdateInterval": 600,
+    "HotDataDir": "",
+    "IncomingConnectionsLimit": 2400,
+    "IncomingMessageFilterBucketCount": 5,
+    "IncomingMessageFilterBucketSize": 512,
+    "LedgerSynchronousMode": 2,
+    "LogArchiveDir": "",
+    "LogArchiveMaxAge": "",
+    "LogArchiveName": "node.archive.log",
+    "LogFileDir": "",
+    "LogSizeLimit": 1073741824,
+    "MaxAPIBoxPerApplication": 100000,
+    "MaxAPIResourcesPerAccount": 100000,
+    "MaxAcctLookback": 4,
+    "MaxBlockHistoryLookback": 0,
+    "MaxCatchpointDownloadDuration": 43200000000000,
+    "MaxConnectionsPerIP": 15,
+    "MinCatchpointFileDownloadBytesPerSecond": 20480,
+    "NetAddress": "",
+    "NetworkMessageTraceServer": "",
+    "NetworkProtocolVersion": "",
+    "NodeExporterListenAddress": ":9100",
+    "NodeExporterPath": "./node_exporter",
+    "OptimizeAccountsDatabaseOnStartup": false,
+    "OutgoingMessageFilterBucketCount": 3,
+    "OutgoingMessageFilterBucketSize": 128,
+    "P2PPersistPeerID": false,
+    "P2PPrivateKeyLocation": "",
+    "ParticipationKeysRefreshInterval": 60000000000,
+    "PeerConnectionsUpdateInterval": 3600,
+    "PeerPingPeriodSeconds": 0,
+    "PriorityPeers": {},
+    "ProposalAssemblyTime": 500000000,
+    "PublicAddress": "",
+    "ReconnectTime": 60000000000,
+    "ReservedFDs": 256,
+    "RestConnectionsHardLimit": 2048,
+    "RestConnectionsSoftLimit": 1024,
+    "RestReadTimeoutSeconds": 15,
+    "RestWriteTimeoutSeconds": 120,
+    "RunHosted": false,
+    "StateproofDir": "",
+    "StorageEngine": "sqlite",
+    "SuggestedFeeBlockHistory": 3,
+    "SuggestedFeeSlidingWindowSize": 50,
+    "TLSCertFile": "",
+    "TLSKeyFile": "",
+    "TelemetryToLog": true,
+    "TrackerDBDir": "",
+    "TransactionSyncDataExchangeRate": 0,
+    "TransactionSyncSignificantMessageThreshold": 0,
+    "TxBacklogAppTxPerSecondRate": 100,
+    "TxBacklogAppTxRateLimiterMaxSize": 1048576,
+    "TxBacklogRateLimitingCongestionPct": 50,
+    "TxBacklogReservedCapacityPerPeer": 20,
+    "TxBacklogServiceRateWindowSeconds": 10,
+    "TxBacklogSize": 26000,
+    "TxIncomingFilterMaxSize": 500000,
+    "TxIncomingFilteringFlags": 1,
+    "TxPoolExponentialIncreaseFactor": 2,
+    "TxPoolSize": 75000,
+    "TxSyncIntervalSeconds": 60,
+    "TxSyncServeResponseSize": 1000000,
+    "TxSyncTimeoutSeconds": 30,
+    "UseXForwardedForAddressField": "",
+    "VerifiedTranscationsCacheSize": 150000
+}