From 6d25447d6492cc36763c49122f92f6d628184e7a Mon Sep 17 00:00:00 2001
From: Peter Fox <31221824+prd-fox@users.noreply.github.com>
Date: Tue, 3 Dec 2019 15:13:33 +0000
Subject: [PATCH] Update file descriptor limits for macOS (#892)
* Introduces proper file handle limits for darwin based OSes (macOS/OSX)
for working with Go 1.12
---
cmd/utils/flags.go | 5 +-
common/fdlimit/fdlimit_darwin.go | 71 ++++++++++++++++++++++++++++
common/fdlimit/fdlimit_freebsd.go | 11 +++--
common/fdlimit/fdlimit_test.go | 2 +-
common/fdlimit/fdlimit_unix.go | 15 ++++--
common/fdlimit/fdlimit_windows.go | 15 +++---
core/tx_pool_test.go | 2 +-
crypto/bn256/cloudflare/main_test.go | 2 +-
crypto/bn256/google/main_test.go | 2 +-
9 files changed, 104 insertions(+), 21 deletions(-)
create mode 100644 common/fdlimit/fdlimit_darwin.go
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
index 88c7a07881..b0d69461ee 100644
--- a/cmd/utils/flags.go
+++ b/cmd/utils/flags.go
@@ -882,10 +882,11 @@ func makeDatabaseHandles() int {
if err != nil {
Fatalf("Failed to retrieve file descriptor allowance: %v", err)
}
- if err := fdlimit.Raise(uint64(limit)); err != nil {
+ raised, err := fdlimit.Raise(uint64(limit))
+ if err != nil {
Fatalf("Failed to raise file descriptor allowance: %v", err)
}
- return limit / 2 // Leave half for networking and other stuff
+ return int(raised / 2) // Leave half for networking and other stuff
}
// MakeAddress converts an account specified directly as a hex encoded string or
diff --git a/common/fdlimit/fdlimit_darwin.go b/common/fdlimit/fdlimit_darwin.go
new file mode 100644
index 0000000000..88dd0f56cb
--- /dev/null
+++ b/common/fdlimit/fdlimit_darwin.go
@@ -0,0 +1,71 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package fdlimit
+
+import "syscall"
+
+// hardlimit is the number of file descriptors allowed at max by the kernel.
+const hardlimit = 10240
+
+// Raise tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+// Returns the size it was set to (may differ from the desired 'max')
+func Raise(max uint64) (uint64, error) {
+ // Get the current limit
+ var limit syscall.Rlimit
+ if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+ return 0, err
+ }
+ // Try to update the limit to the max allowance
+ limit.Cur = limit.Max
+ if limit.Cur > max {
+ limit.Cur = max
+ }
+ if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+ return 0, err
+ }
+ // MacOS can silently apply further caps, so retrieve the actually set limit
+ if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+ return 0, err
+ }
+ return limit.Cur, nil
+}
+
+// Current retrieves the number of file descriptors allowed to be opened by this
+// process.
+func Current() (int, error) {
+ var limit syscall.Rlimit
+ if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+ return 0, err
+ }
+ return int(limit.Cur), nil
+}
+
+// Maximum retrieves the maximum number of file descriptors this process is
+// allowed to request for itself.
+func Maximum() (int, error) {
+ // Retrieve the maximum allowed by dynamic OS limits
+ var limit syscall.Rlimit
+ if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+ return 0, err
+ }
+ // Cap it to OPEN_MAX (10240) because macos is a special snowflake
+ if limit.Max > hardlimit {
+ limit.Max = hardlimit
+ }
+ return int(limit.Max), nil
+}
diff --git a/common/fdlimit/fdlimit_freebsd.go b/common/fdlimit/fdlimit_freebsd.go
index c126b0c265..0d8727138e 100644
--- a/common/fdlimit/fdlimit_freebsd.go
+++ b/common/fdlimit/fdlimit_freebsd.go
@@ -26,11 +26,11 @@ import "syscall"
// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
// Get the current limit
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
- return err
+ return 0, err
}
// Try to update the limit to the max allowance
limit.Cur = limit.Max
@@ -38,9 +38,12 @@ func Raise(max uint64) error {
limit.Cur = int64(max)
}
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
- return err
+ return 0, err
+ }
+ if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+ return 0, err
}
- return nil
+ return uint64(limit.Cur), nil
}
// Current retrieves the number of file descriptors allowed to be opened by this
diff --git a/common/fdlimit/fdlimit_test.go b/common/fdlimit/fdlimit_test.go
index a9ee9ab36a..21362b8463 100644
--- a/common/fdlimit/fdlimit_test.go
+++ b/common/fdlimit/fdlimit_test.go
@@ -36,7 +36,7 @@ func TestFileDescriptorLimits(t *testing.T) {
if limit, err := Current(); err != nil || limit <= 0 {
t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
}
- if err := Raise(uint64(target)); err != nil {
+ if _, err := Raise(uint64(target)); err != nil {
t.Fatalf("failed to raise file allowance")
}
if limit, err := Current(); err != nil || limit < target {
diff --git a/common/fdlimit/fdlimit_unix.go b/common/fdlimit/fdlimit_unix.go
index a258132353..e5a575f7a7 100644
--- a/common/fdlimit/fdlimit_unix.go
+++ b/common/fdlimit/fdlimit_unix.go
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see .
-// +build linux darwin netbsd openbsd solaris
+// +build linux netbsd openbsd solaris
package fdlimit
@@ -22,11 +22,12 @@ import "syscall"
// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+// Returns the size it was set to (may differ from the desired 'max')
+func Raise(max uint64) (uint64, error) {
// Get the current limit
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
- return err
+ return 0, err
}
// Try to update the limit to the max allowance
limit.Cur = limit.Max
@@ -34,9 +35,13 @@ func Raise(max uint64) error {
limit.Cur = max
}
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
- return err
+ return 0, err
+ }
+ // MacOS can silently apply further caps, so retrieve the actually set limit
+ if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+ return 0, err
}
- return nil
+ return limit.Cur, nil
}
// Current retrieves the number of file descriptors allowed to be opened by this
diff --git a/common/fdlimit/fdlimit_windows.go b/common/fdlimit/fdlimit_windows.go
index 863c58bedf..f472153662 100644
--- a/common/fdlimit/fdlimit_windows.go
+++ b/common/fdlimit/fdlimit_windows.go
@@ -16,28 +16,31 @@
package fdlimit
-import "errors"
+import "fmt"
+
+// hardlimit is the number of file descriptors allowed at max by the kernel.
+const hardlimit = 16384
// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
// This method is NOP by design:
// * Linux/Darwin counterparts need to manually increase per process limits
// * On Windows Go uses the CreateFile API, which is limited to 16K files, non
// changeable from within a running process
// This way we can always "request" raising the limits, which will either have
// or not have effect based on the platform we're running on.
- if max > 16384 {
- return errors.New("file descriptor limit (16384) reached")
+ if max > hardlimit {
+ return hardlimit, fmt.Errorf("file descriptor limit (%d) reached", hardlimit)
}
- return nil
+ return max, nil
}
// Current retrieves the number of file descriptors allowed to be opened by this
// process.
func Current() (int, error) {
// Please see Raise for the reason why we use hard coded 16K as the limit
- return 16384, nil
+ return hardlimit, nil
}
// Maximum retrieves the maximum number of file descriptors this process is
diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go
index 0048ff4ae4..8ae210ae59 100644
--- a/core/tx_pool_test.go
+++ b/core/tx_pool_test.go
@@ -139,7 +139,7 @@ func validateEvents(events chan NewTxsEvent, count int) error {
case ev := <-events:
received = append(received, ev.Txs...)
case <-time.After(time.Second):
- return fmt.Errorf("event #%d not fired", received)
+ return fmt.Errorf("event #%d not fired", len(received))
}
}
if len(received) > count {
diff --git a/crypto/bn256/cloudflare/main_test.go b/crypto/bn256/cloudflare/main_test.go
index 0230f1b199..c0c85457be 100644
--- a/crypto/bn256/cloudflare/main_test.go
+++ b/crypto/bn256/cloudflare/main_test.go
@@ -13,7 +13,7 @@ func TestRandomG2Marshal(t *testing.T) {
t.Error(err)
continue
}
- t.Logf("%d: %x\n", n, g2.Marshal())
+ t.Logf("%v: %x\n", n, g2.Marshal())
}
}
diff --git a/crypto/bn256/google/main_test.go b/crypto/bn256/google/main_test.go
index 0230f1b199..c0c85457be 100644
--- a/crypto/bn256/google/main_test.go
+++ b/crypto/bn256/google/main_test.go
@@ -13,7 +13,7 @@ func TestRandomG2Marshal(t *testing.T) {
t.Error(err)
continue
}
- t.Logf("%d: %x\n", n, g2.Marshal())
+ t.Logf("%v: %x\n", n, g2.Marshal())
}
}