Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update file descriptor limits for macOS #892

Merged
merged 7 commits into from
Dec 3, 2019
Merged
5 changes: 3 additions & 2 deletions cmd/utils/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -882,10 +882,11 @@ func makeDatabaseHandles() int {
if err != nil {
Fatalf("Failed to retrieve file descriptor allowance: %v", err)
}
if err := fdlimit.Raise(uint64(limit)); err != nil {
raised, err := fdlimit.Raise(uint64(limit))
if err != nil {
Fatalf("Failed to raise file descriptor allowance: %v", err)
}
return limit / 2 // Leave half for networking and other stuff
return int(raised / 2) // Leave half for networking and other stuff
}

// MakeAddress converts an account specified directly as a hex encoded string or
Expand Down
71 changes: 71 additions & 0 deletions common/fdlimit/fdlimit_darwin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package fdlimit

import "syscall"

// hardlimit is the number of file descriptors allowed at max by the kernel.
const hardlimit = 10240

// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
// Returns the size it was set to (may differ from the desired 'max')
func Raise(max uint64) (uint64, error) {
// Get the current limit
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return 0, err
}
// Try to update the limit to the max allowance
limit.Cur = limit.Max
if limit.Cur > max {
limit.Cur = max
}
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return 0, err
}
// MacOS can silently apply further caps, so retrieve the actually set limit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return 0, err
}
return limit.Cur, nil
}

// Current retrieves the number of file descriptors allowed to be opened by this
// process.
func Current() (int, error) {
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return 0, err
}
return int(limit.Cur), nil
}

// Maximum retrieves the maximum number of file descriptors this process is
// allowed to request for itself.
func Maximum() (int, error) {
// Retrieve the maximum allowed by dynamic OS limits
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return 0, err
}
// Cap it to OPEN_MAX (10240) because macos is a special snowflake
if limit.Max > hardlimit {
limit.Max = hardlimit
}
return int(limit.Max), nil
}
11 changes: 7 additions & 4 deletions common/fdlimit/fdlimit_freebsd.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,24 @@ import "syscall"

// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
func Raise(max uint64) error {
func Raise(max uint64) (uint64, error) {
// Get the current limit
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
// Try to update the limit to the max allowance
limit.Cur = limit.Max
if limit.Cur > int64(max) {
limit.Cur = int64(max)
}
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return 0, err
}
return nil
return uint64(limit.Cur), nil
}

// Current retrieves the number of file descriptors allowed to be opened by this
Expand Down
2 changes: 1 addition & 1 deletion common/fdlimit/fdlimit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func TestFileDescriptorLimits(t *testing.T) {
if limit, err := Current(); err != nil || limit <= 0 {
t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
}
if err := Raise(uint64(target)); err != nil {
if _, err := Raise(uint64(target)); err != nil {
t.Fatalf("failed to raise file allowance")
}
if limit, err := Current(); err != nil || limit < target {
Expand Down
15 changes: 10 additions & 5 deletions common/fdlimit/fdlimit_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,29 +14,34 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

// +build linux darwin netbsd openbsd solaris
// +build linux netbsd openbsd solaris

package fdlimit

import "syscall"

// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
func Raise(max uint64) error {
// Returns the size it was set to (may differ from the desired 'max')
func Raise(max uint64) (uint64, error) {
// Get the current limit
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
// Try to update the limit to the max allowance
limit.Cur = limit.Max
if limit.Cur > max {
limit.Cur = max
}
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
// MacOS can silently apply further caps, so retrieve the actually set limit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return 0, err
}
return nil
return limit.Cur, nil
}

// Current retrieves the number of file descriptors allowed to be opened by this
Expand Down
15 changes: 9 additions & 6 deletions common/fdlimit/fdlimit_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,28 +16,31 @@

package fdlimit

import "errors"
import "fmt"

// hardlimit is the number of file descriptors allowed at max by the kernel.
const hardlimit = 16384

// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
func Raise(max uint64) error {
func Raise(max uint64) (uint64, error) {
// This method is NOP by design:
// * Linux/Darwin counterparts need to manually increase per process limits
// * On Windows Go uses the CreateFile API, which is limited to 16K files, non
// changeable from within a running process
// This way we can always "request" raising the limits, which will either have
// or not have effect based on the platform we're running on.
if max > 16384 {
return errors.New("file descriptor limit (16384) reached")
if max > hardlimit {
return hardlimit, fmt.Errorf("file descriptor limit (%d) reached", hardlimit)
}
return nil
return max, nil
}

// Current retrieves the number of file descriptors allowed to be opened by this
// process.
func Current() (int, error) {
// Please see Raise for the reason why we use hard coded 16K as the limit
return 16384, nil
return hardlimit, nil
}

// Maximum retrieves the maximum number of file descriptors this process is
Expand Down
2 changes: 1 addition & 1 deletion core/tx_pool_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ func validateEvents(events chan NewTxsEvent, count int) error {
case ev := <-events:
received = append(received, ev.Txs...)
case <-time.After(time.Second):
return fmt.Errorf("event #%d not fired", received)
return fmt.Errorf("event #%d not fired", len(received))
}
}
if len(received) > count {
Expand Down
2 changes: 1 addition & 1 deletion crypto/bn256/cloudflare/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ func TestRandomG2Marshal(t *testing.T) {
t.Error(err)
continue
}
t.Logf("%d: %x\n", n, g2.Marshal())
t.Logf("%v: %x\n", n, g2.Marshal())
}
}

Expand Down
2 changes: 1 addition & 1 deletion crypto/bn256/google/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ func TestRandomG2Marshal(t *testing.T) {
t.Error(err)
continue
}
t.Logf("%d: %x\n", n, g2.Marshal())
t.Logf("%v: %x\n", n, g2.Marshal())
}
}

Expand Down