[DO NOT MERGE] swarm-network-rewrite merge to master

cmd/swarm/config.go

@@ -24,6 +24,7 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
+	"time"
 	"unicode"
 
 	cli "gopkg.in/urfave/cli.v1"
@@ -58,19 +59,25 @@ var (
 
 //constants for environment variables
 const (
-	SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR"
-	SWARM_ENV_ACCOUNT         = "SWARM_ACCOUNT"
-	SWARM_ENV_LISTEN_ADDR     = "SWARM_LISTEN_ADDR"
-	SWARM_ENV_PORT            = "SWARM_PORT"
-	SWARM_ENV_NETWORK_ID      = "SWARM_NETWORK_ID"
-	SWARM_ENV_SWAP_ENABLE     = "SWARM_SWAP_ENABLE"
-	SWARM_ENV_SWAP_API        = "SWARM_SWAP_API"
-	SWARM_ENV_SYNC_ENABLE     = "SWARM_SYNC_ENABLE"
-	SWARM_ENV_ENS_API         = "SWARM_ENS_API"
-	SWARM_ENV_ENS_ADDR        = "SWARM_ENS_ADDR"
-	SWARM_ENV_CORS            = "SWARM_CORS"
-	SWARM_ENV_BOOTNODES       = "SWARM_BOOTNODES"
-	GETH_ENV_DATADIR          = "GETH_DATADIR"
+	SWARM_ENV_CHEQUEBOOK_ADDR      = "SWARM_CHEQUEBOOK_ADDR"
+	SWARM_ENV_ACCOUNT              = "SWARM_ACCOUNT"
+	SWARM_ENV_LISTEN_ADDR          = "SWARM_LISTEN_ADDR"
+	SWARM_ENV_PORT                 = "SWARM_PORT"
+	SWARM_ENV_NETWORK_ID           = "SWARM_NETWORK_ID"
+	SWARM_ENV_SWAP_ENABLE          = "SWARM_SWAP_ENABLE"
+	SWARM_ENV_SWAP_API             = "SWARM_SWAP_API"
+	SWARM_ENV_SYNC_ENABLE          = "SWARM_SYNC_ENABLE"
+	SWARM_ENV_SYNC_UPDATE_DELAY    = "SWARM_ENV_SYNC_UPDATE_DELAY"
+	SWARM_ENV_ENS_API              = "SWARM_ENS_API"
+	SWARM_ENV_ENS_ADDR             = "SWARM_ENS_ADDR"
+	SWARM_ENV_CORS                 = "SWARM_CORS"
+	SWARM_ENV_BOOTNODES            = "SWARM_BOOTNODES"
+	SWARM_ENV_PSS_ENABLE           = "SWARM_PSS_ENABLE"
+	SWARM_ENV_STORE_PATH           = "SWARM_STORE_PATH"
+	SWARM_ENV_STORE_CAPACITY       = "SWARM_STORE_CAPACITY"
+	SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
+	SWARM_ENV_STORE_RADIUS         = "SWARM_STORE_RADIUS"
+	GETH_ENV_DATADIR               = "GETH_DATADIR"
 )
 
 // These settings ensure that TOML keys use the same names as Go struct fields.
@@ -95,7 +102,7 @@ func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) {
 	//check for deprecated flags
 	checkDeprecated(ctx)
 	//start by creating a default config
-	config = bzzapi.NewDefaultConfig()
+	config = bzzapi.NewConfig()
 	//first load settings from config file (if provided)
 	config, err = configFileOverride(config, ctx)
 	if err != nil {
@@ -195,6 +202,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.SyncEnabled = true
 	}
 
+	if d := ctx.GlobalDuration(SwarmSyncUpdateDelay.Name); d > 0 {
+		currentConfig.SyncUpdateDelay = d
+	}
+
 	currentConfig.SwapApi = ctx.GlobalString(SwarmSwapAPIFlag.Name)
 	if currentConfig.SwapEnabled && currentConfig.SwapApi == "" {
 		utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
@@ -221,6 +232,26 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.BootNodes = ctx.GlobalString(utils.BootnodesFlag.Name)
 	}
 
+	if ctx.GlobalIsSet(SwarmPssEnabledFlag.Name) {
+		currentConfig.PssEnabled = true
+	}
+
+	if storePath := ctx.GlobalString(SwarmStorePath.Name); storePath != "" {
+		currentConfig.StoreParams.ChunkDbPath = storePath
+	}
+
+	if storeCapacity := ctx.GlobalUint64(SwarmStoreCapacity.Name); storeCapacity != 0 {
+		currentConfig.StoreParams.DbCapacity = storeCapacity
+	}
+
+	if storeCacheCapacity := ctx.GlobalUint(SwarmStoreCacheCapacity.Name); storeCacheCapacity != 0 {
+		currentConfig.StoreParams.CacheCapacity = storeCacheCapacity
+	}
+
+	if storeRadius := ctx.GlobalInt(SwarmStoreRadius.Name); storeRadius != 0 {
+		currentConfig.StoreParams.Radius = storeRadius
+	}
+
 	return currentConfig
 
 }
@@ -268,6 +299,12 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 		}
 	}
 
+	if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" {
+		if d, err := time.ParseDuration(v); err != nil {
+			currentConfig.SyncUpdateDelay = d
+		}
+	}
+
 	if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {
 		currentConfig.SwapApi = swapapi
 	}
@@ -292,6 +329,12 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 		currentConfig.BootNodes = bootnodes
 	}
 
+	if pssenable := os.Getenv(SWARM_ENV_PSS_ENABLE); pssenable != "" {
+		if ps, err := strconv.ParseBool(pssenable); err != nil {
+			currentConfig.PssEnabled = ps
+		}
+	}
+
 	return currentConfig
 }
 

cmd/swarm/config_test.go

@@ -34,7 +34,7 @@ import (
 
 func TestDumpConfig(t *testing.T) {
 	swarm := runSwarm(t, "dumpconfig")
-	defaultConf := api.NewDefaultConfig()
+	defaultConf := api.NewConfig()
 	out, err := tomlSettings.Marshal(&defaultConf)
 	if err != nil {
 		t.Fatal(err)
@@ -43,7 +43,7 @@ func TestDumpConfig(t *testing.T) {
 	swarm.ExpectExit()
 }
 
-func TestFailsSwapEnabledNoSwapApi(t *testing.T) {
+func TestConfigFailsSwapEnabledNoSwapApi(t *testing.T) {
 	flags := []string{
 		fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42",
 		fmt.Sprintf("--%s", SwarmPortFlag.Name), "54545",
@@ -55,7 +55,7 @@ func TestFailsSwapEnabledNoSwapApi(t *testing.T) {
 	swarm.ExpectExit()
 }
 
-func TestFailsNoBzzAccount(t *testing.T) {
+func TestConfigFailsNoBzzAccount(t *testing.T) {
 	flags := []string{
 		fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42",
 		fmt.Sprintf("--%s", SwarmPortFlag.Name), "54545",
@@ -66,7 +66,7 @@ func TestFailsNoBzzAccount(t *testing.T) {
 	swarm.ExpectExit()
 }
 
-func TestCmdLineOverrides(t *testing.T) {
+func TestConfigCmdLineOverrides(t *testing.T) {
 	dir, err := ioutil.TempDir("", "bzztest")
 	if err != nil {
 		t.Fatal(err)
@@ -86,6 +86,7 @@ func TestCmdLineOverrides(t *testing.T) {
 		fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42",
 		fmt.Sprintf("--%s", SwarmPortFlag.Name), httpPort,
 		fmt.Sprintf("--%s", SwarmSyncEnabledFlag.Name),
+		fmt.Sprintf("--%s", SwarmPssEnabledFlag.Name),
 		fmt.Sprintf("--%s", CorsStringFlag.Name), "*",
 		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
 		fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
@@ -128,14 +129,18 @@ func TestCmdLineOverrides(t *testing.T) {
 		t.Fatal("Expected Sync to be enabled, but is false")
 	}
 
+	if !info.PssEnabled {
+		t.Fatal("Expected Pss to be enabled, but is false")
+	}
+
 	if info.Cors != "*" {
 		t.Fatalf("Expected Cors flag to be set to %s, got %s", "*", info.Cors)
 	}
 
 	node.Shutdown()
 }
 
-func TestFileOverrides(t *testing.T) {
+func TestConfigFileOverrides(t *testing.T) {
 
 	// assign ports
 	httpPort, err := assignTCPPort()
@@ -145,16 +150,17 @@ func TestFileOverrides(t *testing.T) {
 
 	//create a config file
 	//first, create a default conf
-	defaultConf := api.NewDefaultConfig()
+	defaultConf := api.NewConfig()
 	//change some values in order to test if they have been loaded
 	defaultConf.SyncEnabled = true
+	defaultConf.PssEnabled = true
 	defaultConf.NetworkId = 54
 	defaultConf.Port = httpPort
 	defaultConf.StoreParams.DbCapacity = 9000000
 	defaultConf.ChunkerParams.Branches = 64
-	defaultConf.HiveParams.CallInterval = 6000000000
+	defaultConf.HiveParams.KeepAliveInterval = 6000000000
 	defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second
-	defaultConf.SyncParams.KeyBufferSize = 512
+	//defaultConf.SyncParams.KeyBufferSize = 512
 	//create a TOML string
 	out, err := tomlSettings.Marshal(&defaultConf)
 	if err != nil {
@@ -223,6 +229,10 @@ func TestFileOverrides(t *testing.T) {
 		t.Fatal("Expected Sync to be enabled, but is false")
 	}
 
+	if !info.PssEnabled {
+		t.Fatal("Expected Pss to be enabled, but is false")
+	}
+
 	if info.StoreParams.DbCapacity != 9000000 {
 		t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId)
 	}
@@ -231,22 +241,22 @@ func TestFileOverrides(t *testing.T) {
 		t.Fatalf("Expected chunker params branches to be %d, got %d", 64, info.ChunkerParams.Branches)
 	}
 
-	if info.HiveParams.CallInterval != 6000000000 {
-		t.Fatalf("Expected HiveParams CallInterval to be %d, got %d", uint64(6000000000), uint64(info.HiveParams.CallInterval))
+	if info.HiveParams.KeepAliveInterval != 6000000000 {
+		t.Fatalf("Expected HiveParams KeepAliveInterval to be %d, got %d", uint64(6000000000), uint64(info.HiveParams.KeepAliveInterval))
 	}
 
 	if info.Swap.Params.Strategy.AutoCashInterval != 600*time.Second {
 		t.Fatalf("Expected SwapParams AutoCashInterval to be %ds, got %d", 600, info.Swap.Params.Strategy.AutoCashInterval)
 	}
 
-	if info.SyncParams.KeyBufferSize != 512 {
-		t.Fatalf("Expected info.SyncParams.KeyBufferSize to be %d, got %d", 512, info.SyncParams.KeyBufferSize)
-	}
+	//	if info.SyncParams.KeyBufferSize != 512 {
+	//		t.Fatalf("Expected info.SyncParams.KeyBufferSize to be %d, got %d", 512, info.SyncParams.KeyBufferSize)
+	//	}
 
 	node.Shutdown()
 }
 
-func TestEnvVars(t *testing.T) {
+func TestConfigEnvVars(t *testing.T) {
 	// assign ports
 	httpPort, err := assignTCPPort()
 	if err != nil {
@@ -258,6 +268,7 @@ func TestEnvVars(t *testing.T) {
 	envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmNetworkIdFlag.EnvVar, "999"))
 	envVars = append(envVars, fmt.Sprintf("%s=%s", CorsStringFlag.EnvVar, "*"))
 	envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmSyncEnabledFlag.EnvVar, "true"))
+	envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmPssEnabledFlag.EnvVar, "true"))
 
 	dir, err := ioutil.TempDir("", "bzztest")
 	if err != nil {
@@ -338,11 +349,15 @@ func TestEnvVars(t *testing.T) {
 		t.Fatal("Expected Sync to be enabled, but is false")
 	}
 
+	if !info.PssEnabled {
+		t.Fatal("Expected Pss to be enabled, but is false")
+	}
+
 	node.Shutdown()
 	cmd.Process.Kill()
 }
 
-func TestCmdLineOverridesFile(t *testing.T) {
+func TestConfigCmdLineOverridesFile(t *testing.T) {
 
 	// assign ports
 	httpPort, err := assignTCPPort()
@@ -352,16 +367,17 @@ func TestCmdLineOverridesFile(t *testing.T) {
 
 	//create a config file
 	//first, create a default conf
-	defaultConf := api.NewDefaultConfig()
+	defaultConf := api.NewConfig()
 	//change some values in order to test if they have been loaded
 	defaultConf.SyncEnabled = false
+	defaultConf.PssEnabled = false
 	defaultConf.NetworkId = 54
 	defaultConf.Port = "8588"
 	defaultConf.StoreParams.DbCapacity = 9000000
 	defaultConf.ChunkerParams.Branches = 64
-	defaultConf.HiveParams.CallInterval = 6000000000
+	defaultConf.HiveParams.KeepAliveInterval = 6000000000
 	defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second
-	defaultConf.SyncParams.KeyBufferSize = 512
+	//defaultConf.SyncParams.KeyBufferSize = 512
 	//create a TOML file
 	out, err := tomlSettings.Marshal(&defaultConf)
 	if err != nil {
@@ -393,6 +409,7 @@ func TestCmdLineOverridesFile(t *testing.T) {
 		fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "77",
 		fmt.Sprintf("--%s", SwarmPortFlag.Name), httpPort,
 		fmt.Sprintf("--%s", SwarmSyncEnabledFlag.Name),
+		fmt.Sprintf("--%s", SwarmPssEnabledFlag.Name),
 		fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
 		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
 		"--ens-api", "",
@@ -443,18 +460,22 @@ func TestCmdLineOverridesFile(t *testing.T) {
 		t.Fatalf("Expected chunker params branches to be %d, got %d", 64, info.ChunkerParams.Branches)
 	}
 
-	if info.HiveParams.CallInterval != 6000000000 {
-		t.Fatalf("Expected HiveParams CallInterval to be %d, got %d", uint64(6000000000), uint64(info.HiveParams.CallInterval))
+	if info.HiveParams.KeepAliveInterval != 6000000000 {
+		t.Fatalf("Expected HiveParams KeepAliveInterval to be %d, got %d", uint64(6000000000), uint64(info.HiveParams.KeepAliveInterval))
 	}
 
 	if info.Swap.Params.Strategy.AutoCashInterval != 600*time.Second {
 		t.Fatalf("Expected SwapParams AutoCashInterval to be %ds, got %d", 600, info.Swap.Params.Strategy.AutoCashInterval)
 	}
 
-	if info.SyncParams.KeyBufferSize != 512 {
-		t.Fatalf("Expected info.SyncParams.KeyBufferSize to be %d, got %d", 512, info.SyncParams.KeyBufferSize)
+	if !info.PssEnabled {
+		t.Fatal("Expected Pss to be enabled, but is false")
 	}
 
+	//	if info.SyncParams.KeyBufferSize != 512 {
+	//		t.Fatalf("Expected info.SyncParams.KeyBufferSize to be %d, got %d", 512, info.SyncParams.KeyBufferSize)
+	//	}
+
 	node.Shutdown()
 }
 

cmd/swarm/db.go

@@ -23,18 +23,19 @@ import (
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 	"gopkg.in/urfave/cli.v1"
 )
 
 func dbExport(ctx *cli.Context) {
 	args := ctx.Args()
-	if len(args) != 2 {
-		utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database) and <file> (path to write the tar archive to, - for stdout)")
+	if len(args) != 3 {
+		utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database), <file> (path to write the tar archive to, - for stdout) and the base key")
 	}
 
-	store, err := openDbStore(args[0])
+	store, err := openLDBStore(args[0], common.Hex2Bytes(args[2]))
 	if err != nil {
 		utils.Fatalf("error opening local chunk database: %s", err)
 	}
@@ -62,11 +63,11 @@ func dbExport(ctx *cli.Context) {
 
 func dbImport(ctx *cli.Context) {
 	args := ctx.Args()
-	if len(args) != 2 {
-		utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database) and <file> (path to read the tar archive from, - for stdin)")
+	if len(args) != 3 {
+		utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database), <file> (path to read the tar archive from, - for stdin) and the base key")
 	}
 
-	store, err := openDbStore(args[0])
+	store, err := openLDBStore(args[0], common.Hex2Bytes(args[2]))
 	if err != nil {
 		utils.Fatalf("error opening local chunk database: %s", err)
 	}
@@ -94,11 +95,11 @@ func dbImport(ctx *cli.Context) {
 
 func dbClean(ctx *cli.Context) {
 	args := ctx.Args()
-	if len(args) != 1 {
-		utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database)")
+	if len(args) != 2 {
+		utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database) and the base key")
 	}
 
-	store, err := openDbStore(args[0])
+	store, err := openLDBStore(args[0], common.Hex2Bytes(args[1]))
 	if err != nil {
 		utils.Fatalf("error opening local chunk database: %s", err)
 	}
@@ -107,10 +108,10 @@ func dbClean(ctx *cli.Context) {
 	store.Cleanup()
 }
 
-func openDbStore(path string) (*storage.DbStore, error) {
+func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) {
 	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
 		return nil, fmt.Errorf("invalid chunkdb path: %s", err)
 	}
 	hash := storage.MakeHashFunc("SHA3")
-	return storage.NewDbStore(path, hash, 10000000, 0)
+	return storage.NewLDBStore(path, hash, 10000000, func(k storage.Key) (ret uint8) { return uint8(storage.Proximity(basekey[:], k[:])) })
 }