diff --git a/core/commands/swarm.go b/core/commands/swarm.go
index 4d8fd3b6396..d6a3e8d696d 100644
--- a/core/commands/swarm.go
+++ b/core/commands/swarm.go
@@ -381,8 +381,7 @@ It is possible to use this command to inspect and tweak limits at runtime:
 	$ vi limit.json
 	$ ipfs swarm limit system limit.json
 
-Changes made via command line are discarded on node shutdown.
-For permanent limits set Swarm.ResourceMgr.Limits in the $IPFS_PATH/config file.
+Changes made via command line are persisted in the Swarm.ResourceMgr.Limits field of the $IPFS_PATH/config file.
 `},
 	Arguments: []cmds.Argument{
 		cmds.StringArg("scope", true, false, "scope of the limit"),
diff --git a/core/node/libp2p/rcmgr.go b/core/node/libp2p/rcmgr.go
index fc466cc3871..938f5eb43f9 100644
--- a/core/node/libp2p/rcmgr.go
+++ b/core/node/libp2p/rcmgr.go
@@ -49,7 +49,13 @@ func ResourceManager(cfg config.SwarmConfig) func(fx.Lifecycle, repo.Repo) (netw
 			}
 
 			defaultLimits := adjustedDefaultLimits(cfg)
-			limiter, err := rcmgr.NewLimiter(*cfg.ResourceMgr.Limits, defaultLimits)
+
+			var limits rcmgr.BasicLimiterConfig
+			if cfg.ResourceMgr.Limits != nil {
+				limits = *cfg.ResourceMgr.Limits
+			}
+
+			limiter, err := rcmgr.NewLimiter(limits, defaultLimits)
 			if err != nil {
 				return nil, opts, err
 			}
@@ -313,39 +319,48 @@ func NetSetLimit(mgr network.ResourceManager, repo repo.Repo, scope string, limi
 		return fmt.Errorf("reading config to set limit: %w", err)
 	}
 
-	setConfigLimit := func(f func(c *rcmgr.BasicLimiterConfig)) {
-		if cfg.Swarm.ResourceMgr.Limits == nil {
-			cfg.Swarm.ResourceMgr.Limits = &rcmgr.BasicLimiterConfig{}
-		}
-		f(cfg.Swarm.ResourceMgr.Limits)
+	if cfg.Swarm.ResourceMgr.Limits == nil {
+		cfg.Swarm.ResourceMgr.Limits = &rcmgr.BasicLimiterConfig{}
 	}
+	configLimits := cfg.Swarm.ResourceMgr.Limits
 
+	var setConfigFunc func()
 	switch {
 	case scope == config.ResourceMgrSystemScope:
 		err = mgr.ViewSystem(func(s network.ResourceScope) error {
 			return setLimit(s)
 		})
-		setConfigLimit(func(c *rcmgr.BasicLimiterConfig) { c.System = &limit })
+		setConfigFunc = func() { configLimits.System = &limit }
 
 	case scope == config.ResourceMgrTransientScope:
 		err = mgr.ViewTransient(func(s network.ResourceScope) error {
 			return setLimit(s)
 		})
-		setConfigLimit(func(c *rcmgr.BasicLimiterConfig) { c.Transient = &limit })
+		setConfigFunc = func() { configLimits.Transient = &limit }
 
 	case strings.HasPrefix(scope, config.ResourceMgrServiceScopePrefix):
 		svc := strings.TrimPrefix(scope, config.ResourceMgrServiceScopePrefix)
 		err = mgr.ViewService(svc, func(s network.ServiceScope) error {
 			return setLimit(s)
 		})
-		setConfigLimit(func(c *rcmgr.BasicLimiterConfig) { c.Service[svc] = limit })
+		setConfigFunc = func() {
+			if configLimits.Service == nil {
+				configLimits.Service = map[string]rcmgr.BasicLimitConfig{}
+			}
+			configLimits.Service[svc] = limit
+		}
 
 	case strings.HasPrefix(scope, config.ResourceMgrProtocolScopePrefix):
 		proto := strings.TrimPrefix(scope, config.ResourceMgrProtocolScopePrefix)
 		err = mgr.ViewProtocol(protocol.ID(proto), func(s network.ProtocolScope) error {
 			return setLimit(s)
 		})
-		setConfigLimit(func(c *rcmgr.BasicLimiterConfig) { c.Service[proto] = limit })
+		setConfigFunc = func() {
+			if configLimits.Protocol == nil {
+				configLimits.Protocol = map[string]rcmgr.BasicLimitConfig{}
+			}
+			configLimits.Protocol[proto] = limit
+		}
 
 	case strings.HasPrefix(scope, config.ResourceMgrPeerScopePrefix):
 		p := strings.TrimPrefix(scope, config.ResourceMgrPeerScopePrefix)
@@ -357,15 +372,25 @@ func NetSetLimit(mgr network.ResourceManager, repo repo.Repo, scope string, limi
 		err = mgr.ViewPeer(pid, func(s network.PeerScope) error {
 			return setLimit(s)
 		})
-		setConfigLimit(func(c *rcmgr.BasicLimiterConfig) { c.Service[p] = limit })
+		setConfigFunc = func() {
+			if configLimits.Peer == nil {
+				configLimits.Peer = map[string]rcmgr.BasicLimitConfig{}
+			}
+			configLimits.Peer[p] = limit
+		}
 
 	default:
 		return fmt.Errorf("invalid scope %q", scope)
 	}
 
 	if err != nil {
-		return err
+		return fmt.Errorf("setting new limits on resource manager: %w", err)
+	}
+
+	if cfg.Swarm.ResourceMgr.Limits == nil {
+		cfg.Swarm.ResourceMgr.Limits = &rcmgr.BasicLimiterConfig{}
 	}
+	setConfigFunc()
 
 	if err := repo.SetConfig(cfg); err != nil {
 		return fmt.Errorf("writing new limits to repo config: %w", err)
diff --git a/test/sharness/t0139-swarm-rcmgr.sh b/test/sharness/t0139-swarm-rcmgr.sh
index 5e922d0464a..793a6f4a86e 100755
--- a/test/sharness/t0139-swarm-rcmgr.sh
+++ b/test/sharness/t0139-swarm-rcmgr.sh
@@ -18,7 +18,7 @@ test_expect_success 'disconnected: swarm stats requires running daemon' '
 '
 
 # swarm limit|stats should fail in online mode by default
-# because Resource Manager is opt-in for now
+# because Resource Manager is opt-in
 test_launch_ipfs_daemon
 
 test_expect_success 'ResourceMgr disabled by default: swarm limit requires Swarm.ResourceMgr.Enabled' '
@@ -30,20 +30,31 @@ test_expect_success 'ResourceMgr disabled by default: swarm stats requires Swarm
   test_should_contain "missing ResourceMgr" actual
 '
 
-# swarm limit|stat should work when Swarm.ResourceMgr.Enabled
 test_kill_ipfs_daemon
 
-test_expect_success "test_config_set succeeds" "
+test_expect_success "setting an invalid limit should result in a failure" "
+  test_expect_code 1 ipfs config --json Swarm.ResourceMgr.Limits.System.Conns 'asdf' 2> actual &&
+  test_should_contain 'failed to unmarshal' actual
+"
+
+# swarm limit|stat should work when Swarm.ResourceMgr.Enabled
+test_expect_success "test enabling resource manager" "
   ipfs config --json Swarm.ResourceMgr.Enabled true &&
-  ipfs config --json Swarm.ResourceMgr.Limits.System.Conns 99999
+  ipfs config --json Swarm.ResourceMgr &&
+  jq -e '.Swarm.ResourceMgr.Enabled == true'
 "
 
 test_launch_ipfs_daemon
 
+test_expect_success "test setting system conns limit" "
+  ipfs config --json Swarm.ResourceMgr.Enabled true &&
+  ipfs config --json Swarm.ResourceMgr.Limits.System.Conns 99999
+"
+
 # every scope has the same fields, so we only inspect System
 test_expect_success 'ResourceMgr enabled: swarm limit' '
   ipfs swarm limit system --enc=json | tee json &&
-  jq -e ".Conns == 99999" < json &&
+  jq -e .Conns < json &&
   jq -e .ConnsInbound < json &&
   jq -e .ConnsOutbound < json &&
   jq -e .FD < json &&
@@ -65,6 +76,20 @@ test_expect_success 'ResourceMgr enabled: swarm stats' '
   jq -e .Transient.Memory < json
 '
 
+# shut down the daemon, set a limit in the config, and verify that it's applied
+test_kill_ipfs_daemon
+
+test_expect_success "set system conn limit" "
+  ipfs config --json Swarm.ResourceMgr.Limits.System.Conns 99999
+"
+
+test_launch_ipfs_daemon
+
+test_expect_success 'ResourceMgr enabled: swarm limit' '
+  ipfs swarm limit system --enc=json | tee json &&
+  jq -e ".Conns == 99999" < json
+'
+
 test_expect_success 'Set system memory limit while the daemon is running' '
   ipfs swarm limit system | jq ".Memory = 99998" > system.json &&
   ipfs swarm limit system system.json
@@ -78,5 +103,53 @@ test_expect_success 'The new system limits are in the swarm limit output' '
   ipfs swarm limit system --enc=json | jq -e ".Memory == 99998"
 '
 
+# now test all the other scopes
+test_expect_success 'Set limit on transient scope' '
+  ipfs swarm limit transient | jq ".Memory = 88888" > transient.json &&
+  ipfs swarm limit transient transient.json &&
+  jq -e ".Swarm.ResourceMgr.Limits.Transient.Memory == 88888" < "$IPFS_PATH/config" &&
+  ipfs swarm limit transient --enc=json | tee limits &&
+  jq -e ".Memory == 88888" < limits
+'
+
+test_expect_success 'Set limit on service scope' '
+  ipfs swarm limit svc:foo | jq ".Memory = 77777" > service-foo.json &&
+  ipfs swarm limit svc:foo service-foo.json --enc=json &&
+  jq -e ".Swarm.ResourceMgr.Limits.Service.foo.Memory == 77777" < "$IPFS_PATH/config" &&
+  ipfs swarm limit svc:foo --enc=json | tee limits &&
+  jq -e ".Memory == 77777" < limits
+'
+
+test_expect_success 'Set limit on protocol scope' '
+  ipfs swarm limit proto:foo | jq ".Memory = 66666" > proto-foo.json &&
+  ipfs swarm limit proto:foo proto-foo.json --enc=json &&
+  jq -e ".Swarm.ResourceMgr.Limits.Protocol.foo.Memory == 66666" < "$IPFS_PATH/config" &&
+  ipfs swarm limit proto:foo --enc=json | tee limits &&
+  jq -e ".Memory == 66666" < limits
+'
+
+# any valid peer id
+PEER_ID=QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN
+
+test_expect_success 'Set limit on peer scope' '
+  ipfs swarm limit peer:$PEER_ID | jq ".Memory = 66666" > peer-$PEER_ID.json &&
+  ipfs swarm limit peer:$PEER_ID peer-$PEER_ID.json --enc=json &&
+  jq -e ".Swarm.ResourceMgr.Limits.Peer.${PEER_ID}.Memory == 66666" < "$IPFS_PATH/config" &&
+  ipfs swarm limit peer:$PEER_ID --enc=json | tee limits &&
+  jq -e ".Memory == 66666" < limits
+'
+
+test_expect_success 'Get limit for peer scope with an invalid peer ID' '
+  test_expect_code 1 ipfs swarm limit peer:foo 2> actual &&
+  test_should_contain "invalid peer ID" actual
+'
+
+test_expect_success 'Set limit for peer scope with an invalid peer ID' '
+  echo "{\"Memory\": 99}" > invalid-peer-id.json &&
+  test_expect_code 1 ipfs swarm limit peer:foo invalid-peer-id.json 2> actual &&
+  test_should_contain "invalid peer ID" actual
+'
+
 test_kill_ipfs_daemon
+
 test_done