diff --git a/go.mod b/go.mod
index ba0874f20e93..854106564e88 100644
--- a/go.mod
+++ b/go.mod
@@ -26,7 +26,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.17.3
 	github.com/onsi/gomega v1.33.1
 	github.com/osrg/gobgp/v3 v3.26.0
-	github.com/ovn-org/libovsdb v0.0.0-20230711201130-6785b52d4020
+	github.com/ovn-org/libovsdb v0.6.1-0.20240125124854-03f787b1a892
 	github.com/parnurzeal/gorequest v0.3.0
 	github.com/prometheus-community/pro-bing v0.4.0
 	github.com/prometheus/client_golang v1.18.0
@@ -270,7 +270,6 @@ require (
 replace (
 	github.com/mdlayher/arp => github.com/kubeovn/arp v0.0.0-20240218024213-d9612a263f68
 	github.com/openshift/client-go => github.com/openshift/client-go v0.0.1
-	github.com/ovn-org/libovsdb => github.com/kubeovn/libovsdb v0.0.0-20240218023647-f0bc3ce57fcd
 	github.com/vishvananda/netlink => github.com/kubeovn/netlink v0.0.0-20240218024530-d3ada5dae96f
 	k8s.io/api => k8s.io/api v0.30.0
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.30.0
diff --git a/go.sum b/go.sum
index 17db14e3e56a..07f209b427e9 100644
--- a/go.sum
+++ b/go.sum
@@ -1089,8 +1089,6 @@ github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589 h1:y9e
 github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589/go.mod h1:49upX+/hUyppWIqu58cumojyIwXdkA8k6reA/mQlKuI=
 github.com/kubeovn/kubevirt-client-go v0.0.0-20240430072310-d367a71d7cd7 h1:1wXNpLgiQNcxjgKYGjwaZMid/veHyWikHe43+n6t5RI=
 github.com/kubeovn/kubevirt-client-go v0.0.0-20240430072310-d367a71d7cd7/go.mod h1:tZBpCFmVsbw2W/Q1kkQ7T9dllhsID5JCWuQJoqYx6iM=
-github.com/kubeovn/libovsdb v0.0.0-20240218023647-f0bc3ce57fcd h1:GhgvSBFKEkVNgDq8IslC04NVuoznreZH/Imz/cr6bhs=
-github.com/kubeovn/libovsdb v0.0.0-20240218023647-f0bc3ce57fcd/go.mod h1:pTnlGt1JZrncr6pJn/Fhnp3FFTMQRaTVxiSKBLVGa5s=
 github.com/kubeovn/netlink v0.0.0-20240218024530-d3ada5dae96f h1:3hH6U+CRilak3SxAX9YykAXxxAY25GTEJANLlJNE2jU=
 github.com/kubeovn/netlink v0.0.0-20240218024530-d3ada5dae96f/go.mod h1:KjTlcXwJZNXDSeBgPMWF8yKVqYrIP1cpe5HfyfEi4Ls=
 github.com/kubeovn/ovsdb v0.0.0-20240410091831-5dd26006c475 h1:KZba2Kj9TXCUdUSqOR3eiy4VvkkIyhDVImYmYs6GQWU=
@@ -1235,6 +1233,8 @@ github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/osrg/gobgp/v3 v3.26.0 h1:/iHaQKNgp0dRI3/RGt/j60aUeoGng6CL0VATVfQXEPE=
 github.com/osrg/gobgp/v3 v3.26.0/go.mod h1:ZGeSti9mURR/o5hf5R6T1FM5g1yiEBZbhP+TuqYJUpI=
+github.com/ovn-org/libovsdb v0.6.1-0.20240125124854-03f787b1a892 h1:/yg3/z+RH+iDLMxp6FTnmlk5bStK542/Rge5EBjnA9A=
+github.com/ovn-org/libovsdb v0.6.1-0.20240125124854-03f787b1a892/go.mod h1:LC5DOvcY58jOG3HTvDyCVidoMJDurPeu+xlxv5Krd9Q=
 github.com/parnurzeal/gorequest v0.3.0 h1:SoFyqCDC9COr1xuS6VA8fC8RU7XyrJZN2ona1kEX7FI=
 github.com/parnurzeal/gorequest v0.3.0/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE=
 github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo=
diff --git a/pkg/ovsdb/client/client.go b/pkg/ovsdb/client/client.go
index 38371fa204c1..f5b5c2031011 100644
--- a/pkg/ovsdb/client/client.go
+++ b/pkg/ovsdb/client/client.go
@@ -25,7 +25,9 @@ const (
 	ICNBDB = "icnbdb"
 	ICSBDB = "icsbdb"
 )
-const timeout = 3 * time.Second
+
+const connectTimeout time.Duration = time.Second * 20
+const inactivityTimeout time.Duration = time.Second * 180
 
 var namedUUIDCounter uint32
 
@@ -45,7 +47,12 @@ func NamedUUID() string {
 func NewOvsDbClient(db, addr string, dbModel model.ClientDBModel, monitors []client.MonitorOption) (client.Client, error) {
 	logger := klog.NewKlogr().WithName("libovsdb").WithValues("db", db)
 	options := []client.Option{
-		client.WithReconnect(timeout, &backoff.ConstantBackOff{Interval: time.Second}),
+		// Reading and parsing the DB after reconnect at scale can (unsurprisingly)
+		// take longer than a normal ovsdb operation. Give it a bit more time so
+		// we don't time out and enter a reconnect loop. In addition it also enables
+		// inactivity check on the ovsdb connection.
+		client.WithInactivityCheck(inactivityTimeout, connectTimeout, &backoff.ZeroBackOff{}),
+
 		client.WithLeaderOnly(true),
 		client.WithLogger(&logger),
 	}
@@ -84,7 +91,7 @@ func NewOvsDbClient(db, addr string, dbModel model.ClientDBModel, monitors []cli
 		klog.Error(err)
 		return nil, err
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(endpoints)+1)*timeout)
+	ctx, cancel := context.WithTimeout(context.Background(), connectTimeout)
 	defer cancel()
 	if err = c.Connect(ctx); err != nil {
 		klog.Errorf("failed to connect to OVN NB server %s: %v", addr, err)