lkl tools: virtio net: add lkl_netdev_free

Right now we free the lkl_netdev structure in lkl_netdev_remove. This
has two issues: it does more then its counterpart lkl_netdev_alloc and
it relies on the facts that the structure is allocated with
lkl_ops.alloc_mem and that lkl_netdev is at the beggining of the backend
structure.

This patch implements a consistent netdev deallocation by adding
lkl_netdev_free and converting the close netdev operation into two other
operations: poll_hup and free. poll_hup is used to stop the polling
thread while free is used to free the backends resources and the
lkl_netdev structure.

Signed-off-by: Octavian Purdila <octavian.purdila@intel.com>

tools/lkl/include/lkl.h

@@ -279,6 +279,13 @@ int lkl_netdev_add(struct lkl_netdev *nd, struct lkl_netdev_args* args);
 */
 void lkl_netdev_remove(int id);
 
+/**
+ * lkl_netdev_free - frees a network device
+ *
+ * @nd - the network device to free
+ */
+void lkl_netdev_free(struct lkl_netdev *nd);
+
 /**
  * lkl_netdev_get_ifindex - retrieve the interface index for a given network
  * device id

tools/lkl/include/lkl_host.h

@@ -97,12 +97,17 @@ struct lkl_dev_net_ops {
 	int (*poll)(struct lkl_netdev *nd);
 
 	/*
-	 * Closes a net device.
+	 * Make poll wakeup and return LKL_DEV_NET_POLL_HUP.
+	 */
+	void (*poll_hup)(struct lkl_netdev *nd);
+
+	/*
+	 * Frees a network device.
 	 *
-	 * Implementation must release its resources and poll must wakeup and
-	 * return LKL_DEV_NET_POLL_HUP.
+	 * Implementation must release its resources and free the network device
+	 * structure.
 	 */
-	void (*close)(struct lkl_netdev *nd);
+	void (*free)(struct lkl_netdev *nd);
 };
 
 #ifdef __cplusplus

tools/lkl/lib/hijack/init.c

@@ -179,6 +179,7 @@ static void PinToFirstCpu(const cpu_set_t* cpus)
 int lkl_debug, lkl_running;
 
 static int nd_id = -1;
+static struct lkl_netdev *nd;
 
 void __attribute__((constructor(102)))
 hijack_init(void)
@@ -199,7 +200,6 @@ hijack_init(void)
 	char *gateway6 = getenv("LKL_HIJACK_NET_GATEWAY6");
 	char *debug = getenv("LKL_HIJACK_DEBUG");
 	char *mount = getenv("LKL_HIJACK_MOUNT");
-	struct lkl_netdev *nd = NULL;
 	struct lkl_netdev_args nd_args;
 	char *neigh_entries = getenv("LKL_HIJACK_NET_NEIGHBOR");
 	/* single_cpu mode:
@@ -436,5 +436,8 @@ hijack_fini(void)
 	if (nd_id >= 0)
 		lkl_netdev_remove(nd_id);
 
+	if (nd)
+		lkl_netdev_free(nd);
+
 	lkl_sys_halt();
 }

tools/lkl/lib/virtio_net.c

@@ -275,6 +275,9 @@ void lkl_netdev_remove(int id)
 
 	dev = registered_devs[id];
 
+	dev->nd->ops->poll_hup(dev->nd);
+	lkl_host_ops.thread_join(dev->poll_tid);
+
 	ret = lkl_netdev_get_ifindex(id);
 	if (ret < 0) {
 		lkl_printf("%s: failed to get ifindex for id %d: %s\n",
@@ -289,13 +292,13 @@ void lkl_netdev_remove(int id)
 		return;
 	}
 
-	dev->nd->ops->close(dev->nd);
-
-	lkl_host_ops.thread_join(dev->poll_tid);
-
 	virtio_dev_cleanup(&dev->dev);
 
-	lkl_host_ops.mem_free(dev->nd);
 	free_queue_locks(dev->queue_locks, NUM_QUEUES);
 	lkl_host_ops.mem_free(dev);
 }
+
+void lkl_netdev_free(struct lkl_netdev *nd)
+{
+	nd->ops->free(nd);
+}

tools/lkl/lib/virtio_net_dpdk.c

@@ -170,19 +170,28 @@ static int net_poll(struct lkl_netdev *nd)
 	return LKL_DEV_NET_POLL_RX | LKL_DEV_NET_POLL_TX;
 }
 
-static void net_close(struct lkl_netdev *nd)
+static void net_poll_hup(struct lkl_netdev *nd)
 {
 	struct lkl_netdev_dpdk *nd_dpdk =
 		container_of(nd, struct lkl_netdev_dpdk, dev);
 
 	nd_dpdk->close = 1;
 }
 
+static void net_free(struct lkl_netdev *nd)
+{
+	struct lkl_netdev_dpdk *nd_dpdk =
+		container_of(nd, struct lkl_netdev_dpdk, dev);
+
+	free(nd_dpdk);
+}
+
 struct lkl_dev_net_ops dpdk_net_ops = {
 	.tx = net_tx,
 	.rx = net_rx,
 	.poll = net_poll,
-	.close = net_close,
+	.poll_hup = net_poll_hup,
+	.free = net_free,
 };
 
 

tools/lkl/lib/virtio_net_fd.c

@@ -152,22 +152,31 @@ static int fd_net_poll(struct lkl_netdev *nd)
 	return ret;
 }
 
-static void fd_net_close(struct lkl_netdev *nd)
+static void fd_net_poll_hup(struct lkl_netdev *nd)
 {
 	struct lkl_netdev_fd *nd_fd =
 		container_of(nd, struct lkl_netdev_fd, dev);
 
-	/* this will cause a POLLHUP in the poll function */
-	close(nd_fd->pipe[1]);
+	/* this will cause a POLLHUP / POLLNVAL in the poll function */
 	close(nd_fd->pipe[0]);
+	close(nd_fd->pipe[1]);
+}
+
+static void fd_net_free(struct lkl_netdev *nd)
+{
+	struct lkl_netdev_fd *nd_fd =
+		container_of(nd, struct lkl_netdev_fd, dev);
+
 	close(nd_fd->fd);
+	free(nd_fd);
 }
 
 struct lkl_dev_net_ops fd_net_ops =  {
 	.tx = fd_net_tx,
 	.rx = fd_net_rx,
 	.poll = fd_net_poll,
-	.close = fd_net_close,
+	.poll_hup = fd_net_poll_hup,
+	.free = fd_net_free,
 };
 
 struct lkl_netdev *lkl_register_netdev_fd(int fd)
@@ -186,27 +195,18 @@ struct lkl_netdev *lkl_register_netdev_fd(int fd)
 	nd->fd = fd;
 	if (pipe(nd->pipe) < 0) {
 		perror("pipe");
-		lkl_unregister_netdev_fd(&nd->dev);
+		free(nd);
 		return NULL;
 	}
 
 	if (fcntl(nd->pipe[0], F_SETFL, O_NONBLOCK) < 0) {
 		perror("fnctl");
 		close(nd->pipe[0]);
 		close(nd->pipe[1]);
-		lkl_unregister_netdev_fd(&nd->dev);
+		free(nd);
 		return NULL;
 	}
 
 	nd->dev.ops = &fd_net_ops;
 	return &nd->dev;
 }
-
-void lkl_unregister_netdev_fd(struct lkl_netdev *nd)
-{
-	struct lkl_netdev_fd *nd_fd =
-		container_of(nd, struct lkl_netdev_fd, dev);
-
-	fd_net_close(nd);
-	free(nd_fd);
-}

tools/lkl/lib/virtio_net_fd.h

@@ -13,14 +13,6 @@ struct ifreq;
 struct lkl_netdev *lkl_register_netdev_fd(int fd);
 
 
-/**
- * lkl_unregister_netdev_linux_fdnet - unregister a file descriptor-based
- * network device as a NIC
- *
- * @nd - a struct lkl_netdev_linux_fdnet entry to be unregistered
- */
-void lkl_unregister_netdev_fd(struct lkl_netdev *nd);
-
 /**
  * lkl_netdev_tap_init - initialize tap related structure fot lkl_netdev.
  *

tools/lkl/lib/virtio_net_vde.c

@@ -22,13 +22,15 @@ static int net_vde_tx(struct lkl_netdev *nd, struct lkl_dev_buf *iov, int cnt);
 static int net_vde_rx(struct lkl_netdev *nd, struct lkl_dev_buf *iov, int cnt);
 static int net_vde_poll_with_timeout(struct lkl_netdev *nd, int timeout);
 static int net_vde_poll(struct lkl_netdev *nd);
-static int net_vde_close(struct lkl_netdev *nd);
+static void net_vde_poll_hup(struct lkl_netdev *nd);
+static void net_vde_free(struct lkl_netdev *nd);
 
 struct lkl_dev_net_ops vde_net_ops = {
 	.tx = net_vde_tx,
 	.rx = net_vde_rx,
 	.poll = net_vde_poll,
-	.close = net_vde_close,
+	.poll_hup = net_vde_poll_hup,
+	.free = net_vde_free,
 };
 
 int net_vde_tx(struct lkl_netdev *nd, struct lkl_dev_buf *iov, int cnt)
@@ -107,14 +109,22 @@ int net_vde_poll(struct lkl_netdev *nd)
 	return net_vde_poll_with_timeout(nd, -1);
 }
 
-void net_vde_close(struct lkl_netdev *nd)
+void net_vde_poll_hup(struct lkl_netdev *nd)
 {
 	struct lkl_netdev_vde *nd_vde =
 		container_of(nd, struct lkl_netdev_vde, dev);
 
 	vde_close(nd_vde->conn);
 }
 
+void net_vde_free(struct lkl_netdev *nd)
+{
+	struct lkl_netdev_vde *nd_vde =
+		container_of(nd, struct lkl_netdev_vde, dev);
+
+	free(nd_vde);
+}
+
 struct lkl_netdev *lkl_netdev_vde_create(char const *switch_path)
 {
 	struct lkl_netdev_vde *nd;