Get destination address of UDP packet

[huysentruitw]

We're porting C# code to NodeJS and found something is missing in the Node datagram API.

In C# we're using Socket.ReceiveMessageFrom to receive the datagram. In addition to the remote end point, we also get the IPPacketInformation which contains the destination address of the packet (== address of local interface that received the packet).

In C# we use this method to get connected with a 3rd party device:

1. we set up a UDP socket that's bound to IPAddress.Any
2. then we receive a discovery packet from the 3rd party device on one of the interfaces
3. we need to send back the local IP address of the interface where the packet arrived on, so the 3rd party device can initiate a TCP/IP connection to it.

Is there a way to achieve the same using the Node datagram API?

[indutny]

Looking.

[huysentruitw]

Seems like socket option IP_PKTINFO has to be used for this in the underlying interface. See http://stackoverflow.com/questions/5281409/get-destination-address-of-a-received-udp-packet/5309155#5309155

[indutny]

Thank you, indeed that's what I'm working on right now :) Its not supported in libuv yet, but if I'll push it to libuv's master - node 0.11 will eventually support it.

[huysentruitw]

Fantastic!

[bnoordhuis]

IP_PKTINFO is Linux-only, the BSDs use IP_RECVDSTADDR. Solaris has a IP_RECVDSTADDR flag too but I've never tried if it actually works. There's also IP_RECVOPTS which is quite portable - except it doesn't work on OS X.

At any rate, this should be an explicit opt-in because it impacts performance.

[indutny]

@bnoordhuis thanks, I figured it out. I'm going to add flag to uv_udp_bind() and an src argument for uv_udp_recv_cb.

[indutny]

The bad news is that it doesn't work with dualstack on osx. I just can't set IP_PKTINFO on ipv6 socket...

[indutny]

I stubbed out something, but not sure if I'm going to finish it, since it doesn't work well with dual-stack udp:

diff --git a/include/uv-unix.h b/include/uv-unix.h
index 4500609..06e4d90 100644
--- a/include/uv-unix.h
+++ b/include/uv-unix.h
@@ -234,6 +234,7 @@ typedef struct {
 #define UV_TCP_PRIVATE_FIELDS /* empty */

 #define UV_UDP_PRIVATE_FIELDS                                                 \
+  int udp_flags;                                                              \
   uv_alloc_cb alloc_cb;                                                       \
   uv_udp_recv_cb recv_cb;                                                     \
   uv__io_t io_watcher;                                                        \
diff --git a/include/uv.h b/include/uv.h
index 7f34ef2..ff241af 100644
--- a/include/uv.h
+++ b/include/uv.h
@@ -811,11 +811,15 @@ struct uv_connect_s {
 enum uv_udp_flags {
   /* Disables dual stack mode. */
   UV_UDP_IPV6ONLY = 1,
+
   /*
    * Indicates message was truncated because read buffer was too small. The
    * remainder was discarded by the OS. Used in uv_udp_recv_cb.
    */
-  UV_UDP_PARTIAL = 2
+  UV_UDP_PARTIAL = 2,
+
+  /* Receive packet info for incoming messages */
+  UV_UDP_PKTINFO = 4
 };

 /*
@@ -833,7 +837,9 @@ typedef void (*uv_udp_send_cb)(uv_udp_send_t* req, int status);
  *          discard or repurpose the read buffer.
  *          < 0 if a transmission error was detected.
  *  buf     uv_buf_t with the received data.
- *  addr    struct sockaddr_in or struct sockaddr_in6.
+ *  dst     struct sockaddr_in or struct sockaddr_in6.
+ *          Valid for the duration of the callback only.
+ *  src     struct sockaddr_in or struct sockaddr_in6.
  *          Valid for the duration of the callback only.
  *  flags   One or more OR'ed UV_UDP_* constants.
  *          Right now only UV_UDP_PARTIAL is used.
@@ -841,7 +847,8 @@ typedef void (*uv_udp_send_cb)(uv_udp_send_t* req, int status);
 typedef void (*uv_udp_recv_cb)(uv_udp_t* handle,
                                ssize_t nread,
                                const uv_buf_t* buf,
-                               const struct sockaddr* addr,
+                               const struct sockaddr* dst,
+                               const struct sockaddr* src,
                                unsigned flags);

 /* uv_udp_t is a subclass of uv_handle_t */
diff --git a/src/unix/udp.c b/src/unix/udp.c
index a2b3dc3..9474f94 100644
--- a/src/unix/udp.c
+++ b/src/unix/udp.c
@@ -164,11 +164,20 @@ static void uv__udp_io(uv_loop_t* loop, uv__io_t* w, unsigned int revents) {
 static void uv__udp_recvmsg(uv_loop_t* loop,
                             uv__io_t* w,
                             unsigned int revents) {
-  struct sockaddr_storage peer;
+  struct sockaddr_storage dst_peer;
+  struct sockaddr_storage src_peer;
+  struct sockaddr_in* src_peer4;
+  struct sockaddr_in6* src_peer6;
+  struct sockaddr* src_peer_ptr;
   struct msghdr h;
   uv_udp_t* handle;
   ssize_t nread;
   uv_buf_t buf;
+  /* Should be enough to contain in_pktinfo, in6_pktinfo */
+  char cbuf[1024];
+  struct cmsghdr* cmsg;
+  struct in_pktinfo* pktinfo;
+  struct in6_pktinfo* pktinfo6;
   int flags;
   int count;

@@ -185,20 +194,25 @@ static void uv__udp_recvmsg(uv_loop_t* loop,
   count = 32;

   memset(&h, 0, sizeof(h));
-  h.msg_name = &peer;
+  h.msg_name = &dst_peer;

   do {
     handle->alloc_cb((uv_handle_t*) handle, 64 * 1024, &buf);
     if (buf.len == 0) {
-      handle->recv_cb(handle, UV_ENOBUFS, &buf, NULL, 0);
+      handle->recv_cb(handle, UV_ENOBUFS, &buf, NULL, NULL, 0);
       return;
     }
     assert(buf.base != NULL);

-    h.msg_namelen = sizeof(peer);
+    h.msg_namelen = sizeof(dst_peer);
     h.msg_iov = (void*) &buf;
     h.msg_iovlen = 1;

+    if (handle->udp_flags & UV_UDP_PKTINFO) {
+      h.msg_control = (void*) cbuf;
+      h.msg_controllen = sizeof(cbuf);
+    }
+
     do {
       nread = recvmsg(handle->io_watcher.fd, &h, 0);
     }
@@ -206,9 +220,9 @@ static void uv__udp_recvmsg(uv_loop_t* loop,

     if (nread == -1) {
       if (errno == EAGAIN || errno == EWOULDBLOCK)
-        handle->recv_cb(handle, 0, &buf, NULL, 0);
+        handle->recv_cb(handle, 0, &buf, NULL, NULL, 0);
       else
-        handle->recv_cb(handle, -errno, &buf, NULL, 0);
+        handle->recv_cb(handle, -errno, &buf, NULL, NULL, 0);
     }
     else {
       flags = 0;
@@ -216,10 +230,60 @@ static void uv__udp_recvmsg(uv_loop_t* loop,
       if (h.msg_flags & MSG_TRUNC)
         flags |= UV_UDP_PARTIAL;

+      src_peer_ptr = NULL;
+      if (handle->udp_flags & UV_UDP_PKTINFO) {
+        cmsg = CMSG_FIRSTHDR(&h);
+        src_peer4 = (struct sockaddr_in*) &src_peer;
+        src_peer6 = (struct sockaddr_in6*) &src_peer;
+        for (; cmsg != NULL; cmsg = CMSG_NXTHDR(&h, cmsg)) {
+          /* Ignore all non-pktinfo messages */
+          if (cmsg->cmsg_level == IPPROTO_IP) {
+            memset(src_peer4, 0, sizeof(*src_peer4));
+            src_peer4->sin_family = AF_INET;
+            if (cmsg->cmsg_type == IP_PKTINFO) {
+              pktinfo = (struct in_pktinfo*) CMSG_DATA(cmsg);
+              memcpy(&src_peer4->sin_addr,
+                     &pktinfo->ipi_addr,
+                     sizeof(pktinfo->ipi_addr));
+              src_peer_ptr = (struct sockaddr*) &src_peer;
+            }
+#ifdef IP_RECVDSTADDR
+            if (cmsg->cmsg_type == IP_RECVDSTADDR) {
+              memcpy(&src_peer4->sin_addr,
+                     CMSG_DATA(cmsg),
+                     sizeof(struct in_addr));
+              src_peer_ptr = (struct sockaddr*) &src_peer;
+            }
+#endif  /* IP_RECVDSTADDR */
+          } else if (cmsg->cmsg_level == IPPROTO_IPV6) {
+            memset(src_peer6, 0, sizeof(*src_peer6));
+            src_peer6->sin6_family = AF_INET6;
+            if (cmsg->cmsg_type == IP_PKTINFO) {
+              pktinfo6 = (struct in6_pktinfo*) CMSG_DATA(cmsg);
+              memcpy(&src_peer6->sin6_addr,
+                     &pktinfo6->ipi6_addr,
+                     sizeof(pktinfo6->ipi6_addr));
+              src_peer_ptr = (struct sockaddr*) &src_peer;
+            }
+#ifdef IPV6_RECVDSTADDR
+            if (cmsg->cmsg_type == IPV6_RECVDSTADDR) {
+              memcpy(&src_peer6->sin6_addr,
+                     CMSG_DATA(cmsg),
+                     sizeof(struct in_addr6));
+              src_peer_ptr = (struct sockaddr*) &src_peer;
+            }
+#endif  /* IPV6_RECVDSTADDR */
+          }
+          if (src_peer_ptr != NULL)
+            break;
+        }
+      }
+
       handle->recv_cb(handle,
                       nread,
                       &buf,
-                      (const struct sockaddr*) &peer,
+                      (const struct sockaddr*) &dst_peer,
+                      src_peer_ptr,
                       flags);
     }
   }
@@ -300,7 +364,7 @@ int uv__udp_bind(uv_udp_t* handle,
   fd = -1;

   /* Check for bad flags. */
-  if (flags & ~UV_UDP_IPV6ONLY)
+  if (flags & ~(UV_UDP_IPV6ONLY | UV_UDP_PKTINFO))
     return -EINVAL;

   /* Cannot set IPv6-only mode on non-IPv6 socket. */
@@ -337,6 +401,37 @@ int uv__udp_bind(uv_udp_t* handle,
     goto out;
   }

+  if (flags & UV_UDP_PKTINFO) {
+    if ((flags & UV_UDP_IPV6ONLY) == 0) {
+#if defined(__linux__) || defined(__sun)
+      yes = 1;
+      if (setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &yes, sizeof(yes)))
+        err = -errno;
+#elif defined(__APPLE__)
+      /* On Mac OS, setting IP_PKTINFO on dual-stack ipv6 socket won't work */
+      err = 0;
+#elif
+      yes = 1;
+      if (setsockopt(fd, IPPROTO_IP, IP_RECVDSTADDR, &yes, sizeof(yes)))
+        err = -errno;
+#endif
+    }
+    if (err == 0 && addr->sa_family == AF_INET6) {
+      yes = 1;
+#if defined(__linux__) || defined(__sun) || defined(__APPLE__)
+      if (setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &yes, sizeof(yes)))
+        err = -errno;
+#else
+      if (setsockopt(fd, IPPROTO_IPV6, IPV6_RECVDSTADDR, &yes, sizeof(yes)))
+        err = -errno;
+#endif  /* defined(__linux__) || defined(__sun) || defined(__APPLE__) */
+    }
+
+    if (err)
+      goto out;
+    handle->udp_flags |= UV_UDP_PKTINFO;
+  }
+
   return 0;

 out:
@@ -424,6 +519,7 @@ int uv__udp_send(uv_udp_send_t* req,

 int uv_udp_init(uv_loop_t* loop, uv_udp_t* handle) {
   uv__handle_init(loop, (uv_handle_t*)handle, UV_UDP);
+  handle->udp_flags = 0;
   handle->alloc_cb = NULL;
   handle->recv_cb = NULL;
   uv__io_init(&handle->io_watcher, uv__udp_io, -1);
diff --git a/test/benchmark-udp-pummel.c b/test/benchmark-udp-pummel.c
index d99250a..d2d9e9b 100644
--- a/test/benchmark-udp-pummel.c
+++ b/test/benchmark-udp-pummel.c
@@ -109,7 +109,8 @@ send:
 static void recv_cb(uv_udp_t* handle,
                     ssize_t nread,
                     const uv_buf_t* buf,
-                    const struct sockaddr* addr,
+                    const struct sockaddr* dst,
+                    const struct sockaddr* src,
                     unsigned flags) {
   if (nread == 0)
     return;
@@ -119,7 +120,7 @@ static void recv_cb(uv_udp_t* handle,
     return;
   }

-  ASSERT(addr->sa_family == AF_INET);
+  ASSERT(dst->sa_family == AF_INET);
   ASSERT(!memcmp(buf->base, EXPECTED, nread));

   recv_cb_called++;
diff --git a/test/echo-server.c b/test/echo-server.c
index 193a168..e32880f 100644
--- a/test/echo-server.c
+++ b/test/echo-server.c
@@ -191,19 +191,20 @@ static void on_send(uv_udp_send_t* req, int status);
 static void on_recv(uv_udp_t* handle,
                     ssize_t nread,
                     const uv_buf_t* rcvbuf,
-                    const struct sockaddr* addr,
+                    const struct sockaddr* dst,
+                    const struct sockaddr* src,
                     unsigned flags) {
   uv_udp_send_t* req;
   uv_buf_t sndbuf;

   ASSERT(nread > 0);
-  ASSERT(addr->sa_family == AF_INET);
+  ASSERT(dst->sa_family == AF_INET);

   req = malloc(sizeof(*req));
   ASSERT(req != NULL);

   sndbuf = *rcvbuf;
-  ASSERT(0 == uv_udp_send(req, handle, &sndbuf, 1, addr, on_send));
+  ASSERT(0 == uv_udp_send(req, handle, &sndbuf, 1, dst, on_send));
 }


diff --git a/test/test-getsockname.c b/test/test-getsockname.c
index dc6a949..e67f207 100644
--- a/test/test-getsockname.c
+++ b/test/test-getsockname.c
@@ -238,7 +238,8 @@ static void tcp_connector(void) {
 static void udp_recv(uv_udp_t* handle,
                      ssize_t nread,
                      const uv_buf_t* buf,
-                     const struct sockaddr* addr,
+                     const struct sockaddr* dst,
+                     const struct sockaddr* src,
                      unsigned flags) {
   struct sockaddr sockname;
   int namelen;
diff --git a/test/test-udp-ipv6.c b/test/test-udp-ipv6.c
index 32cabf0..37f62d5 100644
--- a/test/test-udp-ipv6.c
+++ b/test/test-udp-ipv6.c
@@ -71,7 +71,8 @@ static void send_cb(uv_udp_send_t* req, int status) {
 static void ipv6_recv_fail(uv_udp_t* handle,
                            ssize_t nread,
                            const uv_buf_t* buf,
-                           const struct sockaddr* addr,
+                           const struct sockaddr* dst,
+                           const struct sockaddr* src,
                            unsigned flags) {
   ASSERT(0 && "this function should not have been called");
 }
@@ -80,10 +81,12 @@ static void ipv6_recv_fail(uv_udp_t* handle,
 static void ipv6_recv_ok(uv_udp_t* handle,
                          ssize_t nread,
                          const uv_buf_t* buf,
-                         const struct sockaddr* addr,
+                         const struct sockaddr* dst,
+                         const struct sockaddr* src,
                          unsigned flags) {
   CHECK_HANDLE(handle);
   ASSERT(nread >= 0);
+  ASSERT(dst != NULL);

   if (nread)
     recv_cb_called++;
@@ -108,6 +111,7 @@ static void do_test(uv_udp_recv_cb recv_cb, int bind_flags) {
   r = uv_udp_init(uv_default_loop(), &server);
   ASSERT(r == 0);

+  bind_flags |= UV_UDP_PKTINFO;
   r = uv_udp_bind(&server, (const struct sockaddr*) &addr6, bind_flags);
   ASSERT(r == 0);

diff --git a/test/test-udp-multicast-join.c b/test/test-udp-multicast-join.c
index 686edf3..0274239 100644
--- a/test/test-udp-multicast-join.c
+++ b/test/test-udp-multicast-join.c
@@ -69,7 +69,8 @@ static void sv_send_cb(uv_udp_send_t* req, int status) {
 static void cl_recv_cb(uv_udp_t* handle,
                        ssize_t nread,
                        const uv_buf_t* buf,
-                       const struct sockaddr* addr,
+                       const struct sockaddr* dst,
+                       const struct sockaddr* src,
                        unsigned flags) {
   CHECK_HANDLE(handle);
   ASSERT(flags == 0);
@@ -83,11 +84,11 @@ static void cl_recv_cb(uv_udp_t* handle,
   if (nread == 0) {
     /* Returning unused buffer */
     /* Don't count towards cl_recv_cb_called */
-    ASSERT(addr == NULL);
+    ASSERT(dst == NULL);
     return;
   }

-  ASSERT(addr != NULL);
+  ASSERT(dst != NULL);
   ASSERT(nread == 4);
   ASSERT(!memcmp("PING", buf->base, nread));

diff --git a/test/test-udp-open.c b/test/test-udp-open.c
index 9a97303..68d4b93 100644
--- a/test/test-udp-open.c
+++ b/test/test-udp-open.c
@@ -86,7 +86,8 @@ static void close_cb(uv_handle_t* handle) {
 static void recv_cb(uv_udp_t* handle,
                        ssize_t nread,
                        const uv_buf_t* buf,
-                       const struct sockaddr* addr,
+                       const struct sockaddr* dst,
+                       const struct sockaddr* src,
                        unsigned flags) {
   int r;

@@ -97,13 +98,13 @@ static void recv_cb(uv_udp_t* handle,
   if (nread == 0) {
     /* Returning unused buffer */
     /* Don't count towards sv_recv_cb_called */
-    ASSERT(addr == NULL);
+    ASSERT(dst == NULL);
     return;
   }

   ASSERT(flags == 0);

-  ASSERT(addr != NULL);
+  ASSERT(dst != NULL);
   ASSERT(nread == 4);
   ASSERT(memcmp("PING", buf->base, nread) == 0);

diff --git a/test/test-udp-send-and-recv.c b/test/test-udp-send-and-recv.c
index 3020ded..ef2542f 100644
--- a/test/test-udp-send-and-recv.c
+++ b/test/test-udp-send-and-recv.c
@@ -62,7 +62,8 @@ static void close_cb(uv_handle_t* handle) {
 static void cl_recv_cb(uv_udp_t* handle,
                        ssize_t nread,
                        const uv_buf_t* buf,
-                       const struct sockaddr* addr,
+                       const struct sockaddr* dst,
+                       const struct sockaddr* src,
                        unsigned flags) {
   CHECK_HANDLE(handle);
   ASSERT(flags == 0);
@@ -74,11 +75,11 @@ static void cl_recv_cb(uv_udp_t* handle,
   if (nread == 0) {
     /* Returning unused buffer */
     /* Don't count towards cl_recv_cb_called */
-    ASSERT(addr == NULL);
+    ASSERT(dst == NULL);
     return;
   }

-  ASSERT(addr != NULL);
+  ASSERT(dst != NULL);
   ASSERT(nread == 4);
   ASSERT(!memcmp("PONG", buf->base, nread));

@@ -117,7 +118,8 @@ static void sv_send_cb(uv_udp_send_t* req, int status) {
 static void sv_recv_cb(uv_udp_t* handle,
                        ssize_t nread,
                        const uv_buf_t* rcvbuf,
-                       const struct sockaddr* addr,
+                       const struct sockaddr* dst,
+                       const struct sockaddr* src,
                        unsigned flags) {
   uv_udp_send_t* req;
   uv_buf_t sndbuf;
@@ -130,14 +132,16 @@ static void sv_recv_cb(uv_udp_t* handle,
   if (nread == 0) {
     /* Returning unused buffer */
     /* Don't count towards sv_recv_cb_called */
-    ASSERT(addr == NULL);
+    ASSERT(dst == NULL);
+    ASSERT(src == NULL);
     return;
   }

   CHECK_HANDLE(handle);
   ASSERT(flags == 0);

-  ASSERT(addr != NULL);
+  ASSERT(dst != NULL);
+  ASSERT(src != NULL);
   ASSERT(nread == 4);
   ASSERT(!memcmp("PING", rcvbuf->base, nread));

@@ -152,7 +156,7 @@ static void sv_recv_cb(uv_udp_t* handle,
   ASSERT(req != NULL);

   sndbuf = uv_buf_init("PONG", 4);
-  r = uv_udp_send(req, handle, &sndbuf, 1, addr, sv_send_cb);
+  r = uv_udp_send(req, handle, &sndbuf, 1, dst, sv_send_cb);
   ASSERT(r == 0);

   sv_recv_cb_called++;
@@ -170,7 +174,7 @@ TEST_IMPL(udp_send_and_recv) {
   r = uv_udp_init(uv_default_loop(), &server);
   ASSERT(r == 0);

-  r = uv_udp_bind(&server, (const struct sockaddr*) &addr, 0);
+  r = uv_udp_bind(&server, (const struct sockaddr*) &addr, UV_UDP_PKTINFO);
   ASSERT(r == 0);

   r = uv_udp_recv_start(&server, alloc_cb, sv_recv_cb);
diff --git a/test/test-watcher-cross-stop.c b/test/test-watcher-cross-stop.c
index c701dd2..8035306 100644
--- a/test/test-watcher-cross-stop.c
+++ b/test/test-watcher-cross-stop.c
@@ -42,7 +42,8 @@ static void alloc_cb(uv_handle_t* handle, size_t size, uv_buf_t* buf) {
 static void recv_cb(uv_udp_t* handle,
                     ssize_t nread,
                     const uv_buf_t* buf,
-                    const struct sockaddr* addr,
+                    const struct sockaddr* dst,
+                    const struct sockaddr* src,
                     unsigned flags) {
   recv_cb_called++;
 }
diff --git a/uv.gyp b/uv.gyp
index fb37ef5..7cb598a 100644
--- a/uv.gyp
+++ b/uv.gyp
@@ -190,6 +190,7 @@
           ],
           'defines': [
             '_DARWIN_USE_64_BIT_INODE=1',
+            '__APPLE_USE_RFC_3542=1',
           ]
         }],
         [ 'OS!="mac"', {

[indutny]

@WouterHuysentruit sorry, I don't think I'm going to finish it. There're really unsolvable problems with dual-stack support. Closing it.

[chrisdew]

@indutny This is an issue for me too.

I have two interfaces on the same subnet (a real and a cluster address) and I need to send UDP replies from the address to which the original datagram was sent.

Any chance of getting it reopened?

@WouterHuysentruit Did you find a work-around?

[indutny]

@chrisdew as far as I remember, it doesn't work well on OSX and probably on FreeBSD. You'll get better results by listening on two interfaces explicitly.

[chrisdew]

I've hacked around this issue by adding a custom route (with a defined 'src') when the cluster alias interface is brought up.

(Listening on interfaces explicitly is too complicated when those interface can come and go at runtime.)

[mikinho]

@indutny Listening on two interfaces explicitly does not negate needing IP_PKTINFO, at least when receiving udp multicast broadcast messages which you have to bind to INADDR_ANY or in6addr_any.

I have a working patch for IPv4 (IP_PKTINFO) and IPv6 (IPV6_RECVPKTINFO) for the 0.10.x stable branch (what I need for) but I haven't had a chance to test BSD yet, only Windows, Linux and OSX.

Can you think of any other issues you had?

[indutny]

@mikinho Sorry, but we won't be able to land it in v0.10, the API is locked there.

As far as I understand your patch still won't work with dualstack sockets (or work only for incoming ipv6 udp packets), right? Feel free to submit it to https://github.com/joyent/libuv anyway.

Thank you!

[mikinho]

Dual-Stack Sockets does work with the exception that if IPv4 is disabled then the setsockopt will fail. I could easily detect IPv4 being disabled on the system and use the appropriate flag to get around that but in my mind that should be up to the application using the API, not libuv. I've updated my patch to v0.11. I just did v0.10 since that is what I needed it for. I'll move the discussion elsewhere after submitting the patch, I hadn't noticed this was the node repo, I assumed it was the libuv.

[bushje]

@mikinho Hi there, Id like to give your patch a try if possible. Have you submitted it to the main node repo?

Many thanks.