From f4d37fb62e97a61030baeb36865c75d0b3b0d04a Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Mon, 16 Sep 2019 04:12:37 -0700 Subject: [PATCH 1/5] Changes to support NAT feature. * Add new tables in CONFIG_DB, APP_DB, COUNTERS_DB * Netlink class for abstracting netfilter application socket Signed-off-by: kiran.kella@broadcom.com --- common/Makefile.am | 1 + common/ipaddress.h | 5 ++ common/nfnetlink.cpp | 208 +++++++++++++++++++++++++++++++++++++++++++ common/nfnetlink.h | 39 ++++++++ common/schema.h | 20 +++++ configure.ac | 2 +- 6 files changed, 274 insertions(+), 1 deletion(-) create mode 100644 common/nfnetlink.cpp create mode 100644 common/nfnetlink.h diff --git a/common/Makefile.am b/common/Makefile.am index f28eda096..b71b28f1d 100644 --- a/common/Makefile.am +++ b/common/Makefile.am @@ -44,6 +44,7 @@ libswsscommon_la_SOURCES = \ macaddress.cpp \ netdispatcher.cpp \ netlink.cpp \ + nfnetlink.cpp \ notificationconsumer.cpp \ notificationproducer.cpp \ linkcache.cpp \ diff --git a/common/ipaddress.h b/common/ipaddress.h index bf890e435..920f335b0 100644 --- a/common/ipaddress.h +++ b/common/ipaddress.h @@ -72,6 +72,11 @@ class IpAddress ); } + inline bool operator!=(const IpAddress &o) const + { + return (! (*this == o) ); + } + std::string to_string() const; enum AddrScope { diff --git a/common/nfnetlink.cpp b/common/nfnetlink.cpp new file mode 100644 index 000000000..eeff840e7 --- /dev/null +++ b/common/nfnetlink.cpp @@ -0,0 +1,208 @@ +#include +#include +#include +#include "common/logger.h" +#include "common/netmsg.h" +#include "common/netdispatcher.h" +#include "common/netlink.h" +#include "common/nfnetlink.h" + +using namespace swss; +using namespace std; + +NfNetlink::NfNetlink(int pri) : + Selectable(pri) +{ + m_socket = nl_socket_alloc(); + if (!m_socket) + { + SWSS_LOG_ERROR("Unable to allocated nfnetlink socket"); + throw system_error(make_error_code(errc::address_not_available), + "Unable to allocate nfnetlink socket"); + } + + nl_socket_disable_seq_check(m_socket); + nl_socket_disable_auto_ack(m_socket); + + int err = nfnl_connect(m_socket); + if (err < 0) + { + SWSS_LOG_ERROR("Unable to connect nfnetlink socket: %s", nl_geterror(err)); + nl_socket_free(m_socket); + m_socket = NULL; + throw system_error(make_error_code(errc::address_not_available), + "Unable to connect nfnetlink socket"); + } + + nl_socket_set_nonblocking(m_socket); + + /* Set socket buffer size to 10 MB */ + nl_socket_set_buffer_size(m_socket, 10485760, 0); + +#ifdef NETFILTER_UNIT_TEST + /* For netlink packet tracing purposes */ + nfPktsLogFile = fopen("/var/log/nf_netlink_pkts.log", "w"); + if (nfPktsLogFile == NULL) + { + SWSS_LOG_ERROR("Unable to open netfilter netlink packets log file: %s", strerror(errno)); + } +#endif +} + +NfNetlink::~NfNetlink() +{ + if (m_socket != NULL) + { + nl_close(m_socket); + nl_socket_free(m_socket); + } +} + +void NfNetlink::registerRecvCallbacks(void) +{ +#ifdef NETFILTER_UNIT_TEST + nl_socket_modify_cb(m_socket, NL_CB_MSG_IN, NL_CB_CUSTOM, onNetlinkRcv, this); +#endif + + nl_socket_modify_cb(m_socket, NL_CB_VALID, NL_CB_CUSTOM, onNetlinkMsg, this); +} + +void NfNetlink::registerGroup(int nfnlGroup) +{ + int err = nl_socket_add_membership(m_socket, nfnlGroup); + if (err < 0) + { + SWSS_LOG_ERROR("Unable to register to netfilter group %d: %s", nfnlGroup, + nl_geterror(err)); + throw system_error(make_error_code(errc::address_not_available), + "Unable to register to netfilter group"); + } +} +void NfNetlink::dumpRequest(int getCommand) +{ + int err = nfnl_ct_dump_request(m_socket); + if (err < 0) + { + SWSS_LOG_ERROR("Unable to request netfilter conntrack dump: %s", + nl_geterror(err)); + throw system_error(make_error_code(errc::address_not_available), + "Unable to request netfilter conntrack dump"); + } +} + +void NfNetlink::updateConnTrackEntry(struct nfnl_ct *ct) +{ + if (nfnl_ct_add(m_socket, ct, NLM_F_REPLACE) < 0) + { + SWSS_LOG_ERROR("Failed to update conntrack object in the kernel"); + } +} + +void NfNetlink::deleteConnTrackEntry(struct nfnl_ct *ct) +{ + if (nfnl_ct_del(m_socket, ct, 0) < 0) + { + SWSS_LOG_ERROR("Failed to delete conntrack object in the kernel"); + } +} + +struct nfnl_ct *NfNetlink::getCtObject(const IpAddress &sourceIpAddr) +{ + struct nfnl_ct *ct = nfnl_ct_alloc(); + struct nl_addr *orig_src_ip; + + if (! ct) + { + SWSS_LOG_ERROR("Unable to allocate memory for conntrack object"); + return NULL; + } + + nfnl_ct_set_family(ct, AF_INET); + if (nl_addr_parse(sourceIpAddr.to_string().c_str(), AF_INET, &orig_src_ip) == 0) + { + nfnl_ct_set_src(ct, 0, orig_src_ip); + } + else + { + SWSS_LOG_ERROR("Failed to parse orig src ip into conntrack object"); + nfnl_ct_put(ct); + return NULL; + } + + return ct; +} + +struct nfnl_ct *NfNetlink::getCtObject(uint8_t protoType, const IpAddress &sourceIpAddr, uint16_t srcPort) +{ + struct nfnl_ct *ct = nfnl_ct_alloc(); + struct nl_addr *orig_src_ip; + + if (! ct) + { + SWSS_LOG_ERROR("Unable to allocate memory for conntrack object"); + return NULL; + } + + nfnl_ct_set_family(ct, AF_INET); + nfnl_ct_set_proto(ct, protoType); + + if (nl_addr_parse(sourceIpAddr.to_string().c_str(), AF_INET, &orig_src_ip) == 0) + { + nfnl_ct_set_src(ct, 0, orig_src_ip); + } + else + { + SWSS_LOG_ERROR("Failed to parse orig src ip into conntrack object"); + nfnl_ct_put(ct); + return NULL; + } + nfnl_ct_set_src_port(ct, 0, srcPort); + + return ct; +} + +int NfNetlink::getFd() +{ + return nl_socket_get_fd(m_socket); +} + +void NfNetlink::readData() +{ + int err; + + do + { + err = nl_recvmsgs_default(m_socket); + } + while(err == -NLE_INTR); // Retry if the process was interrupted by a signal + + if (err < 0) + { + if (err == -NLE_NOMEM) + SWSS_LOG_ERROR("netlink reports out of memory on reading a netfilter netlink socket. High possiblity of a lost message"); + else if (err == -NLE_AGAIN) + SWSS_LOG_DEBUG("netlink reports NLE_AGAIN on reading a netfilter netlink socket"); + else + SWSS_LOG_ERROR("netlink reports an error=%d on reading a netfilter netlink socket", err); + } +} + +int NfNetlink::onNetlinkMsg(struct nl_msg *msg, void *arg) +{ + NetDispatcher::getInstance().onNetlinkMessage(msg); + return NL_OK; +} + +#ifdef NETFILTER_UNIT_TEST +int NfNetlink::onNetlinkRcv(struct nl_msg *msg, void *arg) +{ + NfNetlink *nfnlink = (NfNetlink *)arg; + + if ((nfnlink == NULL) || (nfnlink->nfPktsLogFile == NULL)) + return NL_OK; + + nl_msg_dump(msg, nfnlink->nfPktsLogFile); + + return NL_OK; +} +#endif diff --git a/common/nfnetlink.h b/common/nfnetlink.h new file mode 100644 index 000000000..1245813a1 --- /dev/null +++ b/common/nfnetlink.h @@ -0,0 +1,39 @@ +#ifndef __NFNETLINK__ +#define __NFNETLINK__ + +#include "ipaddress.h" +#include +#include + +namespace swss { + +class NfNetlink : public Selectable { +public: + NfNetlink(int pri = 0); + ~NfNetlink() override; + + void registerRecvCallbacks(void); + void registerGroup(int nfnlGroup); + void dumpRequest(int getCommand); + + int getFd() override; + void readData() override; + + void updateConnTrackEntry(struct nfnl_ct *ct); + void deleteConnTrackEntry(struct nfnl_ct *ct); + +private: + struct nfnl_ct *getCtObject(const IpAddress &sourceIpAddr); + struct nfnl_ct *getCtObject(uint8_t protoType, const IpAddress &sourceIpAddr, uint16_t srcPort); + + static int onNetlinkRcv(struct nl_msg *msg, void *arg); + static int onNetlinkMsg(struct nl_msg *msg, void *arg); + + FILE *nfPktsLogFile; + nl_sock *m_socket; +}; + +} + +#endif /* __NFNETLINK__ */ + diff --git a/common/schema.h b/common/schema.h index d60c37053..22973f462 100644 --- a/common/schema.h +++ b/common/schema.h @@ -45,6 +45,13 @@ namespace swss { #define APP_SFLOW_SESSION_TABLE_NAME "SFLOW_SESSION_TABLE" #define APP_SFLOW_SAMPLE_RATE_TABLE_NAME "SFLOW_SAMPLE_RATE_TABLE" +#define APP_NAT_TABLE_NAME "NAT_TABLE" +#define APP_NAPT_TABLE_NAME "NAPT_TABLE" +#define APP_NAT_TWICE_TABLE_NAME "NAT_TWICE_TABLE" +#define APP_NAPT_TWICE_TABLE_NAME "NAPT_TWICE_TABLE" +#define APP_NAT_GLOBAL_TABLE_NAME "NAT_GLOBAL_TABLE" +#define APP_NAPT_POOL_IP_TABLE_NAME "NAPT_POOL_IP_TABLE" + /***** TO BE REMOVED *****/ #define APP_TC_TO_QUEUE_MAP_TABLE_NAME "TC_TO_QUEUE_MAP_TABLE" @@ -83,6 +90,12 @@ namespace swss { #define COUNTERS_DEBUG_NAME_PORT_STAT_MAP "COUNTERS_DEBUG_NAME_PORT_STAT_MAP" #define COUNTERS_DEBUG_NAME_SWITCH_STAT_MAP "COUNTERS_DEBUG_NAME_SWITCH_STAT_MAP" +#define COUNTERS_NAT_TABLE "COUNTERS_NAT" +#define COUNTERS_NAPT_TABLE "COUNTERS_NAPT" +#define COUNTERS_TWICE_NAT_TABLE "COUNTERS_TWICE_NAT" +#define COUNTERS_TWICE_NAPT_TABLE "COUNTERS_TWICE_NAPT" +#define COUNTERS_GLOBAL_NAT_TABLE "COUNTERS_GLOBAL_NAT" + #define PERIODIC_WATERMARKS_TABLE "PERIODIC_WATERMARKS" #define PERSISTENT_WATERMARKS_TABLE "PERSISTENT_WATERMARKS" #define USER_WATERMARKS_TABLE "USER_WATERMARKS" @@ -200,6 +213,12 @@ namespace swss { #define CFG_DEBUG_COUNTER_TABLE_NAME "DEBUG_COUNTER" #define CFG_DEBUG_COUNTER_DROP_REASON_TABLE_NAME "DEBUG_COUNTER_DROP_REASON" +#define CFG_STATIC_NAT_TABLE_NAME "STATIC_NAT" +#define CFG_STATIC_NAPT_TABLE_NAME "STATIC_NAPT" +#define CFG_NAT_POOL_TABLE_NAME "NAT_POOL" +#define CFG_NAT_BINDINGS_TABLE_NAME "NAT_BINDINGS" +#define CFG_NAT_GLOBAL_TABLE_NAME "NAT_GLOBAL" + /***** STATE DATABASE *****/ #define STATE_SWITCH_CAPABILITY_TABLE_NAME "SWITCH_CAPABILITY_TABLE" @@ -219,6 +238,7 @@ namespace swss { #define STATE_VXLAN_TABLE_NAME "VXLAN_TABLE" #define STATE_BGP_TABLE_NAME "BGP_STATE_TABLE" #define STATE_DEBUG_COUNTER_CAPABILITIES_NAME "DEBUG_COUNTER_CAPABILITIES" +#define STATE_NAT_RESTORE_TABLE_NAME "NAT_RESTORE_TABLE" /***** MISC *****/ diff --git a/configure.ac b/configure.ac index fe252abae..d99202352 100644 --- a/configure.ac +++ b/configure.ac @@ -13,7 +13,7 @@ AC_HEADER_STDC AM_PATH_PYTHON AC_CHECK_LIB([hiredis], [redisConnect]) -PKG_CHECK_MODULES([LIBNL], [libnl-3.0 libnl-genl-3.0 libnl-route-3.0]) +PKG_CHECK_MODULES([LIBNL], [libnl-3.0 libnl-genl-3.0 libnl-route-3.0 libnl-nf-3.0]) CFLAGS="$CFLAGS $LIBNL_CFLAGS" LIBS="$LIBS $LIBNL_LIBS" From 921de819338635ad6339d1edf7ef77d888dacb1d Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Tue, 15 Oct 2019 02:49:08 -0700 Subject: [PATCH 2/5] fix compilation errors after rebase --- common/nfnetlink.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/nfnetlink.h b/common/nfnetlink.h index 1245813a1..0970d99e3 100644 --- a/common/nfnetlink.h +++ b/common/nfnetlink.h @@ -17,7 +17,7 @@ class NfNetlink : public Selectable { void dumpRequest(int getCommand); int getFd() override; - void readData() override; + uint64_t readData() override; void updateConnTrackEntry(struct nfnl_ct *ct); void deleteConnTrackEntry(struct nfnl_ct *ct); From 8af5cc24ff27f7f7b604b6cfec617da46ab8fd3e Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Tue, 15 Oct 2019 02:50:16 -0700 Subject: [PATCH 3/5] fix compilation errors after rebase --- common/nfnetlink.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/nfnetlink.cpp b/common/nfnetlink.cpp index eeff840e7..d67113854 100644 --- a/common/nfnetlink.cpp +++ b/common/nfnetlink.cpp @@ -166,7 +166,7 @@ int NfNetlink::getFd() return nl_socket_get_fd(m_socket); } -void NfNetlink::readData() +uint64_t NfNetlink::readData() { int err; From 94201c86d30f1e842d7db1afbd3d695fe0f7c24f Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Tue, 15 Oct 2019 03:12:39 -0700 Subject: [PATCH 4/5] fix compilation errors after rebase --- common/nfnetlink.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/common/nfnetlink.cpp b/common/nfnetlink.cpp index d67113854..a0e4033a4 100644 --- a/common/nfnetlink.cpp +++ b/common/nfnetlink.cpp @@ -185,6 +185,7 @@ uint64_t NfNetlink::readData() else SWSS_LOG_ERROR("netlink reports an error=%d on reading a netfilter netlink socket", err); } + return 0; } int NfNetlink::onNetlinkMsg(struct nl_msg *msg, void *arg) From 8facdd42625f54cd1144b86ea95ed7e4636553cf Mon Sep 17 00:00:00 2001 From: Kiran Kella Date: Fri, 18 Oct 2019 03:12:24 -0700 Subject: [PATCH 5/5] Addressed review comments --- common/nfnetlink.cpp | 83 ++++++++++---------------------------------- common/nfnetlink.h | 12 +++---- 2 files changed, 25 insertions(+), 70 deletions(-) diff --git a/common/nfnetlink.cpp b/common/nfnetlink.cpp index a0e4033a4..0a8f5aaf3 100644 --- a/common/nfnetlink.cpp +++ b/common/nfnetlink.cpp @@ -58,7 +58,16 @@ NfNetlink::~NfNetlink() } } -void NfNetlink::registerRecvCallbacks(void) +bool NfNetlink::setSockBufSize(uint32_t sockBufSize) +{ + if (nl_socket_set_buffer_size(m_socket, sockBufSize, 0) < 0) + { + return false; + } + return true; +} + +void NfNetlink::registerRecvCallbacks() { #ifdef NETFILTER_UNIT_TEST nl_socket_modify_cb(m_socket, NL_CB_MSG_IN, NL_CB_CUSTOM, onNetlinkRcv, this); @@ -72,93 +81,39 @@ void NfNetlink::registerGroup(int nfnlGroup) int err = nl_socket_add_membership(m_socket, nfnlGroup); if (err < 0) { - SWSS_LOG_ERROR("Unable to register to netfilter group %d: %s", nfnlGroup, + SWSS_LOG_THROW("Unable to register to netfilter group %d: %s", nfnlGroup, nl_geterror(err)); - throw system_error(make_error_code(errc::address_not_available), - "Unable to register to netfilter group"); } } + void NfNetlink::dumpRequest(int getCommand) { int err = nfnl_ct_dump_request(m_socket); if (err < 0) { - SWSS_LOG_ERROR("Unable to request netfilter conntrack dump: %s", + SWSS_LOG_THROW("Unable to request netfilter conntrack dump: %s", nl_geterror(err)); - throw system_error(make_error_code(errc::address_not_available), - "Unable to request netfilter conntrack dump"); } } -void NfNetlink::updateConnTrackEntry(struct nfnl_ct *ct) +bool NfNetlink::updateConnTrackEntry(struct nfnl_ct *ct) { if (nfnl_ct_add(m_socket, ct, NLM_F_REPLACE) < 0) { SWSS_LOG_ERROR("Failed to update conntrack object in the kernel"); + return false; } + return true; } -void NfNetlink::deleteConnTrackEntry(struct nfnl_ct *ct) +bool NfNetlink::deleteConnTrackEntry(struct nfnl_ct *ct) { if (nfnl_ct_del(m_socket, ct, 0) < 0) { SWSS_LOG_ERROR("Failed to delete conntrack object in the kernel"); + return false; } -} - -struct nfnl_ct *NfNetlink::getCtObject(const IpAddress &sourceIpAddr) -{ - struct nfnl_ct *ct = nfnl_ct_alloc(); - struct nl_addr *orig_src_ip; - - if (! ct) - { - SWSS_LOG_ERROR("Unable to allocate memory for conntrack object"); - return NULL; - } - - nfnl_ct_set_family(ct, AF_INET); - if (nl_addr_parse(sourceIpAddr.to_string().c_str(), AF_INET, &orig_src_ip) == 0) - { - nfnl_ct_set_src(ct, 0, orig_src_ip); - } - else - { - SWSS_LOG_ERROR("Failed to parse orig src ip into conntrack object"); - nfnl_ct_put(ct); - return NULL; - } - - return ct; -} - -struct nfnl_ct *NfNetlink::getCtObject(uint8_t protoType, const IpAddress &sourceIpAddr, uint16_t srcPort) -{ - struct nfnl_ct *ct = nfnl_ct_alloc(); - struct nl_addr *orig_src_ip; - - if (! ct) - { - SWSS_LOG_ERROR("Unable to allocate memory for conntrack object"); - return NULL; - } - - nfnl_ct_set_family(ct, AF_INET); - nfnl_ct_set_proto(ct, protoType); - - if (nl_addr_parse(sourceIpAddr.to_string().c_str(), AF_INET, &orig_src_ip) == 0) - { - nfnl_ct_set_src(ct, 0, orig_src_ip); - } - else - { - SWSS_LOG_ERROR("Failed to parse orig src ip into conntrack object"); - nfnl_ct_put(ct); - return NULL; - } - nfnl_ct_set_src_port(ct, 0, srcPort); - - return ct; + return true; } int NfNetlink::getFd() diff --git a/common/nfnetlink.h b/common/nfnetlink.h index 0970d99e3..5e3c8fc11 100644 --- a/common/nfnetlink.h +++ b/common/nfnetlink.h @@ -12,21 +12,21 @@ class NfNetlink : public Selectable { NfNetlink(int pri = 0); ~NfNetlink() override; - void registerRecvCallbacks(void); + void registerRecvCallbacks(); + bool setSockBufSize(uint32_t sockBufSize); void registerGroup(int nfnlGroup); void dumpRequest(int getCommand); int getFd() override; uint64_t readData() override; - void updateConnTrackEntry(struct nfnl_ct *ct); - void deleteConnTrackEntry(struct nfnl_ct *ct); + bool updateConnTrackEntry(struct nfnl_ct *ct); + bool deleteConnTrackEntry(struct nfnl_ct *ct); private: - struct nfnl_ct *getCtObject(const IpAddress &sourceIpAddr); - struct nfnl_ct *getCtObject(uint8_t protoType, const IpAddress &sourceIpAddr, uint16_t srcPort); - +#ifdef NETFILTER_UNIT_TEST static int onNetlinkRcv(struct nl_msg *msg, void *arg); +#endif static int onNetlinkMsg(struct nl_msg *msg, void *arg); FILE *nfPktsLogFile;