From 0bcd230661b3e6ba8d5f5705d41dff1b81b36b8f Mon Sep 17 00:00:00 2001 From: Aleksandr Lyapunov Date: Sun, 28 Aug 2022 16:24:46 +0300 Subject: [PATCH] Client: improve operation with iovec Allocate iovec simply on stack. Don't use NETWORK::readyToRecv anymore, use READAHEAD instead. Part of #28 --- src/Client/Connection.hpp | 41 ++++++--------------------------- src/Client/EpollNetProvider.hpp | 29 ++++++++++------------- src/Client/LibevNetProvider.hpp | 25 ++++++++++---------- 3 files changed, 32 insertions(+), 63 deletions(-) diff --git a/src/Client/Connection.hpp b/src/Client/Connection.hpp index fb19a5030..8d0135ea0 100644 --- a/src/Client/Connection.hpp +++ b/src/Client/Connection.hpp @@ -41,6 +41,9 @@ /** rid == request id */ typedef size_t rid_t; +static constexpr size_t CONN_READAHEAD = 64 * 1024; +static constexpr size_t IOVEC_MAX_SIZE = 32; + struct ConnectionError { std::string msg; //Saved in case connection fails due to system error. @@ -86,9 +89,6 @@ struct ConnectionImpl Greeting greeting; bool is_greeting_received; std::unordered_map> futures; - - static constexpr size_t AVAILABLE_IOVEC_COUNT = 32; - struct iovec m_IOVecs[AVAILABLE_IOVEC_COUNT]; }; template @@ -172,15 +172,7 @@ class Connection int getSocket() const; void setSocket(int socket); BUFFER& getInBuf(); - - template - friend - struct iovec * outBufferToIOV(Connection &conn, size_t *iov_len); - - template - friend - struct iovec * inBufferToIOV(Connection &conn, size_t size, - size_t *iov_len); + BUFFER& getOutBuf(); template friend @@ -446,29 +438,10 @@ Connection::getInBuf() } template -struct iovec * -inBufferToIOV(Connection &conn, size_t size, size_t *iov_len) -{ - assert(iov_len != NULL); - BUFFER &buf = conn.impl->inBuf; - struct iovec *vecs = conn.impl->m_IOVecs; - typename BUFFER::iterator itr = buf.end(); - buf.write({size}); - *iov_len = buf.getIOV(itr, vecs, - ConnectionImpl::AVAILABLE_IOVEC_COUNT); - return vecs; -} - -template -struct iovec * -outBufferToIOV(Connection &conn, size_t *iov_len) +BUFFER& +Connection::getOutBuf() { - assert(iov_len != NULL); - BUFFER &buf = conn.impl->outBuf; - struct iovec *vecs = conn.impl->m_IOVecs; - *iov_len = buf.getIOV(buf.begin(), buf.end(), vecs, - ConnectionImpl::AVAILABLE_IOVEC_COUNT); - return vecs; + return impl->outBuf; } template diff --git a/src/Client/EpollNetProvider.hpp b/src/Client/EpollNetProvider.hpp index e4d8a2d4b..49a10dcb5 100644 --- a/src/Client/EpollNetProvider.hpp +++ b/src/Client/EpollNetProvider.hpp @@ -189,21 +189,14 @@ template int EpollNetProvider::recv(Conn_t &conn) { - size_t total = NETWORK::readyToRecv(conn.getSocket()); - if (total < 0) { - LOG_ERROR("Failed to check socket: ioctl returned errno ", - strerror(errno)); - return -1; - } - if (total == 0) { - LOG_DEBUG("Socket ", conn.getSocket(), " has no data to read"); - return 0; - } - size_t iov_cnt = 0; - /* Get IO vectors array pointing to the input buffer. */ - struct iovec *iov = inBufferToIOV(conn, total, &iov_cnt); + auto &buf = conn.getInBuf(); + auto itr = buf.template end(); + buf.write({CONN_READAHEAD}); + struct iovec iov[IOVEC_MAX_SIZE]; + size_t iov_cnt = buf.getIOV(itr, iov, IOVEC_MAX_SIZE); + ssize_t rcvd = NETWORK::recvall(conn.getSocket(), iov, iov_cnt, true); - hasNotRecvBytes(conn, total - (rcvd < 0 ? 0 : rcvd)); + hasNotRecvBytes(conn, CONN_READAHEAD - (rcvd < 0 ? 0 : rcvd)); if (rcvd < 0) { //Don't consider EWOULDBLOCK to be an error. if (netWouldBlock(errno)) @@ -235,10 +228,12 @@ EpollNetProvider::send(Conn_t &conn) { while (hasDataToSend(conn)) { size_t sent_bytes = 0; - size_t iov_cnt = 0; - struct iovec *iov = outBufferToIOV(conn, &iov_cnt); + struct iovec iov[IOVEC_MAX_SIZE]; + auto &buf = conn.getOutBuf(); + size_t iov_cnt = buf.getIOV(buf.template begin(), + iov, IOVEC_MAX_SIZE); int rc = NETWORK::sendall(conn.getSocket(), iov, iov_cnt, - &sent_bytes); + &sent_bytes); hasSentBytes(conn, sent_bytes); if (rc != 0) { if (netWouldBlock(errno)) { diff --git a/src/Client/LibevNetProvider.hpp b/src/Client/LibevNetProvider.hpp index 0ec2e52b3..022c29bf6 100644 --- a/src/Client/LibevNetProvider.hpp +++ b/src/Client/LibevNetProvider.hpp @@ -120,17 +120,14 @@ template static inline int connectionReceive(Connection> &conn) { - size_t total = NETWORK::readyToRecv(conn.getSocket()); - if (total < 0) { - LOG_ERROR("Failed to check socket: ioctl returned errno ", - strerror(errno)); - return -1; - } - size_t iov_cnt = 0; - struct iovec *iov = - inBufferToIOV(conn, total, &iov_cnt); + auto &buf = conn.getInBuf(); + auto itr = buf.template end(); + buf.write({CONN_READAHEAD}); + struct iovec iov[IOVEC_MAX_SIZE]; + size_t iov_cnt = buf.getIOV(itr, iov, IOVEC_MAX_SIZE); + ssize_t rcvd = NETWORK::recvall(conn.getSocket(), iov, iov_cnt, true); - hasNotRecvBytes(conn, total - (rcvd < 0 ? 0 : rcvd)); + hasNotRecvBytes(conn, CONN_READAHEAD - (rcvd < 0 ? 0 : rcvd)); if (rcvd < 0) { if (netWouldBlock(errno)) { return 0; @@ -184,8 +181,12 @@ connectionSend(Connection> &conn) { while (hasDataToSend(conn)) { size_t sent_bytes = 0; - size_t iov_cnt = 0; - struct iovec *iov = outBufferToIOV(conn, &iov_cnt); + + struct iovec iov[IOVEC_MAX_SIZE]; + auto &buf = conn.getOutBuf(); + size_t iov_cnt = buf.getIOV(buf.template begin(), + iov, IOVEC_MAX_SIZE); + int rc = NETWORK::sendall(conn.getSocket(), iov, iov_cnt, &sent_bytes); hasSentBytes(conn, sent_bytes);