Merge branch 'release-6.0.8' into 'master'

Release 6.0.8

See merge request knot/knot-resolver!1572

NEWS

@@ -1,44 +1,70 @@
-Knot Resolver 6.0.8 (2024-0m-dd)
+Knot Resolver 6.0.8 (2024-07-23)
 ================================
 
+Security
+--------
+- reduce buffering of transmitted data, especially TCP-based in userspace
+  Also expose some of the new tweaks in lua:
+   (require 'ffi').C.the_worker.engine.net.tcp.user_timeout = 1000
+   (require 'ffi').C.the_worker.engine.net.listen_{tcp,udp}_buflens.{snd,rcv}
+
+Packaging
+---------
+
+- all packages:
+	- remove unused dependency on `libedit` (!1553)
+- deb packages:
+	- packages ``knot-resolver-core`` and ``knot-resolver-manager`` have
+	  been merged into a single ``knot-resolver6`` package. Suffix packages
+	  ``knot-resolver-*`` have been renamed to ``knot-resolver6-*``. This
+	  change _should_ be transparent, but please do let us know if you
+	  encounter any issues while updating. (!1549)
+	- package ``python3-prometheus-client`` is now only an optional dependency
+- rpm packages:
+	- packages ``knot-resolver-core`` and ``knot-resolver-manager`` have
+	  been merged into a single ``knot-resolver`` package. This change
+	  _should_ be transparent, but please do let us know if you encounter
+	  any issues while updating. (!1549)
+	- bugfix: do not overwrite config.yaml (!1525)
+	- package ``python3-prometheus_client`` is now only an optional dependency
+- arch package:
+	- fix after they renamed a dependency (!1536)
+
 Improvements
 ------------
 - TLS (DoT, DoH): respect crypto policy overrides in OS (!1526)
-- arch package: fix after they renamed a dependency (!1536)
 - manager: export metrics to JSON via management HTTP API (!1527)
-
   * JSON is the new default metrics output format
   * the ``prometheus-client`` Python package is now an optional dependency,
     required only for Prometheus export to work
 - cache: prefetching records
-
   * predict module: prefetching expiring records moved to prefetch module
   * prefetch module: new module to prefetch expiring records
 - stats: add separate metrics for IPv6 and IPv4 (!1545)
-- remove unused dependency on `libedit` (!1553)
 - add the fresh DNSSEC root key "KSK-2024" already, Key ID 38696 (!1556)
-
-- manager: policy-loader: new component for separate loading policy rules (!1540)
-
+- manager: policy-loader: new component for separate loading of policy rules (!1540)
   The ``policy-loader`` ensures that configured policies are loaded into the rules database
   where they are made available to all running kresd workers. This loading is no longer done
   by all kresd workers as it was before, so this should significantly improve the resolver's
   startup/reload time when loading large sets of policy rules, e.g. large RPZs.
 
-.. TODO: Change the link below to a versioned one when releasing.
-
 Incompatible changes
 --------------------
 - cache: the ``cache.prediction`` configuration property has been reorganized
   into ``cache.prefetch.expiring`` and ``cache.prefetch.prediction``, changing
   the default behaviour as well. See the `relevant documentation section
-  <https://www.knot-resolver.cz/documentation/latest/config-cache-predict.html>`_
+  <https://www.knot-resolver.cz/documentation/v6.0.8/config-cache-predict.html>`_
   for more.
 - libknot <=3.2.x support is dropped (!1565)
 
 Bugfixes
 --------
+- arch package: fix after they renamed a dependency (!1536)
 - fix startup with `dnssec: false` (!1548)
+- rpm packages: do not overwrite config.yaml (!1525)
+- fix NSEC3 records missing in answer for positive wildcard expansion
+  with the NSEC3 having over-limit iteration count (#910, !1550)
+- views: fix a bug in subnet matching (!1562)
 
 
 Knot Resolver 6.0.7 (2024-03-27)
@@ -115,9 +141,16 @@ https://www.knot-resolver.cz/documentation/latest/upgrading-to-6.html
 5.x branch longterm support
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Knot Resolver 5.7.4 (2024-0m-dd)
+Knot Resolver 5.7.4 (2024-07-23)
 ================================
 
+Security
+--------
+- reduce buffering of transmitted data, especially TCP-based in userspace
+  Also expose some of the new tweaks in lua:
+   (require 'ffi').C.the_worker.engine.net.tcp.user_timeout = 1000
+   (require 'ffi').C.the_worker.engine.net.listen_{tcp,udp}_buflens.{snd,rcv}
+
 Improvements
 ------------
 - add the fresh DNSSEC root key "KSK-2024" already, Key ID 38696 (!1556)

daemon/bindings/net_buffering.rst

@@ -0,0 +1,25 @@
+.. SPDX-License-Identifier: GPL-3.0-or-later
+
+Buffering tweaks
+----------------
+
+We (can) set various server-side socket options that affect buffering.
+The values are stored in C structures without real Lua bindings,
+so setting them is a bit long.
+
+.. py:data:: (require 'ffi').C.the_worker.engine.net.tcp.user_timeout
+ 
+   On TCP-based server-side sockets we set ``TCP_USER_TIMEOUT`` option if available (~Linux).
+   We use default 1000, i.e. one second.  For details see the definition in ``man tcp.7``.
+
+.. py:data:: (require 'ffi').C.the_worker.engine.net.listen_tcp_buflens.snd
+.. py:data:: (require 'ffi').C.the_worker.engine.net.listen_tcp_buflens.rcv
+.. py:data:: (require 'ffi').C.the_worker.engine.net.listen_udp_buflens.snd
+.. py:data:: (require 'ffi').C.the_worker.engine.net.listen_udp_buflens.rcv
+
+   If overridden to nonzero, these variables instruct the OS to modify kernel-space buffers
+   for server-side sockets.  We split the setting for UDP vs. TCP and sending vs. receiving.
+
+   For details see ``SO_SNDBUF`` and ``SO_RCVBUF`` in ``man socket.7``.
+   There is no user-space buffering beyond immediate manipulation, only the OS keeps some.
+

daemon/io.c

@@ -165,15 +165,20 @@ static enum protolayer_event_cb_result pl_tcp_event_wrap(
 		enum protolayer_event_type event, void **baton,
 		struct session2 *session, void *sess_data)
 {
-	if (event == PROTOLAYER_EVENT_STATS_SEND_ERR) {
+	switch (event) {
+	case PROTOLAYER_EVENT_STATS_SEND_ERR:
 		the_worker->stats.err_tcp += 1;
 		return PROTOLAYER_EVENT_CONSUME;
-	} else if (event == PROTOLAYER_EVENT_STATS_QRY_OUT) {
+	case PROTOLAYER_EVENT_STATS_QRY_OUT:
 		the_worker->stats.tcp += 1;
 		return PROTOLAYER_EVENT_CONSUME;
+	case PROTOLAYER_EVENT_OS_BUFFER_FULL:
+		session2_force_close(session);
+		return PROTOLAYER_EVENT_CONSUME;
+	default:
+		return PROTOLAYER_EVENT_PROPAGATE;
 	}
 
-	return PROTOLAYER_EVENT_PROPAGATE;
 }
 
 __attribute__((constructor))
@@ -261,6 +266,17 @@ int io_bind(const struct sockaddr *addr, int type, const endpoint_flags_t *flags
 	return fd;
 }
 
+/// Optionally set a socket option and log error on failure.
+static void set_so(int fd, int so_option, int value, const char *descr)
+{
+	if (!value) return;
+	if (setsockopt(fd, SOL_SOCKET, so_option, &value, sizeof(value))) {
+		kr_log_error(IO, "failed to set %s to %d: %s\n",
+				descr, value, strerror(errno));
+		// we treat this as non-critical failure
+	}
+}
+
 int io_listen_udp(uv_loop_t *loop, uv_udp_t *handle, int fd)
 {
 	if (!handle) {
@@ -272,6 +288,9 @@ int io_listen_udp(uv_loop_t *loop, uv_udp_t *handle, int fd)
 	ret = uv_udp_open(handle, fd);
 	if (ret) return ret;
 
+	set_so(fd, SO_SNDBUF, the_network->listen_udp_buflens.snd, "UDP send buffer size");
+	set_so(fd, SO_RCVBUF, the_network->listen_udp_buflens.rcv, "UDP receive buffer size");
+
 	uv_handle_t *h = (uv_handle_t *)handle;
 	check_bufsize(h);
 	/* Handle is already created, just create context. */
@@ -314,7 +333,7 @@ static void tcp_recv(uv_stream_t *handle, ssize_t nread, const uv_buf_t *buf)
 				       uv_strerror(nread));
 		}
 		session2_penalize(s);
-		worker_end_tcp(s);
+		session2_force_close(s);
 		return;
 	}
 
@@ -464,6 +483,17 @@ int io_listen_tcp(uv_loop_t *loop, uv_tcp_t *handle, int fd, int tcp_backlog, bo
 	}
 #endif
 
+	/* These get inherited into the individual connections (on Linux at least). */
+	set_so(fd, SO_SNDBUF, the_network->listen_tcp_buflens.snd, "TCP send buffer size");
+	set_so(fd, SO_RCVBUF, the_network->listen_tcp_buflens.rcv, "TCP receive buffer size");
+#ifdef TCP_USER_TIMEOUT
+	val = the_network->tcp.user_timeout;
+	if (val && setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, &val, sizeof(val))) {
+		kr_log_error(IO, "listen TCP (user_timeout): %s\n", strerror(errno));
+	}
+	// TODO: also for upstream connections, at least this one option?
+#endif
+
 	handle->data = NULL;
 	return 0;
 }

daemon/lua/kres-gen-33.lua

@@ -509,6 +509,7 @@ int kr_rule_zonefile(const struct kr_rule_zonefile_config *);
 int kr_rule_forward(const knot_dname_t *, kr_rule_fwd_flags_t, const struct sockaddr **);
 int kr_rule_local_address(const char *, const char *, _Bool, uint32_t, kr_rule_tags_t);
 int kr_rule_local_hosts(const char *, _Bool, uint32_t, kr_rule_tags_t);
+struct tls_credentials;
 typedef struct {
 	int sock_type;
 	_Bool tls;
@@ -559,6 +560,36 @@ typedef struct {
 	zi_callback cb;
 	void *cb_param;
 } zi_config_t;
+typedef struct uv_loop_s uv_loop_t;
+typedef struct trie tls_client_params_t;
+struct net_tcp_param {
+	uint64_t in_idle_timeout;
+	uint64_t tls_handshake_timeout;
+	unsigned int user_timeout;
+};
+struct network {
+	uv_loop_t *loop;
+	trie_t *endpoints;
+	trie_t *endpoint_kinds;
+	_Bool missing_kind_is_error : 1;
+	_Bool proxy_all4 : 1;
+	_Bool proxy_all6 : 1;
+	trie_t *proxy_addrs4;
+	trie_t *proxy_addrs6;
+	struct tls_credentials *tls_credentials;
+	tls_client_params_t *tls_client_params;
+	struct tls_session_ticket_ctx *tls_session_ticket_ctx;
+	struct net_tcp_param tcp;
+	int tcp_backlog;
+	struct {
+		int snd;
+		int rcv;
+	} listen_udp_buflens;
+	struct {
+		int snd;
+		int rcv;
+	} listen_tcp_buflens;
+};
 struct args *the_args;
 struct endpoint {
 	void *handle;
@@ -591,6 +622,7 @@ struct worker_ctx {
 struct kr_context *the_resolver;
 struct worker_ctx *the_worker;
 struct engine *the_engine;
+struct network *the_network;
 typedef struct {
 	uint8_t *params_position;
 	uint8_t *mandatory_position;

daemon/lua/kres-gen.sh

@@ -312,6 +312,7 @@ EOF
 
 
 ## kresd itself: worker stuff
+echo "struct tls_credentials;"
 
 ${CDEFS} ${KRESD} types <<-EOF
 	endpoint_flags_t
@@ -322,6 +323,11 @@ ${CDEFS} ${KRESD} types <<-EOF
 	config_array_t
 	struct args
 	zi_config_t
+	# struct network - and all requirements that are missing so far
+	typedef uv_loop_t
+	typedef tls_client_params_t
+	struct net_tcp_param
+	struct network
 EOF
 echo "struct args *the_args;"
 
@@ -349,6 +355,7 @@ printf "\tchar _stub[];\n};\n"
 echo "struct kr_context *the_resolver;"
 echo "struct worker_ctx *the_worker;"
 echo "struct engine *the_engine;"
+echo "struct network *the_network;"
 
 
 ## libzscanner API for ./zonefile.lua

daemon/network.c

@@ -76,7 +76,12 @@ void network_init(uv_loop_t *loop, int tcp_backlog)
 			tls_session_ticket_ctx_create(loop, NULL, 0);
 	the_network->tcp.in_idle_timeout = 10000;
 	the_network->tcp.tls_handshake_timeout = TLS_MAX_HANDSHAKE_TIME;
+	the_network->tcp.user_timeout = 1000; // 1s should be more than enough
 	the_network->tcp_backlog = tcp_backlog;
+
+	// On Linux, unset means some auto-tuning mechanism also depending on RAM,
+	// which might be OK default (together with the user_timeout above)
+	//the_network->listen_{tcp,udp}_buflens.{snd,rcv}
 }
 
 /** Notify the registered function about endpoint getting open.

daemon/network.h

@@ -67,6 +67,10 @@ typedef array_t(struct endpoint) endpoint_array_t;
 struct net_tcp_param {
 	uint64_t in_idle_timeout;
 	uint64_t tls_handshake_timeout;
+
+	/** Milliseconds of unacknowledged data; see TCP_USER_TIMEOUT in man tcp.7
+	 * Linux only, probably. */
+	unsigned int user_timeout;
 };
 
 /** Information about an address that is allowed to use PROXYv2. */
@@ -104,6 +108,13 @@ struct network {
 	struct tls_session_ticket_ctx *tls_session_ticket_ctx;
 	struct net_tcp_param tcp;
 	int tcp_backlog;
+
+	/** Kernel-side buffer sizes for sending and receiving. (in bytes)
+	 * They are per socket, so in the TCP case they are per connection.
+	 * See SO_SNDBUF and SO_RCVBUF in man socket.7  These are in POSIX. */
+	struct {
+		int snd, rcv;
+	} listen_udp_buflens, listen_tcp_buflens;
 };
 
 /** Pointer to the singleton network state. NULL if not initialized. */

daemon/proxyv2.c

@@ -407,7 +407,7 @@ static enum protolayer_iter_cb_result pl_proxyv2_stream_unwrap(
 						"for this peer, close\n",
 						kr_straddr(peer));
 			}
-			worker_end_tcp(s);
+			session2_force_close(s);
 			return protolayer_break(ctx, kr_error(ECONNRESET));
 		}
 
@@ -424,7 +424,7 @@ static enum protolayer_iter_cb_result pl_proxyv2_stream_unwrap(
 							kr_straddr(comm->src_addr));
 				}
 			}
-			worker_end_tcp(s);
+			session2_force_close(s);
 			return protolayer_break(ctx, kr_error(ECONNRESET));
 		} else if (trimmed == 0) {
 			session2_close(s);