Skip to content

Commit

Permalink
hetzner: Don't use localCommands, set interface device of default gat…
Browse files Browse the repository at this point in the history 
…eway. Should help #833.

Original localCommands patch by aszlig.
Fixups and interface device stuff from me.

Note that this new way of doing it without localCommands is backwards
incompatible with NixOS < 18.03, see:
https://nixos.org/nixos/manual/release-notes.html#sec-release-18.03-notable-changes

As described on

    https://wiki.hetzner.de/index.php/Netzkonfiguration_Debian/en#IPv4

Hetzner requires that even traffic to the local subnet goes via the gateway.

NixOps already had code for that (set in `localCommands`), but it wasn't enough:
The

    ip route replace default  via "GATEWAY_IP"  proto static

generated by `network-setup.service` in NixOS's `network-interface-scripted.nix`
fails with

    RTNETLINK answers: Network is unreachable

because all routes added so far had `via "GATEWAY_IP"`, but
the kernel didn't know how to actually reach GATEWAY_IP because
there was no static route for that IP, see.

    https://serverfault.com/questions/581159/unable-to-add-a-static-route-sioaddrt-network-is-unreachable/581162#581162

This commit adds an explicit static route to the default gateway, on the
relevant network device.

This allows us to generate the following instead:

    ip route replace GATEWAY_IP dev THE_DEVICE proto static
    ip route replace default  via "GATEWAY_IP" dev THE_DEVICE proto static

so that the kernel knows how to reach the gateway.

An example of what routes look like in `ip route` after deploying and 18.03
machine, where 1.2.3.4 is the server's IP and 1.2.3.1 is the gateway:

    default via 1.2.3.1 dev eth0                         src 1.2.3.4 metric 202
    1.2.3.0/27          dev eth0 proto kernel scope link src 1.2.3.4 metric 202
    1.2.3.1             dev eth0 proto static scope link

For IPv6, the link-local `fe80::1` is the gateway, as described on

    https://wiki.hetzner.de/index.php/Netzkonfiguration_Debian/en#IPv6

Signed-off-by: Niklas Hambüchen <mail@nh2.me>
  • Loading branch information
@nh2
nh2 committed Jun 20, 2020
1 parent c2f0b08 commit 188c312
Showing 1 changed file with 47 additions and 32 deletions.
79 changes: 47 additions & 32 deletions nixops/backends/hetzner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -538,9 +538,12 @@ def _get_default_gw(self):
"""
Return the default gateway of the currently running machine.
"""
cmd = "ip route list | sed -n -e 's/^default *via *//p'"
cmd += " | cut -d' ' -f1"
return self.run_command(cmd, capture_stdout=True).strip()
default_gw_cmd = "ip route list | sed -n -e 's/^default *via *//p'"
default_gw_output = self.run_command(default_gw_cmd, capture_stdout=True).strip()
default_gw_output_split = default_gw_output.split(' ')
gw_ip = default_gw_output_split[0]
gw_dev = default_gw_output_split[2]
return (gw_ip, gw_dev)

def _get_nameservers(self):
"""
Expand Down Expand Up @@ -573,13 +576,11 @@ def _gen_network_spec(self):
"""
udev_rules = []
iface_attrs = {}
extra_routes = []
ipv6_commands = []

server = self._get_server_by_ip(self.main_ipv4)

# Global networking options
defgw = self._get_default_gw()
defgw_ip, defgw_dev = self._get_default_gw()
v6defgw = None

# Interface-specific networking options
Expand All @@ -593,51 +594,65 @@ def _gen_network_spec(self):

udev_rules.append(self._get_udev_rule_for(iface))

ipv4, prefix = result
ipv4addr, prefix = result
iface_attrs[iface] = {
'ipAddress': ipv4,
'prefixLength': int(prefix),
'ipv4': {
'addresses': [
{'address': ipv4addr, 'prefixLength': int(prefix)},
],
},
}

# We can't handle Hetzner-specific networking info in test mode.
if TEST_MODE:
continue

# Extra route for accessing own subnet
net = self._calculate_ipv4_subnet(ipv4, int(prefix))
extra_routes.append(("{0}/{1}".format(net, prefix), defgw, iface))

# IPv6 subnets only for eth0 (XXX: more flexibility here?)
v6addr_command = "ip -6 addr add '{0}' dev '{1}' || true"
# Extra route for accessing own subnet for this interface
# (see https://wiki.hetzner.de/index.php/Netzkonfiguration_Debian/en#IPv4),
# but only if it's not the interface for the default gateway,
# because that one will already get such a route generated
# by NixOS's `network-setup.service`. See also:
# https://github.com/NixOS/nixops/pull/1032#issuecomment-433741624
if iface != defgw_dev:
net = self._calculate_ipv4_subnet(ipv4addr, int(prefix))
iface_attrs[iface]['ipv4'] = {
'routes': [{
'address': net,
'prefixLength': int(prefix),
'via': defgw_ip,
}],
}

# IPv6 subnets only for eth0
v6subnets = []
for subnet in server.subnets:
if "." in subnet.net_ip:
# skip IPv4 addresses
continue
v6addr = "{0}/{1}".format(subnet.net_ip, subnet.mask)
ipv6_commands.append(v6addr_command.format(v6addr, iface))
assert v6defgw is None or v6defgw == subnet.gateway
v6defgw = subnet.gateway

# Extra routes
route4_cmd = "ip -4 route change '{0}' via '{1}' dev '{2}' || true"
route_commands = [route4_cmd.format(network, gw, iface)
for network, gw, iface in extra_routes]

# IPv6 configuration
route6_cmd = "ip -6 route add default via '{0}' dev eth0 || true"
route_commands.append(route6_cmd.format(v6defgw))

local_commands = '\n'.join(ipv6_commands + route_commands) + '\n'
v6subnets.append({
'address': subnet.net_ip,
'prefixLength': int(subnet.mask)
})
assert (v6defgw is None or
v6defgw.get('address') == subnet.gateway)
v6defgw = {
'address': subnet.gateway,
'interface': defgw_dev,
}
iface_attrs[iface]['ipv6'] = { 'addresses': v6subnets }

self.net_info = {
'services': {
'udev': {'extraRules': '\n'.join(udev_rules) + '\n'},
},
'networking': {
'interfaces': iface_attrs,
'defaultGateway': defgw,
'defaultGateway': {
'address': defgw_ip,
'interface': defgw_dev,
},
'defaultGateway6': v6defgw,
'nameservers': self._get_nameservers(),
'localCommands': local_commands,
}
}

Expand Down

0 comments on commit 188c312

Please sign in to comment.