Merge pull request #799 from yggdrasil-network/develop

Version 0.4.0

.circleci/config.yml

@@ -16,6 +16,11 @@ jobs:
               go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.31.0
               golangci-lint run
 
+      - run:
+          name: Run Go tests
+          command: |
+              go test ./...
+
   build-linux:
     docker:
       - image: circleci/golang:1.16
@@ -157,43 +162,6 @@ jobs:
           paths:
             - upload
 
-  build-windows:
-    docker:
-      - image: circleci/golang:1.16
-
-    steps:
-      - checkout
-
-      - run:
-          name: Create artifact upload directory and set variables
-          command: |
-              mkdir /tmp/upload
-              echo 'export CINAME=$(sh contrib/semver/name.sh)' >> $BASH_ENV
-              echo 'export CIVERSION=$(sh contrib/semver/version.sh --bare)' >> $BASH_ENV
-              git config --global user.email "$(git log --format='%ae' HEAD -1)";
-              git config --global user.name "$(git log --format='%an' HEAD -1)";
-
-      - run:
-          name: Install tools
-          command: |
-              sudo apt-get update
-              sudo apt-get -y install msitools wixl
-
-      - run:
-          name: Build for Windows
-          command: |
-              rm -f {yggdrasil,yggdrasilctl}
-              GOOS=windows GOARCH=amd64 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-amd64.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-amd64.exe;
-              GOOS=windows GOARCH=386 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-i386.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-i386.exe;
-              bash contrib/msi/build-msi.sh x64
-              bash contrib/msi/build-msi.sh x86
-              mv *.msi /tmp/upload
-
-      - persist_to_workspace:
-          root: /tmp
-          paths:
-            - upload
-
   build-other:
     docker:
       - image: circleci/golang:1.16
@@ -224,6 +192,13 @@ jobs:
               GOOS=freebsd GOARCH=amd64 ./build && mv yggdrasil /tmp/upload/$CINAME-$CIVERSION-freebsd-amd64 && mv yggdrasilctl /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-freebsd-amd64;
               GOOS=freebsd GOARCH=386 ./build && mv yggdrasil /tmp/upload/$CINAME-$CIVERSION-freebsd-i386 && mv yggdrasilctl /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-freebsd-i386;
 
+      - run:
+          name: Build for Windows
+          command: |
+              rm -f {yggdrasil,yggdrasilctl}
+              GOOS=windows GOARCH=amd64 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-amd64.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-amd64.exe;
+              GOOS=windows GOARCH=386 ./build && mv yggdrasil.exe /tmp/upload/$CINAME-$CIVERSION-windows-i386.exe && mv yggdrasilctl.exe /tmp/upload/$CINAME-$CIVERSION-yggdrasilctl-windows-i386.exe;
+
       - persist_to_workspace:
           root: /tmp
           paths:
@@ -247,11 +222,9 @@ workflows:
       - lint
       - build-linux
       - build-macos
-      - build-windows
       - build-other
       - upload:
           requires:
             - build-linux
             - build-macos
-            - build-windows
             - build-other

CHANGELOG.md

@@ -25,6 +25,58 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - in case of vulnerabilities.
 -->
 
+## [0.4.0] - 2021-07-04
+### Added
+- New routing scheme, which is backwards incompatible with previous versions of Yggdrasil
+    - The wire protocol version number, exchanged as part of the peer setup handshake, has been increased to 0.4
+    - Nodes running this new version **will not** be able to peer with earlier versions of Yggdrasil
+    - Please note that **the network may be temporarily unstable** while infrastructure is being upgraded to the new release
+- TLS connections now use public key pinning
+    - If no public key was already pinned, then the public key received as part of the TLS handshake is pinned to the connection
+    - The public key received as part of the handshake is checked against the pinned keys, and if no match is found, the connection is rejected
+
+### Changed
+- IP addresses are now derived from ed25519 public (signing) keys
+    - Previously, addresses were derived from a hash of X25519 (Diffie-Hellman) keys
+    - Importantly, this means that **all internal IPv6 addresses will change with this release** — this will affect anyone running public services or relying on Yggdrasil for remote access
+- It is now recommended to peer over TLS
+    - Link-local peers from multicast peer discovery will now connect over TLS, with the key from the multicast beacon pinned to the connection
+    - `socks://` peers now expect the destination endpoint to be a `tls://` listener, instead of a `tcp://` listener
+- Multicast peer discovery is now more configurable
+    - There are separate configuration options to control if beacons are sent, what port to listen on for incoming connections (if sending beacons), and whether or not to listen for beacons from other nodes (and open connections when receiving a beacon)
+    - Each configuration entry in the list specifies a regular expression to match against interface names
+    - If an interface matches multiple regex in the list, it will use the settings for the first entry in the list that it matches with
+- The session and routing code has been entirely redesigned and rewritten
+    - This is still an early work-in-progress, so the code hasn't been as well tested or optimized as the old code base — please bear with us for these next few releases as we work through any bugs or issues
+    - Generally speaking, we expect to see reduced bandwidth use and improved reliability with the new design, especially in cases where nodes move around or change peerings frequently
+    - Cryptographic sessions no longer use a single shared (ephemeral) secret for the entire life of the session. Keys are now rotated regularly for ongoing sessions (currently rotated at least once per round trip exchange of traffic, subject to change in future releases)
+    - Source routing has been added. Under normal circumstances, this is what is used to forward session traffic (e.g. the user's IPv6 traffic)
+    - DHT-based routing has been added. This is used when the sender does not know a source route to the destination. Forwarding through the DHT is less efficient, but the only information that it requires the sender to know is the destination node's (static) key. This is primarily used during the key exchange at session setup, or as a temporary fallback when a source route fails due to changes in the network
+    - The new DHT design is no longer RPC-based, does not support crawling and does not inherently allow nodes to look up the owner of an arbitrary key. Responding to lookups is now implemented at the application level and a response is only sent if the destination key matches the node's `/128` IP or `/64` prefix
+    - The greedy routing scheme, used to forward all traffic in previous releases, is now only used for protocol traffic (i.e. DHT setup and source route discovery)
+    - The routing logic now lives in a [standalone library](https://github.com/Arceliar/ironwood). You are encouraged **not** to use it, as it's still considered pre-alpha, but it's available for those who want to experiment with the new routing algorithm in other contexts
+    - Session MTUs may be slightly lower now, in order to accommodate large packet headers if required
+- Many of the admin functions available over `yggdrasilctl` have been changed or removed as part of rewrites to the code
+    - Several remote `debug` functions have been added temporarily, to allow for crawling and census gathering during the transition to the new version, but we intend to remove this at some point in the (possibly distant) future
+    - The list of available functions will likely be expanded in future releases
+- The configuration file format has been updated in response to the changed/removed features
+
+### Removed
+- Tunnel routing (a.k.a. crypto-key routing or "CKR") has been removed
+    - It was far too easy to accidentally break routing altogether by capturing the route to peers with the TUN adapter
+    - We recommend tunnelling an existing standard over Yggdrasil instead (e.g. `ip6gre`, `ip6gretap` or other similar encapsulations, using Yggdrasil IPv6 addresses as the tunnel endpoints)
+    - All `TunnelRouting` configuration options will no longer take effect
+- Session firewall has been removed
+    - This was never a true firewall — it didn't behave like a stateful IP firewall, often allowed return traffic unexpectedly and was simply a way to prevent a node from being flooded with unwanted sessions, so the name could be misleading and usually lead to a false sense of security
+    - Due to design changes, the new code needs to address the possible memory exhaustion attacks in other ways and a single configurable list no longer makes sense
+    - Users who want a firewall or other packet filter mechansim should configure something supported by their OS instead (e.g. `ip6tables`)
+    - All `SessionFirewall` configuration options will no longer take effect
+- `SIGHUP` handling to reload the configuration at runtime has been removed
+    - It was not obvious which parts of the configuration could be reloaded at runtime, and which required the application to be killed and restarted to take effect
+    - Reloading the config without restarting was also a delicate and bug-prone process, and was distracting from more important developments
+    - `SIGHUP` will be handled normally (i.e. by exiting)
+- `cmd/yggrasilsim` has been removed, and is unlikely to return to this repository
+
 ## [0.3.16] - 2021-03-18
 ### Added
 - New simulation code under `cmd/yggdrasilsim` (work-in-progress)

README.md

@@ -11,37 +11,14 @@ allows pretty much any IPv6-capable application to communicate securely with
 other Yggdrasil nodes. Yggdrasil does not require you to have IPv6 Internet
 connectivity - it also works over IPv4.
 
-Although Yggdrasil shares many similarities with
-[cjdns](https://github.com/cjdelisle/cjdns), it employs a different routing
-algorithm based on a globally-agreed spanning tree and greedy routing in a
-metric space, and aims to implement some novel local backpressure routing
-techniques. In theory, Yggdrasil should scale well on networks with
-internet-like topologies.
-
 ## Supported Platforms
 
-We actively support the following platforms, and packages are available for
-some of the below:
-
-- Linux
-  - `.deb` and `.rpm` packages are built by CI for Debian and Red Hat-based
-    distributions
-  - Arch, Nix, Void packages also available within their respective repositories
-- macOS
-  - `.pkg` packages are built by CI
-- Ubiquiti EdgeOS
-  - `.deb` Vyatta packages are built by CI
-- Windows
-- FreeBSD
-- OpenBSD
-- OpenWrt
-
-Please see our [Platforms](https://yggdrasil-network.github.io/platforms.html) pages for more
-specific information about each of our supported platforms, including
-installation steps and caveats.
-
-You may also find other platform-specific wrappers, scripts or tools in the
-`contrib` folder.
+Yggdrasil works on a number of platforms, including Linux, macOS, Ubiquiti
+EdgeRouter, VyOS, Windows, FreeBSD, OpenBSD and OpenWrt.
+
+Please see our [Installation](https://yggdrasil-network.github.io/installation.html) 
+page for more information. You may also find other platform-specific wrappers, scripts
+or tools in the `contrib` folder.
 
 ## Building
 
@@ -97,21 +74,18 @@ by giving the Yggdrasil binary the `CAP_NET_ADMIN` capability.
 
 ## Documentation
 
-Documentation is available on our [GitHub
-Pages](https://yggdrasil-network.github.io) site, or in the base submodule
-repository within `doc/yggdrasil-network.github.io`.
+Documentation is available [on our website](https://yggdrasil-network.github.io).
 
-- [Configuration file options](https://yggdrasil-network.github.io/configuration.html)
-- [Platform-specific documentation](https://yggdrasil-network.github.io/platforms.html)
+- [Installing Yggdrasil](https://yggdrasil-network.github.io/installation.html)
+- [Configuring Yggdrasil](https://yggdrasil-network.github.io/configuration.html)
 - [Frequently asked questions](https://yggdrasil-network.github.io/faq.html)
-- [Admin API documentation](https://yggdrasil-network.github.io/admin.html)
 - [Version changelog](CHANGELOG.md)
 
 ## Community
 
 Feel free to join us on our [Matrix
 channel](https://matrix.to/#/#yggdrasil:matrix.org) at `#yggdrasil:matrix.org`
-or in the `#yggdrasil` IRC channel on Freenode.
+or in the `#yggdrasil` IRC channel on [libera.chat](https://libera.chat).
 
 ## License
 

appveyor.yml

@@ -0,0 +1,20 @@
+version: '{build}'
+pull_requests:
+  do_not_increment_build_number: true
+os: Visual Studio 2019
+shallow_clone: false
+
+environment:
+  MSYS2_PATH_TYPE: inherit
+  CHERE_INVOKING: enabled_from_arguments
+
+build_script:
+- cmd: >-
+    cd %APPVEYOR_BUILD_FOLDER%
+- c:\msys64\usr\bin\bash -lc "./contrib/msi/build-msi.sh x64"
+- c:\msys64\usr\bin\bash -lc "./contrib/msi/build-msi.sh x86"
+
+test: off
+
+artifacts:
+- path: '*.msi'

build

@@ -32,18 +32,16 @@ fi
 
 if [ $IOS ]; then
   echo "Building framework for iOS"
-  gomobile bind -target ios -tags mobile -ldflags="$LDFLAGS $STRIP" -gcflags="$GCFLAGS" \
-    github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil \
-    github.com/yggdrasil-network/yggdrasil-go/src/config \
+  go get golang.org/x/mobile/bind
+  gomobile bind -target ios -tags mobile -o Yggdrasil.framework -ldflags="$LDFLAGS $STRIP" -gcflags="$GCFLAGS" \
     github.com/yggdrasil-network/yggdrasil-extras/src/mobile \
-    github.com/yggdrasil-network/yggdrasil-extras/src/dummy
+    github.com/yggdrasil-network/yggdrasil-go/src/config
 elif [ $ANDROID ]; then
   echo "Building aar for Android"
-  gomobile bind -target android -tags mobile -ldflags="$LDFLAGS $STRIP" -gcflags="$GCFLAGS" \
-    github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil \
-    github.com/yggdrasil-network/yggdrasil-go/src/config \
+  go get golang.org/x/mobile/bind
+  gomobile bind -target android -tags mobile -o yggdrasil.aar -ldflags="$LDFLAGS $STRIP" -gcflags="$GCFLAGS" \
     github.com/yggdrasil-network/yggdrasil-extras/src/mobile \
-    github.com/yggdrasil-network/yggdrasil-extras/src/dummy
+    github.com/yggdrasil-network/yggdrasil-go/src/config
 else
   for CMD in yggdrasil yggdrasilctl ; do
     echo "Building: $CMD"