BREAKING CHANGE: fix: pin certificates to hosts

Before writing this diff, netem automatically generated the correct
cert on the fly based on the given SNI or local addr.

While this behavior has been okay so far, it turns out there is a
set of tests we cannot write because of it.

Namely, we cannot check whether we can connect to a given host
using another SNI, because netem would generate a certificate
for the provided SNI, which is not how the internet works.

If I'm connecting to www.example.com with www.google.com as the
SNI, in the internet the server would return a valid certificate
for www.example.com, rather than for www.google.com.

This diff rectifies netem's behavior by forcing the user to pin
the certificate to a set of names and addresses when creating the
server that needs such a certificate.

Accordingly, we can stop using a forked google/martian/v3/mitm
implementation and we can just roll out our own, which is still
based on martian (feat not, not reinventing the wheel here)
but allows us to create a certificate with specific addresses
and domain names pinned to it.

While there, notice that there was code we were not using,
such as stdlib.go, and that we also don't need in ooni/probe-cli,
so we can definitely kill this piece of code.

Part of ooni/probe#2531, because the
tests I was trying to write belong to such an issue.

ca.go

@@ -0,0 +1,210 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package netem
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha1"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"math/big"
+	"net"
+	"time"
+)
+
+// caMaxSerialNumber is the upper boundary that is used to create unique serial
+// numbers for the certificate. This can be any unsigned integer up to 20
+// bytes (2^(8*20)-1).
+var caMaxSerialNumber = big.NewInt(0).SetBytes(bytes.Repeat([]byte{255}, 20))
+
+// caMustNewAuthority creates a new CA certificate and associated private key or PANICS.
+//
+// This code is derived from github.com/google/martian/v3.
+//
+// SPDX-License-Identifier: Apache-2.0.
+func caMustNewAuthority(name, organization string, validity time.Duration,
+	timeNow func() time.Time) (*x509.Certificate, *rsa.PrivateKey) {
+	priv := Must1(rsa.GenerateKey(rand.Reader, 2048))
+	pub := priv.Public()
+
+	// Subject Key Identifier support for end entity certificate.
+	// https://www.ietf.org/rfc/rfc3280.txt (section 4.2.1.2)
+	pkixpub := Must1(x509.MarshalPKIXPublicKey(pub))
+	h := sha1.New()
+	h.Write(pkixpub)
+	keyID := h.Sum(nil)
+
+	// TODO(bassosimone): keep a map of used serial numbers to avoid potentially
+	// reusing a serial multiple times.
+	serial := Must1(rand.Int(rand.Reader, caMaxSerialNumber))
+
+	tmpl := &x509.Certificate{
+		SerialNumber: serial,
+		Subject: pkix.Name{
+			CommonName:   name,
+			Organization: []string{organization},
+		},
+		SubjectKeyId:          keyID,
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		NotBefore:             timeNow().Add(-validity),
+		NotAfter:              timeNow().Add(validity),
+		DNSNames:              []string{name},
+		IsCA:                  true,
+	}
+
+	raw := Must1(x509.CreateCertificate(rand.Reader, tmpl, tmpl, pub, priv))
+
+	// Parse certificate bytes so that we have a leaf certificate.
+	x509c := Must1(x509.ParseCertificate(raw))
+
+	return x509c, priv
+}
+
+// CA is a certification authority.
+//
+// The zero value is invalid, please use [NewCA] to construct.
+//
+// This code is derived from github.com/google/martian/v3.
+//
+// SPDX-License-Identifier: Apache-2.0.
+type CA struct {
+	ca       *x509.Certificate
+	capriv   any
+	keyID    []byte
+	org      string
+	priv     *rsa.PrivateKey
+	validity time.Duration
+}
+
+// NewCA creates a new certification authority.
+func MustNewCA() *CA {
+	return MustNewCAWithTimeNow(time.Now)
+}
+
+// MustNewCA is like [NewCA] but uses a custom [time.Now] func.
+//
+// This code is derived from github.com/google/martian/v3.
+//
+// SPDX-License-Identifier: Apache-2.0.
+func MustNewCAWithTimeNow(timeNow func() time.Time) *CA {
+	ca, privateKey := caMustNewAuthority("jafar", "OONI", 24*time.Hour, timeNow)
+
+	roots := x509.NewCertPool()
+	roots.AddCert(ca)
+
+	priv := Must1(rsa.GenerateKey(rand.Reader, 2048))
+	pub := priv.Public()
+
+	// Subject Key Identifier support for end entity certificate.
+	// https://www.ietf.org/rfc/rfc3280.txt (section 4.2.1.2)
+	pkixpub := Must1(x509.MarshalPKIXPublicKey(pub))
+	h := sha1.New()
+	h.Write(pkixpub)
+	keyID := h.Sum(nil)
+
+	return &CA{
+		ca:       ca,
+		capriv:   privateKey,
+		priv:     priv,
+		keyID:    keyID,
+		validity: time.Hour,
+		org:      "OONI Netem CA",
+	}
+}
+
+// CertPool returns an [x509.CertPool] using the given [*CA].
+func (c *CA) CertPool() *x509.CertPool {
+	pool := x509.NewCertPool()
+	pool.AddCert(c.ca)
+	return pool
+}
+
+// MustNewCert creates a new certificate for the given common name or PANICS.
+//
+// The common name and the extra names could contain domain names or IP addresses.
+//
+// For example:
+//
+// - www.example.com
+//
+// - 10.0.0.1
+//
+// - ::1
+//
+// are all valid values you can pass as common name or extra names.
+func (c *CA) MustNewCert(commonName string, extraNames ...string) *tls.Certificate {
+	return c.MustNewCertWithTimeNow(time.Now, commonName, extraNames...)
+}
+
+// MustNewCertWithTimeNow is like [MustNewCert] but uses a custom [time.Now] func.
+//
+// This code is derived from github.com/google/martian/v3.
+//
+// SPDX-License-Identifier: Apache-2.0.
+func (c *CA) MustNewCertWithTimeNow(timeNow func() time.Time, commonName string, extraNames ...string) *tls.Certificate {
+	serial := Must1(rand.Int(rand.Reader, caMaxSerialNumber))
+
+	tmpl := &x509.Certificate{
+		SerialNumber: serial,
+		Subject: pkix.Name{
+			CommonName:   commonName,
+			Organization: []string{c.org},
+		},
+		SubjectKeyId:          c.keyID,
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		NotBefore:             timeNow().Add(-c.validity),
+		NotAfter:              timeNow().Add(c.validity),
+	}
+
+	allNames := []string{commonName}
+	allNames = append(allNames, extraNames...)
+	for _, name := range allNames {
+		if ip := net.ParseIP(name); ip != nil {
+			tmpl.IPAddresses = append(tmpl.IPAddresses, ip)
+		} else {
+			tmpl.DNSNames = append(tmpl.DNSNames, name)
+		}
+	}
+
+	raw := Must1(x509.CreateCertificate(rand.Reader, tmpl, c.ca, c.priv.Public(), c.capriv))
+
+	// Parse certificate bytes so that we have a leaf certificate.
+	x509c := Must1(x509.ParseCertificate(raw))
+
+	tlsc := &tls.Certificate{
+		Certificate: [][]byte{raw, c.ca.Raw},
+		PrivateKey:  c.priv,
+		Leaf:        x509c,
+	}
+
+	return tlsc
+}
+
+// MustServerTLSConfig generates a server-side [*tls.Config] that uses the given [*CA] and
+// a generated certificate for the given common name and extra names.
+//
+// See [CA.MustNewCert] documentation for more details about what common name and extra names should be.
+func (ca *CA) MustServerTLSConfig(commonName string, extraNames ...string) *tls.Config {
+	return &tls.Config{
+		Certificates: []tls.Certificate{*ca.MustNewCert(commonName, extraNames...)},
+	}
+}

ca_test.go

@@ -0,0 +1,142 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package netem
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"net"
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestCAMustNewCert(t *testing.T) {
+	ca := MustNewCA()
+
+	tlsc := ca.MustNewCert("example.com", "www.example.com", "10.0.0.1", "10.0.0.2")
+
+	if tlsc.Certificate == nil {
+		t.Error("tlsc.Certificate: got nil, want certificate bytes")
+	}
+	if tlsc.PrivateKey == nil {
+		t.Error("tlsc.PrivateKey: got nil, want private key")
+	}
+
+	x509c := tlsc.Leaf
+	if x509c == nil {
+		t.Fatal("x509c: got nil, want *x509.Certificate")
+	}
+
+	if got := x509c.SerialNumber; got.Cmp(caMaxSerialNumber) >= 0 {
+		t.Errorf("x509c.SerialNumber: got %v, want <= MaxSerialNumber", got)
+	}
+	if got, want := x509c.Subject.CommonName, "example.com"; got != want {
+		t.Errorf("X509c.Subject.CommonName: got %q, want %q", got, want)
+	}
+	if err := x509c.VerifyHostname("example.com"); err != nil {
+		t.Errorf("x509c.VerifyHostname(%q): got %v, want no error", "example.com", err)
+	}
+
+	if got, want := x509c.Subject.Organization, []string{"OONI Netem CA"}; !reflect.DeepEqual(got, want) {
+		t.Errorf("x509c.Subject.Organization: got %v, want %v", got, want)
+	}
+
+	if got := x509c.SubjectKeyId; got == nil {
+		t.Error("x509c.SubjectKeyId: got nothing, want key ID")
+	}
+	if !x509c.BasicConstraintsValid {
+		t.Error("x509c.BasicConstraintsValid: got false, want true")
+	}
+
+	if got, want := x509c.KeyUsage, x509.KeyUsageKeyEncipherment; got&want == 0 {
+		t.Error("x509c.KeyUsage: got nothing, want to include x509.KeyUsageKeyEncipherment")
+	}
+	if got, want := x509c.KeyUsage, x509.KeyUsageDigitalSignature; got&want == 0 {
+		t.Error("x509c.KeyUsage: got nothing, want to include x509.KeyUsageDigitalSignature")
+	}
+
+	want := []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}
+	if got := x509c.ExtKeyUsage; !reflect.DeepEqual(got, want) {
+		t.Errorf("x509c.ExtKeyUsage: got %v, want %v", got, want)
+	}
+
+	if got, want := x509c.DNSNames, []string{"example.com", "www.example.com"}; !reflect.DeepEqual(got, want) {
+		t.Errorf("x509c.DNSNames: got %v, want %v", got, want)
+	}
+
+	if diff := cmp.Diff(x509c.IPAddresses, []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("10.0.0.2")}); diff != "" {
+		t.Errorf(diff)
+	}
+
+	before := time.Now().Add(-2 * time.Hour)
+	if got := x509c.NotBefore; before.After(got) {
+		t.Errorf("x509c.NotBefore: got %v, want after %v", got, before)
+	}
+
+	after := time.Now().Add(2 * time.Hour)
+	if got := x509c.NotAfter; !after.After(got) {
+		t.Errorf("x509c.NotAfter: got %v, want before %v", got, want)
+	}
+}
+
+func TestCAWeCanGenerateAnExpiredCertificate(t *testing.T) {
+	topology := MustNewStarTopology(log.Log)
+	defer topology.Close()
+
+	serverStack := Must1(topology.AddHost("10.0.0.1", "0.0.0.0", &LinkConfig{}))
+	clientStack := Must1(topology.AddHost("10.0.0.2", "0.0.0.0", &LinkConfig{}))
+
+	serverAddr := &net.TCPAddr{IP: net.IPv4(10, 0, 0, 1), Port: 443}
+	serverListener := Must1(serverStack.ListenTCP("tcp", serverAddr))
+
+	serverServer := &http.Server{
+		Handler: http.NewServeMux(),
+		TLSConfig: &tls.Config{
+			Certificates: []tls.Certificate{
+				*serverStack.CA().MustNewCertWithTimeNow(func() time.Time {
+					return time.Date(2017, time.July, 17, 0, 0, 0, 0, time.UTC)
+				},
+					"www.example.com",
+					"10.0.0.1",
+				),
+			},
+		},
+	}
+	go serverServer.ServeTLS(serverListener, "", "")
+	defer serverServer.Close()
+
+	tcpConn, err := clientStack.DialContext(context.Background(), "tcp", "10.0.0.1:443")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tcpConn.Close()
+
+	tlsClientConfig := &tls.Config{
+		RootCAs:    clientStack.DefaultCertPool(),
+		ServerName: "www.example.com",
+	}
+	tlsConn := tls.Client(tcpConn, tlsClientConfig)
+	err = tlsConn.HandshakeContext(context.Background())
+	if err == nil || !strings.Contains(err.Error(), "x509: certificate has expired or is not yet valid") {
+		t.Fatal("unexpected error", err)
+	}
+}

cmd/calibrate/topology.go

@@ -64,7 +64,7 @@ func newTopologyStar(
 	dnsConfig *netem.DNSConfig,
 ) (topologyCloser, *netem.UNetStack, *netem.UNetStack) {
 	// create an empty topology
-	topology := netem.Must1(netem.NewStarTopology(log.Log))
+	topology := netem.MustNewStarTopology(log.Log)
 
 	// add the client to the topology
 	clientStack := netem.Must1(topology.AddHost(clientAddress, serverAddress, clientLink))

example_dpi_test.go

@@ -16,10 +16,7 @@ import (
 // This example shows how to use DPI to provoke an EOF when you see an offending string.
 func Example_dpiCloseConnectionForString() {
 	// Create a star topology for our hosts.
-	topology, err := netem.NewStarTopology(&netem.NullLogger{})
-	if err != nil {
-		log.Fatal(err)
-	}
+	topology := netem.MustNewStarTopology(&netem.NullLogger{})
 	defer topology.Close()
 
 	// Create DPI engine in the client link
@@ -150,10 +147,7 @@ func Example_dpiCloseConnectionForString() {
 // This example shows how to use DPI to drop traffic after you see a given string,
 func Example_dpiDropTrafficForString() {
 	// Create a star topology for our hosts.
-	topology, err := netem.NewStarTopology(&netem.NullLogger{})
-	if err != nil {
-		log.Fatal(err)
-	}
+	topology := netem.MustNewStarTopology(&netem.NullLogger{})
 	defer topology.Close()
 
 	// Create DPI engine in the client link
@@ -287,10 +281,7 @@ func Example_dpiDropTrafficForString() {
 // This example shows how to use DPI to spoof a blockpage for a string
 func Example_dpiSpoofBlockpageForString() {
 	// Create a star topology for our hosts.
-	topology, err := netem.NewStarTopology(&netem.NullLogger{})
-	if err != nil {
-		log.Fatal(err)
-	}
+	topology := netem.MustNewStarTopology(&netem.NullLogger{})
 	defer topology.Close()
 
 	// Create DPI engine in the client link