Skip to content

Commit

Permalink
refactor: run networkd as a goroutine in machined
Browse files Browse the repository at this point in the history 
This removes networkd as a separate container and image.

Reasons:

* `machined` becomes more and more bound into the core flow - now it
interacts with `etcd` for VIPs, so container has more and more
mounts/permissions
* it should be easier to COSIfy machined piece by piece if we have it
running in the same process
* initramfs size

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>
  • Loading branch information
@smira @talos-bot
smira authored and talos-bot committed Mar 23, 2021
1 parent f4a6a19 commit 89a4b09
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 149 deletions.
21 changes: 0 additions & 21 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,26 +236,6 @@ WORKDIR /scratch
RUN printf "FROM scratch\nCOPY ./trustd /trustd\nENTRYPOINT [\"/trustd\"]" > Dockerfile
RUN --security=insecure img build --tag ${USERNAME}/trustd:${TAG} --output type=docker,dest=/trustd.tar --no-console .

# The networkd target builds the networkd image.

FROM base AS networkd-build
ARG SHA
ARG TAG
ARG PKGS
ARG EXTRAS
ARG VERSION_PKG="github.com/talos-systems/talos/pkg/version"
WORKDIR /src/internal/app/networkd
RUN --mount=type=cache,target=/.cache/go-build go build -ldflags "-s -w -X ${VERSION_PKG}.Name=Server -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG} -X ${VERSION_PKG}.PkgsVersion=${PKGS} -X ${VERSION_PKG}.ExtrasVersion=${EXTRAS}" -o /networkd
RUN chmod +x /networkd

FROM base AS networkd-image
ARG TAG
ARG USERNAME
COPY --from=networkd-build /networkd /scratch/networkd
WORKDIR /scratch
RUN printf "FROM scratch\nCOPY ./networkd /networkd\nENTRYPOINT [\"/networkd\"]" > Dockerfile
RUN --security=insecure img build --tag ${USERNAME}/networkd:${TAG} --output type=docker,dest=/networkd.tar --no-console .

# The routerd target builds the routerd image.

FROM base AS routerd-build
Expand Down Expand Up @@ -394,7 +374,6 @@ COPY --from=machined /machined /rootfs/sbin/init
COPY --from=apid-image /apid.tar /rootfs/usr/images/
COPY --from=timed-image /timed.tar /rootfs/usr/images/
COPY --from=trustd-image /trustd.tar /rootfs/usr/images/
COPY --from=networkd-image /networkd.tar /rootfs/usr/images/
COPY --from=routerd-image /routerd.tar /rootfs/usr/images/
# NB: We run the cleanup step before creating extra directories, files, and
# symlinks to avoid accidentally cleaning them up.
Expand Down
96 changes: 8 additions & 88 deletions internal/app/machined/pkg/system/services/networkd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,32 +6,26 @@
package services

import (
"bytes"
"context"
"errors"
"fmt"
"log"
"os"
"path/filepath"
"strings"

"github.com/containerd/containerd/oci"
"github.com/golang/protobuf/ptypes/empty"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/syndtr/gocapability/capability"
"google.golang.org/grpc"

"github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
"github.com/talos-systems/talos/internal/app/machined/pkg/system/events"
"github.com/talos-systems/talos/internal/app/machined/pkg/system/health"
"github.com/talos-systems/talos/internal/app/machined/pkg/system/runner"
"github.com/talos-systems/talos/internal/app/machined/pkg/system/runner/containerd"
"github.com/talos-systems/talos/internal/app/machined/pkg/system/runner/goroutine"
"github.com/talos-systems/talos/internal/app/machined/pkg/system/runner/restart"
"github.com/talos-systems/talos/internal/pkg/containers/image"
"github.com/talos-systems/talos/internal/app/networkd"
"github.com/talos-systems/talos/pkg/conditions"
"github.com/talos-systems/talos/pkg/grpc/dialer"
healthapi "github.com/talos-systems/talos/pkg/machinery/api/health"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
"github.com/talos-systems/talos/pkg/machinery/constants"
)

Expand All @@ -46,7 +40,7 @@ func (n *Networkd) ID(r runtime.Runtime) string {

// PreFunc implements the Service interface.
func (n *Networkd) PreFunc(ctx context.Context, r runtime.Runtime) error {
return image.Import(ctx, "/usr/images/networkd.tar", "talos/networkd")
return os.MkdirAll(filepath.Dir(constants.NetworkSocketPath), 0o750)
}

// PostFunc implements the Service interface.
Expand All @@ -61,89 +55,15 @@ func (n *Networkd) Condition(r runtime.Runtime) conditions.Condition {

// DependsOn implements the Service interface.
func (n *Networkd) DependsOn(r runtime.Runtime) []string {
return []string{"containerd"}
return nil
}

func (n *Networkd) Runner(r runtime.Runtime) (runner.Runner, error) {
image := "talos/networkd"

args := runner.Args{
ID: n.ID(r),
ProcessArgs: []string{
"/networkd",
},
}

// Ensure socket dir exists
if err := os.MkdirAll(filepath.Dir(constants.NetworkSocketPath), 0o750); err != nil {
return nil, err
}

mounts := []specs.Mount{
{Type: "bind", Destination: "/etc/resolv.conf", Source: "/etc/resolv.conf", Options: []string{"rbind", "rw"}},
{Type: "bind", Destination: "/etc/hosts", Source: "/etc/hosts", Options: []string{"rbind", "rw"}},
{Type: "bind", Destination: filepath.Dir(constants.NetworkSocketPath), Source: filepath.Dir(constants.NetworkSocketPath), Options: []string{"rbind", "rw"}},
}

// etcd is used for VIP controller on control plane nodes
if r.Config().Machine().Type() != machine.TypeJoin {
// Ensure etcd PKI dir exists
if err := os.MkdirAll(constants.EtcdPKIPath, 0o700); err != nil {
return nil, err
}

// Fix up permissions, as after upgrade with preserve EtcdPKIPath might retain old 0o644 permissions
if err := os.Chmod(constants.EtcdPKIPath, 0o700); err != nil {
return nil, err
}

mounts = append(mounts,
specs.Mount{Type: "bind", Destination: constants.EtcdPKIPath, Source: constants.EtcdPKIPath, Options: []string{"rbind", "ro"}},
)
}

if r.State().Platform().Mode() == runtime.ModeContainer {
mounts = append(mounts,
specs.Mount{Type: "bind", Destination: "/etc/hostname", Source: "/etc/hostname", Options: []string{"rbind", "ro"}},
)
}

env := []string{}
for key, val := range r.Config().Machine().Env() {
env = append(env, fmt.Sprintf("%s=%s", key, val))
}

// This is really only here to support container runtime
if p, ok := os.LookupEnv("PLATFORM"); ok {
env = append(env, fmt.Sprintf("%s=%s", "PLATFORM", p))
}

b, err := r.Config().Bytes()
if err != nil {
return nil, err
}

stdin := bytes.NewReader(b)

return restart.New(containerd.NewRunner(
r.Config().Debug(),
&args,
runner.WithStdin(stdin),
return restart.New(goroutine.NewRunner(
r,
"networkd",
networkd.Main,
runner.WithLoggingManager(r.Logging()),
runner.WithContainerdAddress(constants.SystemContainerdAddress),
runner.WithContainerImage(image),
runner.WithEnv(env),
runner.WithOCISpecOpts(
containerd.WithMemoryLimit(int64(1000000*32)),
oci.WithCapabilities([]string{
strings.ToUpper("CAP_" + capability.CAP_NET_ADMIN.String()),
strings.ToUpper("CAP_" + capability.CAP_SYS_ADMIN.String()),
strings.ToUpper("CAP_" + capability.CAP_NET_RAW.String()),
strings.ToUpper("CAP_" + capability.CAP_NET_BIND_SERVICE.String()),
}),
oci.WithHostNamespace(specs.NetworkNamespace),
oci.WithMounts(mounts),
),
),
restart.WithType(restart.Forever),
), nil
Expand Down
51 changes: 11 additions & 40 deletions internal/app/networkd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,66 +2,37 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main
package networkd

import (
"context"
"io"
"log"
"os"
"os/signal"
"runtime"
"syscall"

"golang.org/x/sync/errgroup"

"github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
"github.com/talos-systems/talos/internal/app/networkd/pkg/networkd"
"github.com/talos-systems/talos/internal/app/networkd/pkg/reg"
"github.com/talos-systems/talos/pkg/grpc/factory"
"github.com/talos-systems/talos/pkg/machinery/config/configloader"
"github.com/talos-systems/talos/pkg/machinery/constants"
)

func init() {
// Explicitly disable memory profiling to save around 1.4MiB of memory.
runtime.MemProfileRate = 0
}

func main() {
logger := log.New(os.Stderr, "", log.Lshortfile|log.Ldate|log.Lmicroseconds|log.Ltime)

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Main is the entrypoint into networkd.
func Main(ctx context.Context, r runtime.Runtime, logOutput io.Writer) error {
logger := log.New(logOutput, "", log.Lshortfile|log.Ldate|log.Lmicroseconds|log.Ltime)

go func() {
defer cancel()
defer logger.Println("networkd stopped")

sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, syscall.SIGTERM, os.Interrupt)

select {
case <-sigCh:
case <-ctx.Done():
}
}()

if err := run(ctx, logger); err != nil {
logger.Fatal(err)
}

logger.Println("networkd stopped")
return run(ctx, r, logger)
}

func run(ctx context.Context, logger *log.Logger) error {
func run(ctx context.Context, r runtime.Runtime, logger *log.Logger) error {
var eg errgroup.Group

config, err := configloader.NewFromStdin()
if err != nil {
return err
}

logger.Println("starting initial network configuration")

nwd, err := networkd.New(logger, config)
nwd, err := networkd.New(logger, r.Config())
if err != nil {
return err
}
Expand All @@ -85,7 +56,7 @@ func run(ctx context.Context, logger *log.Logger) error {

server := factory.NewServer(
registrator,
factory.WithDefaultLog(),
factory.WithLog("", logger.Writer()),
)

listener, err := factory.NewListener(
Expand Down

0 comments on commit 89a4b09

Please sign in to comment.