From 616ab83ef3c5b089420cae7c5395962e2de2f44f Mon Sep 17 00:00:00 2001 From: John Stairs Date: Thu, 12 Dec 2024 19:41:51 -0500 Subject: [PATCH] Support custom installation path for Docker mode (#163) --- .devcontainer/devcontainer.json | 6 +- .devcontainer/prepare-host.sh | 31 ----- .vscode/launch.json | 2 +- Makefile.docker | 27 +++-- cli/integrationtest/migrations_test.go | 32 +++-- cli/internal/client/client.go | 14 ++- cli/internal/client/client_test.go | 21 +--- cli/internal/client/sshurl.go | 2 +- cli/internal/cmd/stdioproxy.go | 2 +- cli/internal/controlplane/login.go | 24 +++- cli/internal/install/dockerinstall/docker.go | 113 +++++++++++++++--- .../install/dockerinstall/migrations.go | 4 +- .../install/dockerinstall/validation.go | 15 ++- deploy/config/microsoft/dockerconfig.yml | 2 + .../installation/docker-installation.md | 41 ++++++- install/.gitignore | 1 + scripts/get-config.sh | 3 + scripts/run-ssh-tests.sh | 19 ++- server/ControlPlane/Buffers/Buffers.cs | 2 +- server/ControlPlane/Compute/Docker/Docker.cs | 7 +- .../Compute/Docker/DockerRunCreator.cs | 39 ++++-- 21 files changed, 282 insertions(+), 125 deletions(-) delete mode 100755 .devcontainer/prepare-host.sh create mode 100644 install/.gitignore diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a09af024..1fb456aa 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -79,21 +79,19 @@ "mounts": [ // Bind mount docker socket under an alias to support docker-from-docker "source=/var/run/docker.sock,target=/var/run/docker-host.sock,type=bind", - "source=/opt/tyger,target=/opt/tyger,type=bind", - "source=/tmp/tyger,target=/tmp/tyger,type=bind", "source=tyger-local-buffers,target=/docker-volumes/buffers,type=volume", "source=tyger-local-run-logs,target=/docker-volumes/run-logs,type=volume" ], "remoteUser": "vscode", "overrideCommand": false, - "initializeCommand": ".devcontainer/prepare-host.sh", "onCreateCommand": ".devcontainer/devcontainer-on-create.sh", "containerEnv": { "DOCKER_BUIKDKIT": "1", "DEVCONTAINER_HOST_HOME": "${localEnv:HOME}", - "TYGER_ACCESSING_FROM_DOCKER": "1" + "TYGER_ACCESSING_FROM_DOCKER": "1", + "TYGER_DOCKER_HOST_PATH_TRANSLATIONS": "${containerWorkspaceFolder}=${localWorkspaceFolder}" }, "forwardPorts": [ diff --git a/.devcontainer/prepare-host.sh b/.devcontainer/prepare-host.sh deleted file mode 100755 index dbdec975..00000000 --- a/.devcontainer/prepare-host.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -set -euo pipefail - -# Check if /opt/tyger exists. If if does, exit. Otherwise, create it. This will require sudo. -# The ownership should be the same as if it sudo hasn't been used. - -if [ ! -d /opt/tyger ]; then - uid=$(id -u) - gid=$(id -g) - sudo mkdir /opt/tyger - sudo chown -R "$uid":"$gid" /opt/tyger -fi - -if [ ! -d /tmp/tyger ]; then - mkdir -m 777 /tmp/tyger -fi - -if [ -n "${WSL_DISTRO_NAME:-}" ]; then - minimum_docker_desktop_version="4.31.0" - - docker_version=$(docker.exe version | grep -oP 'Server: Docker Desktop \d+\.\d+(\.\d+)?') - docker_version=$(echo "$docker_version" | grep -oP '\d+\.\d+(\.\d+)?') - if [ "$(printf '%s\n' $minimum_docker_desktop_version "$docker_version" | sort -V | head -n1)" != $minimum_docker_desktop_version ]; then - echo "Docker Desktop version $docker_version is not supported. Please upgrade to version $minimum_docker_desktop_version or later." - exit 1 - fi -fi diff --git a/.vscode/launch.json b/.vscode/launch.json index 5226e470..8cd222f2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -49,7 +49,7 @@ "mode": "auto", "program": "cli/cmd/tyger", "cwd": "${workspaceFolder}", - "args": ["login", "status", "--format", "json"], + "args": ["api", "install", "-f", "/home/vscode/tyger.config"], "env": { "TYGER_CACHE_FILE": "/home/vscode/.cache/tyger/.tyger-docker" }, diff --git a/Makefile.docker b/Makefile.docker index db5159cf..84a4c749 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -9,13 +9,22 @@ login-wip-acr: # do nothing. We don't push any images local docker mode. set-localsettings: ensure-data-plane-cert - run_secrets_path="/opt/tyger/control-plane/run-secrets/" - ephemeral_buffers_path="/opt/tyger/control-plane/ephemeral-buffers" + install_path=$$(realpath "install/local") + run_secrets_path="$${install_path}/control-plane/run-secrets" + ephemeral_buffers_path="$${install_path}//ephemeral" logs_path="/docker-volumes/run-logs" + host_path_translations=$$(echo "$$TYGER_DOCKER_HOST_PATH_TRANSLATIONS" | tr ':' '\n' \ + | awk -F= '{ \ + s=$$1; d=$$2; \ + if (substr(s, length(s), 1) != "/") s = s"/"; \ + if (substr(d, length(d), 1) != "/") d = d"/"; \ + printf "\"%s\":\"%s\",", s, d \ + }' \ + | sed 's/,$$//'); \ jq <<- EOF > ${CONTROL_PLANE_SERVER_PATH}/appsettings.local.json { - "urls": "http://unix:/opt/tyger/control-plane/tyger.sock", + "urls": "http://unix:$${install_path}/api.sock", "logging": { "Console": {"FormatterName": "simple" } }, "auth": { "enabled": "false" @@ -24,7 +33,8 @@ set-localsettings: ensure-data-plane-cert "docker": { "runSecretsPath": "$${run_secrets_path}", "ephemeralBuffersPath": "$${ephemeral_buffers_path}", - "networkName": "tyger-local-network" + "networkName": "tyger-local-network", + "hostPathTranslations": {$$host_path_translations} } }, "logArchive": { @@ -36,12 +46,12 @@ set-localsettings: ensure-data-plane-cert "primarySigningPrivateKeyPath": "$$(echo '${ENVIRONMENT_CONFIG_JSON}' | jq -r '.signingKeys.primary.private')", "bufferSidecarImage": "$$(echo '${ENVIRONMENT_CONFIG_JSON}' | jq -r '.bufferSidecarImage')", "localStorage": { - "dataPlaneEndpoint": "http+unix:///opt/tyger/data-plane/tyger.data.sock", + "dataPlaneEndpoint": "http+unix://$${install_path}/data-plane/tyger.data.sock", "tcpDataPlaneEndpoint": "http://localhost:$$(echo '${ENVIRONMENT_CONFIG_JSON}' | jq -r '.dataPlanePort')" } }, "database": { - "host": "/opt/tyger/database", + "host": "$${install_path}/database", "username": "tyger-server", "databaseName": "tyger-server", "autoMigrate": "true", @@ -51,9 +61,10 @@ set-localsettings: ensure-data-plane-cert EOF set-data-plane-localsettings: + install_path=$$(realpath "install/local") jq <<- EOF > ${DATA_PLANE_SERVER_PATH}/appsettings.local.json { - "urls": "http://unix:/opt/tyger/data-plane/tyger.data.sock", + "urls": "http://unix:$${install_path}/data-plane/tyger.data.sock", "logging": { "Console": {"FormatterName": "simple" } }, "dataDirectory": "/docker-volumes/buffers", "PrimarySigningPublicKeyPath": "$$(echo '${ENVIRONMENT_CONFIG_JSON}' | jq -r '.signingKeys.primary.public')" @@ -81,7 +92,7 @@ run-data-plane: set-data-plane-localsettings dotnet run -v m --no-restore login: - tyger login --local + tyger login unix://install/local/api.sock connect-db: docker exec -it tyger-local-db psql -U tyger-server tyger-server diff --git a/cli/integrationtest/migrations_test.go b/cli/integrationtest/migrations_test.go index 32984074..6436c853 100644 --- a/cli/integrationtest/migrations_test.go +++ b/cli/integrationtest/migrations_test.go @@ -126,23 +126,33 @@ func TestCloudMigrations(t *testing.T) { assert.Contains(t, logs, "Migration 2 complete") } +func getTempDockerInstallationPath(t *testing.T) string { + lowercaseTestName := strings.ToLower(t.Name()) + + installationPath := fmt.Sprintf("../../install/%s", lowercaseTestName) + require.NoError(t, os.MkdirAll(installationPath, 0755)) + installationPath, err := filepath.Abs(installationPath) + require.NoError(t, err) + installationPath, err = filepath.EvalSymlinks(installationPath) + require.NoError(t, err) + + return installationPath +} + func TestDockerOnlineMigrations(t *testing.T) { t.Parallel() skipUnlessUsingUnixSocket(t) skipIfNotUsingUnixSocketDirectly(t) + installationPath := getTempDockerInstallationPath(t) + defer os.RemoveAll(installationPath) + + environmentConfig := runCommandSucceeds(t, "../../scripts/get-config.sh", "--docker") + lowercaseTestName := strings.ToLower(t.Name()) if len(lowercaseTestName) > 23 { lowercaseTestName = lowercaseTestName[:23] } - - environmentConfig := runCommandSucceeds(t, "../../scripts/get-config.sh", "--docker") - - installationPath := fmt.Sprintf("/tmp/tyger/%s", lowercaseTestName) - defer func() { - os.RemoveAll(installationPath) - }() - configMap := make(map[string]any) require.NoError(t, yaml.Unmarshal([]byte(environmentConfig), &configMap)) configMap["environmentName"] = lowercaseTestName @@ -193,10 +203,8 @@ func TestDockerOfflineMigrations(t *testing.T) { environmentConfig := runCommandSucceeds(t, "../../scripts/get-config.sh", "--docker") devConfig := getDevConfig(t) - installationPath := fmt.Sprintf("/tmp/tyger/%s", lowercaseTestName) - defer func() { - os.RemoveAll(installationPath) - }() + installationPath := getTempDockerInstallationPath(t) + defer os.RemoveAll(installationPath) configMap := make(map[string]any) require.NoError(t, yaml.Unmarshal([]byte(environmentConfig), &configMap)) diff --git a/cli/internal/client/client.go b/cli/internal/client/client.go index 5de4fd14..051dc6e4 100644 --- a/cli/internal/client/client.go +++ b/cli/internal/client/client.go @@ -10,6 +10,7 @@ import ( "net" "net/http" "net/url" + "os" "strings" "time" @@ -21,8 +22,8 @@ import ( ) const ( - DefaultControlPlaneUnixSocketPath = "/opt/tyger/api.sock" - DefaultControlPlaneUnixSocketUrl = "http+unix://" + DefaultControlPlaneUnixSocketPath + ":" + DefaultControlPlaneSocketPathEnvVar = "TYGER_SOCKET_PATH" + defaultControlPlaneUnixSocketPath = "/opt/tyger/api.sock" ) var ( @@ -31,6 +32,15 @@ var ( DefaultRetryableClient *retryablehttp.Client ) +func GetDefaultSocketUrl() string { + path := os.Getenv(DefaultControlPlaneSocketPathEnvVar) + if path == "" { + path = defaultControlPlaneUnixSocketPath + } + + return "http+unix://" + path + ":" +} + type MakeRoundTripper func(next http.RoundTripper) http.RoundTripper type MakeDialer func(next dialContextFunc) dialContextFunc diff --git a/cli/internal/client/client_test.go b/cli/internal/client/client_test.go index 30d72de5..45f42ef0 100644 --- a/cli/internal/client/client_test.go +++ b/cli/internal/client/client_test.go @@ -4,7 +4,6 @@ package client import ( - "context" "net/http" "net/url" "testing" @@ -109,28 +108,10 @@ func TestGetProxyFuncExplicitProxyWithoutScheme(t *testing.T) { } func TestGetProxyFuncWithDataPlaneProxy(t *testing.T) { - cpClient, err := NewClient(&ClientOptions{ProxyString: "none"}) - require.NoError(t, err) - dpProxy := "http://111.222.333.444:5555" dpClient, err := NewClient(&ClientOptions{ProxyString: dpProxy}) require.NoError(t, err) - controlPlaneUrl, err := url.Parse("https://example.com") - require.NoError(t, err) - - tygerClient := &TygerClient{ - ControlPlaneUrl: controlPlaneUrl, - ControlPlaneClient: cpClient, - GetAccessToken: func(ctx context.Context) (string, error) { - return "", nil - }, - DataPlaneClient: dpClient, - Principal: "me", - RawControlPlaneUrl: controlPlaneUrl, - RawProxy: nil, - } - dataPlaneUrl, err := url.Parse("https://dataplane.example.com") require.NoError(t, err) @@ -138,7 +119,7 @@ func TestGetProxyFuncWithDataPlaneProxy(t *testing.T) { URL: dataPlaneUrl, } - proxyURL, err := tygerClient.DataPlaneClient.Proxy(req) + proxyURL, err := dpClient.Proxy(req) require.NoError(t, err) require.NotNil(t, proxyURL) require.Equal(t, dpProxy, proxyURL.String()) diff --git a/cli/internal/client/sshurl.go b/cli/internal/client/sshurl.go index dc40f9f9..b9aff6eb 100644 --- a/cli/internal/client/sshurl.go +++ b/cli/internal/client/sshurl.go @@ -173,7 +173,7 @@ func (sp *SshParams) FormatLoginArgs(add ...string) []string { args := []string{"login"} if sp.SocketPath != "" { - args = append(args, "--socket-path", sp.SocketPath) + args = append(args, "--server-url", fmt.Sprintf("http+unix://%s", sp.SocketPath)) } args = append(args, add...) diff --git a/cli/internal/cmd/stdioproxy.go b/cli/internal/cmd/stdioproxy.go index 4e8f6387..df84444e 100644 --- a/cli/internal/cmd/stdioproxy.go +++ b/cli/internal/cmd/stdioproxy.go @@ -96,7 +96,7 @@ func writeResponseForError(err error) { } func newStdioProxyLoginCommand() *cobra.Command { - serverUrl := client.DefaultControlPlaneUnixSocketUrl + serverUrl := client.GetDefaultSocketUrl() preflight := false cmd := &cobra.Command{ Use: "login", diff --git a/cli/internal/controlplane/login.go b/cli/internal/controlplane/login.go index c9d759db..a8713699 100644 --- a/cli/internal/controlplane/login.go +++ b/cli/internal/controlplane/login.go @@ -15,6 +15,7 @@ import ( "net/url" "os" "os/exec" + "path" "path/filepath" "strings" "time" @@ -80,7 +81,7 @@ type serviceInfo struct { func Login(ctx context.Context, options LoginConfig) (*client.TygerClient, error) { if options.ServerUri == LocalUriSentinel { optionsClone := options - optionsClone.ServerUri = client.DefaultControlPlaneUnixSocketUrl + optionsClone.ServerUri = client.GetDefaultSocketUrl() c, errUnix := Login(ctx, optionsClone) if errUnix == nil { return c, nil @@ -344,7 +345,26 @@ func NormalizeServerUri(uri string) (*url.URL, error) { } if parsedUrl.Scheme == "http+unix" || parsedUrl.Scheme == "https+unix" { - if !strings.HasSuffix(uri, ":") { + // Turn a relative path into an absolute path + // If the path is already absolute, Host will be empty + // Otherwise, host will be the first path segment. + if parsedUrl.Host != "" { + if parsedUrl.Path == "" { + parsedUrl.Path = parsedUrl.Host + } else { + parsedUrl.Path = parsedUrl.Host + parsedUrl.Path + } + parsedUrl.Host = "" + } + + if !path.IsAbs(parsedUrl.Path) { + parsedUrl.Path, err = filepath.Abs(parsedUrl.Path) + if err != nil { + return nil, fmt.Errorf("failed to make path absolute: %w", err) + } + } + + if !strings.HasSuffix(parsedUrl.Path, ":") { parsedUrl.Path += ":" } } diff --git a/cli/internal/install/dockerinstall/docker.go b/cli/internal/install/dockerinstall/docker.go index 200c404f..c1004022 100644 --- a/cli/internal/install/dockerinstall/docker.go +++ b/cli/internal/install/dockerinstall/docker.go @@ -15,6 +15,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "regexp" "runtime" "strconv" @@ -31,6 +32,7 @@ import ( "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" "github.com/docker/go-connections/nat" + tygerclient "github.com/microsoft/tyger/cli/internal/client" "github.com/microsoft/tyger/cli/internal/install" "github.com/psanford/memfs" "github.com/rs/zerolog/log" @@ -69,8 +71,9 @@ func (s *containerSpec) computeHash() string { } type Installer struct { - Config *DockerEnvironmentConfig - client *client.Client + Config *DockerEnvironmentConfig + client *client.Client + hostPathTranslations map[string]string } func NewInstaller(config *DockerEnvironmentConfig) (*Installer, error) { @@ -78,9 +81,32 @@ func NewInstaller(config *DockerEnvironmentConfig) (*Installer, error) { if err != nil { return nil, fmt.Errorf("error creating docker client: %w", err) } + + hostPathTranslations := map[string]string{} + translationsEnvVar := os.Getenv("TYGER_DOCKER_HOST_PATH_TRANSLATIONS") + if translationsEnvVar != "" { + for _, spec := range strings.Split(translationsEnvVar, ":") { + tokens := strings.Split(spec, "=") + if len(tokens) == 2 { + source := tokens[0] + if !strings.HasSuffix(source, "/") { + source += "/" + } + + dest := tokens[1] + if !strings.HasSuffix(dest, "/") { + dest += "/" + } + + hostPathTranslations[source] = dest + } + } + } + return &Installer{ - Config: config, - client: dockerClient, + Config: config, + client: dockerClient, + hostPathTranslations: hostPathTranslations, }, nil } @@ -88,6 +114,20 @@ func (inst *Installer) resourceName(suffix string) string { return fmt.Sprintf("tyger-%s-%s", inst.Config.EnvironmentName, suffix) } +func (inst *Installer) translateToHostPath(path string) string { + for source, dest := range inst.hostPathTranslations { + if strings.HasPrefix(path, source) { + return dest + strings.TrimPrefix(path, source) + } + + if len(path)+1 == len(source) && path == source[:len(source)-1] { + return dest[:len(dest)-1] + } + } + + return path +} + func (inst *Installer) InstallTyger(ctx context.Context) error { if runtime.GOOS == "windows" { log.Error().Msg("Installing Tyger in Docker on Windows must be done from a WSL shell. Once installed, other commands can be run from within Windows.") @@ -196,6 +236,10 @@ func (inst *Installer) createControlPlaneContainer(ctx context.Context, checkGpu return err } + if err := inst.ensureDirectoryExists(fmt.Sprintf("%s/ephemeral", inst.Config.InstallationPath)); err != nil { + return err + } + if err := inst.pullImage(ctx, inst.Config.BufferSidecarImage, false); err != nil { return fmt.Errorf("error pulling buffer sidecar image: %w", err) } @@ -227,6 +271,7 @@ func (inst *Installer) createControlPlaneContainer(ctx context.Context, checkGpu log.Warn().Msg("GPU support is not available.") } + hostInstallationPath := inst.translateToHostPath(inst.Config.InstallationPath) containerSpec := containerSpec{ ContainerConfig: &container.Config{ Image: image, @@ -235,8 +280,8 @@ func (inst *Installer) createControlPlaneContainer(ctx context.Context, checkGpu fmt.Sprintf("Urls=http://unix:%s/control-plane/tyger.sock", inst.Config.InstallationPath), "SocketPermissions=660", "Auth__Enabled=false", - fmt.Sprintf("Compute__Docker__RunSecretsPath=%s/control-plane/run-secrets/", inst.Config.InstallationPath), - fmt.Sprintf("Compute__Docker__EphemeralBuffersPath=%s/control-plane/ephemeral-buffers/", inst.Config.InstallationPath), + fmt.Sprintf("Compute__Docker__RunSecretsPath=%s/control-plane/run-secrets", inst.Config.InstallationPath), + fmt.Sprintf("Compute__Docker__EphemeralBuffersPath=%s/ephemeral", inst.Config.InstallationPath), fmt.Sprintf("Compute__Docker__GpuSupport=%t", gpuAvailable), fmt.Sprintf("Compute__Docker__NetworkName=%s", inst.resourceName("network")), "LogArchive__LocalStorage__LogsDirectory=/app/logs", @@ -270,17 +315,22 @@ func (inst *Installer) createControlPlaneContainer(ctx context.Context, checkGpu }, { Type: "bind", - Source: fmt.Sprintf("%s/control-plane", inst.Config.InstallationPath), + Source: fmt.Sprintf("%s/control-plane", hostInstallationPath), Target: fmt.Sprintf("%s/control-plane", inst.Config.InstallationPath), }, { Type: "bind", - Source: fmt.Sprintf("%s/data-plane", inst.Config.InstallationPath), + Source: fmt.Sprintf("%s/ephemeral", hostInstallationPath), + Target: fmt.Sprintf("%s/ephemeral", inst.Config.InstallationPath), + }, + { + Type: "bind", + Source: fmt.Sprintf("%s/data-plane", hostInstallationPath), Target: fmt.Sprintf("%s/data-plane", inst.Config.InstallationPath), }, { Type: "bind", - Source: fmt.Sprintf("%s/database", inst.Config.InstallationPath), + Source: fmt.Sprintf("%s/database", hostInstallationPath), Target: fmt.Sprintf("%s/database", inst.Config.InstallationPath), }, { @@ -296,6 +346,10 @@ func (inst *Installer) createControlPlaneContainer(ctx context.Context, checkGpu }, } + for source, dest := range inst.hostPathTranslations { + containerSpec.ContainerConfig.Env = append(containerSpec.ContainerConfig.Env, fmt.Sprintf("Compute__Docker__HostPathTranslations__%s=%s", source, dest)) + } + // See if there is a group that has access to the docker socket. // If there is, add that group to the container. _, dockerSocketGroupId, dockerSocketPerms, err := inst.statDockerSocket(ctx) @@ -343,13 +397,36 @@ func (inst *Installer) createControlPlaneContainer(ctx context.Context, checkGpu return err } - if err := os.Symlink(fmt.Sprintf("%s/control-plane/tyger.sock", inst.Config.InstallationPath), fmt.Sprintf("%s/api.sock", inst.Config.InstallationPath)); err != nil && !os.IsExist(err) { + linkPath, err := inst.getApiSocketPath() + if err != nil { + return err + } + + relativeTargetPath := "control-plane/tyger.sock" + + if existingTarget, err := os.Readlink(linkPath); err == nil { + if existingTarget == relativeTargetPath { + return nil + } + if err := os.Remove(linkPath); err != nil { + return fmt.Errorf("error removing existing symlink: %w", err) + } + } else if !os.IsNotExist(err) { + return fmt.Errorf("error reading existing symlink: %w", err) + } + + if err := os.Symlink(relativeTargetPath, linkPath); err != nil { return fmt.Errorf("error creating symlink: %w", err) } return nil } +func (inst *Installer) getApiSocketPath() (string, error) { + path := fmt.Sprintf("%s/api.sock", inst.Config.InstallationPath) + return filepath.Abs(path) +} + func fileHash(path string) (string, error) { hash := sha256.New() f, err := os.Open(path) @@ -507,7 +584,7 @@ func (inst *Installer) createDatabaseContainer(ctx context.Context) error { }, { Type: "bind", - Source: fmt.Sprintf("%s/database", inst.Config.InstallationPath), + Source: fmt.Sprintf("%s/database", inst.translateToHostPath(inst.Config.InstallationPath)), Target: "/var/run/postgresql/", }, }, @@ -587,7 +664,7 @@ func (inst *Installer) createDataPlaneContainer(ctx context.Context) error { }, { Type: "bind", - Source: fmt.Sprintf("%s/data-plane", inst.Config.InstallationPath), + Source: fmt.Sprintf("%s/data-plane", inst.translateToHostPath(inst.Config.InstallationPath)), Target: fmt.Sprintf("%s/data-plane", inst.Config.InstallationPath), }, }, @@ -645,19 +722,25 @@ func (inst *Installer) createDataPlaneContainer(ctx context.Context) error { } func (inst *Installer) createGatewayContainer(ctx context.Context) error { - image := inst.Config.GatewayImage + socketPath, err := inst.getApiSocketPath() + if err != nil { + return err + } spec := containerSpec{ ContainerConfig: &container.Config{ - Image: image, + Image: inst.Config.GatewayImage, User: inst.Config.UserId, Cmd: []string{"stdio-proxy", "sleep"}, + Env: []string{ + fmt.Sprintf("%s=%s", tygerclient.DefaultControlPlaneSocketPathEnvVar, socketPath), + }, }, HostConfig: &container.HostConfig{ Mounts: []mount.Mount{ { Type: "bind", - Source: inst.Config.InstallationPath, + Source: inst.translateToHostPath(inst.Config.InstallationPath), Target: inst.Config.InstallationPath, }, }, diff --git a/cli/internal/install/dockerinstall/migrations.go b/cli/internal/install/dockerinstall/migrations.go index dfce0463..e77d0393 100644 --- a/cli/internal/install/dockerinstall/migrations.go +++ b/cli/internal/install/dockerinstall/migrations.go @@ -283,6 +283,8 @@ func (inst *Installer) initializeDatabase(ctx context.Context) error { } func (inst *Installer) startMigrationRunner(ctx context.Context, containerName string, args []string, labels map[string]string) error { + translatedInstallationPath := inst.translateToHostPath(inst.Config.InstallationPath) + containerSpec := containerSpec{ ContainerConfig: &container.Config{ Image: inst.Config.ControlPlaneImage, @@ -302,7 +304,7 @@ func (inst *Installer) startMigrationRunner(ctx context.Context, containerName s Mounts: []mount.Mount{ { Type: "bind", - Source: inst.Config.InstallationPath, + Source: translatedInstallationPath, Target: inst.Config.InstallationPath, }, }, diff --git a/cli/internal/install/dockerinstall/validation.go b/cli/internal/install/dockerinstall/validation.go index 6d4cf0a4..003010ce 100644 --- a/cli/internal/install/dockerinstall/validation.go +++ b/cli/internal/install/dockerinstall/validation.go @@ -18,9 +18,12 @@ import ( var ( NameRegex = regexp.MustCompile(`^[a-z][a-z\-0-9]{1,23}$`) - DefaultEnvironmentName = "local" - DefaultPostgresImage = "postgres:16.2" - DefaultMarinerImage = "mcr.microsoft.com/azurelinux/base/core:3.0" + DefaultEnvironmentName = "local" + DefaultInstallationPath = "/opt/tyger" + DefaultPostgresImage = "postgres:16.2" + DefaultMarinerImage = "mcr.microsoft.com/azurelinux/base/core:3.0" + + InstallationPathMaxLength = 70 ) func (inst *Installer) QuickValidateConfig() bool { @@ -33,11 +36,15 @@ func (inst *Installer) QuickValidateConfig() bool { } if inst.Config.InstallationPath == "" { - inst.Config.InstallationPath = "/opt/tyger" + inst.Config.InstallationPath = DefaultInstallationPath } else if inst.Config.InstallationPath[len(inst.Config.InstallationPath)-1] == '/' { inst.Config.InstallationPath = inst.Config.InstallationPath[:len(inst.Config.InstallationPath)-1] } + if len(inst.Config.InstallationPath) > InstallationPathMaxLength { + validationError(&success, "The `installationPath` field must be at most %d characters long", InstallationPathMaxLength) + } + if _, err := strconv.Atoi(inst.Config.UserId); err != nil { if inst.Config.UserId == "" { currentUser, err := user.Current() diff --git a/deploy/config/microsoft/dockerconfig.yml b/deploy/config/microsoft/dockerconfig.yml index 271662db..11d072a3 100644 --- a/deploy/config/microsoft/dockerconfig.yml +++ b/deploy/config/microsoft/dockerconfig.yml @@ -1,5 +1,7 @@ kind: docker + +installationPath: ${TYGER_INSTALLATION_PATH} dataPlanePort: 46339 signingKeys: diff --git a/docs/introduction/installation/docker-installation.md b/docs/introduction/installation/docker-installation.md index 13e3e216..73a4cf19 100644 --- a/docs/introduction/installation/docker-installation.md +++ b/docs/introduction/installation/docker-installation.md @@ -42,6 +42,9 @@ The installation configuration file typically looks like this: ```yaml kind: docker +# The installation path. Defaults to /opt/tyger. +installationPath: + # Optionally specify the user id that the services will run as userId: @@ -97,17 +100,39 @@ tyger api install -f config.yml If using Windows, you will need to run this command from a WSL prompt. ::: -Tyger requires the directory `/opt/tyger` to exist. You many run the command -with `sudo` in order to create it. This path is currently not configurable. +If using the default `installationDirectory` (`/opt/tyger`), you will probably +need to create it using ahead of time using `sudo`. For example: + +```bash +uid=$(id -u) +gid=$(id -g) +sudo mkdir /opt/tyger +sudo chown -R "$uid":"$gid" /opt/tyger +``` + +We have an open [issue](https://github.com/microsoft/tyger/issues/146) to reduce the permissions of the installation directory. ## Testing it out -Log in with the `tyger` CLI using +If using the default installation directory, you can log in with the `tyger` CLI using: ```bash tyger login --local ``` +If using a different directory, you can log specifying the socket path: + +```bash +tyger login unix:///path/to/installation/dir/api.sock +``` + +Or you can set the `TYGER_SOCKET_PATH` environment variable: + +```bash +export TYGER_SOCKET_PATH=/path/to/installation/dir/api.sock +tyger login --local +``` + Once logged in, you should be able to run any of the core commands, such as: ```bash @@ -149,12 +174,16 @@ not a password. The format of the SSH URL is: ``` -ssh://[user@]host[:port][?key1=value1&key2=value2] +ssh://[user@]host[:port][/path/to/installation/directory/api.sock][?key1=value1&key2=value2] ``` All values in `[]` are optional. The user and port default values will come from -your SSH config file (~/.ssh/config). Additional parameters can be passed in -as query parameters (after the `?`). These are: +your SSH config file (~/.ssh/config). The API socket path can be omitted if the +socket path is the default `/opt/tyger/api.sock` or if the `TYGER_SOCKET_PATH` +environment variable is set on the SSH host. + +Additional parameters can be passed in as query parameters (after the `?`). +These are: - `cliPath`, to speciy that path to the `tyger` CLI on the host. This is only necessary if the localtion is not part of the `PATH` variable. diff --git a/install/.gitignore b/install/.gitignore new file mode 100644 index 00000000..df3359dd --- /dev/null +++ b/install/.gitignore @@ -0,0 +1 @@ +*/* \ No newline at end of file diff --git a/scripts/get-config.sh b/scripts/get-config.sh index 06a183f8..20360d0b 100755 --- a/scripts/get-config.sh +++ b/scripts/get-config.sh @@ -75,6 +75,9 @@ if [[ "$dev" == true ]]; then else if [[ "$docker" == true ]]; then config_path="${config_dir}/dockerconfig.yml" + + TYGER_INSTALLATION_PATH="$(realpath "${this_dir}/../install/local")" + export TYGER_INSTALLATION_PATH else config_path="${config_dir}/cloudconfig.yml" diff --git a/scripts/run-ssh-tests.sh b/scripts/run-ssh-tests.sh index 95464efc..5f898610 100755 --- a/scripts/run-ssh-tests.sh +++ b/scripts/run-ssh-tests.sh @@ -45,13 +45,18 @@ if [[ -z ${start_only:-} ]]; then trap cleanup SIGINT SIGTERM EXIT fi +# Copy bind mounts from tyger-local-gateway +gateway_bind_mounts=$(docker inspect -f '{{range .Mounts}}{{if eq .Type "bind"}}-v {{.Source}}:{{.Destination}} {{end}}{{end}}' tyger-local-gateway) + docker rm -f $container_name &>/dev/null + +# shellcheck disable=SC2086 docker create \ -p $ssh_port:22 \ -e "SSH_ENABLE_ROOT=true" \ -e "TCP_FORWARDING=true" \ - -v "/opt/tyger:/opt/tyger" \ -v "/var/run/docker.sock:/var/run/docker.sock" \ + $gateway_bind_mounts \ --name $container_name \ quay.io/panubo/sshd:1.8.0 >/dev/null @@ -106,7 +111,7 @@ $end_marker" mkdir -p "${HOME}/.ssh" cleanup_ssh_config -echo "$host_config" >> "${HOME}/.ssh/config" +echo "$host_config" >>"${HOME}/.ssh/config" touch "${HOME}/.ssh/known_hosts" ssh-keygen -f "${HOME}/.ssh/known_hosts" -R "$ssh_connection_host" @@ -126,10 +131,14 @@ fi echo "SSH server is ready" -TYGER_CACHE_FILE=$(mktemp) -export TYGER_CACHE_FILE +if [[ -z ${start_only:-} ]]; then + TYGER_CACHE_FILE=$(mktemp) + export TYGER_CACHE_FILE +fi + +tyger_socket_path="$(realpath "$(dirname "$0")/../install/local")/api.sock" -tyger login "ssh://$ssh_host?option[StrictHostKeyChecking]=no" +tyger login "ssh://${ssh_host}${tyger_socket_path}?option[StrictHostKeyChecking]=no" tyger login status if [[ -z ${start_only:-} ]]; then diff --git a/server/ControlPlane/Buffers/Buffers.cs b/server/ControlPlane/Buffers/Buffers.cs index e9ca767b..07bd48c3 100644 --- a/server/ControlPlane/Buffers/Buffers.cs +++ b/server/ControlPlane/Buffers/Buffers.cs @@ -54,7 +54,7 @@ public static void MapBuffers(this WebApplication app) var newBuffer = await context.Request.ReadAndValidateJson(context.RequestAborted); var buffer = await manager.CreateBuffer(newBuffer, cancellationToken); context.Response.Headers.ETag = buffer.ETag; - return Results.CreatedAtRoute("getBufferById", new { buffer.Id }, buffer); + return Results.Created($"/v1/buffers/{buffer.Id}", buffer); }) .Accepts("application/json") .WithName("createBuffer") diff --git a/server/ControlPlane/Compute/Docker/Docker.cs b/server/ControlPlane/Compute/Docker/Docker.cs index d0e9441a..2a860a5d 100644 --- a/server/ControlPlane/Compute/Docker/Docker.cs +++ b/server/ControlPlane/Compute/Docker/Docker.cs @@ -17,7 +17,10 @@ public static void AddDocker(this IHostApplicationBuilder builder) { if (builder is WebApplicationBuilder) { - builder.Services.AddOptions().BindConfiguration("compute:docker").ValidateDataAnnotations().ValidateOnStart(); + builder.Services.AddOptions().BindConfiguration("compute:docker").ValidateDataAnnotations().ValidateOnStart().PostConfigure(options => + { + options.HostPathTranslations = options.HostPathTranslations.ToDictionary(kvp => kvp.Key.EndsWith('/') ? kvp.Key : kvp.Key + "/", kvp => kvp.Value.EndsWith('/') ? kvp.Value : kvp.Value + "/"); + }); } builder.Services.AddSingleton(sp => new DockerClientConfiguration().CreateClient()); @@ -50,4 +53,6 @@ public class DockerOptions [Required] public required string NetworkName { get; set; } + + public Dictionary HostPathTranslations { get; set; } = []; } diff --git a/server/ControlPlane/Compute/Docker/DockerRunCreator.cs b/server/ControlPlane/Compute/Docker/DockerRunCreator.cs index 3ef797d1..3981d6bb 100644 --- a/server/ControlPlane/Compute/Docker/DockerRunCreator.cs +++ b/server/ControlPlane/Compute/Docker/DockerRunCreator.cs @@ -128,12 +128,13 @@ public async Task CreateRun(Run run, string? idempotencyKey, CancellationTo { if (bufferId.StartsWith("temp-", StringComparison.Ordinal)) { + var bufferIdWithoutPrefix = bufferId[5..]; var newBufferId = $"run-{run.Id}-{bufferId}"; run.Job.Buffers[bufferParameterName] = newBufferId; (var write, _) = bufferMap[bufferParameterName]; var unqualifiedBufferId = BufferManager.GetUnqualifiedBufferId(newBufferId); var sasQueryString = _ephemeralBufferProvider.GetSasQueryString(unqualifiedBufferId, write); - var accessUri = new Uri($"http+unix://{_dockerOptions.EphemeralBuffersPath}/{bufferId}.sock:{sasQueryString}"); + var accessUri = new Uri($"http+unix://{_dockerOptions.EphemeralBuffersPath}/{bufferIdWithoutPrefix}:{sasQueryString}"); bufferMap[bufferParameterName] = (write, accessUri); } } @@ -254,21 +255,21 @@ string RelativePipePath(string bufferParameterName) [ new() { - Source = Path.Combine(absoluteSecretsBase, relativePipesPath), + Source = TranslateToHostPath(Path.Combine(absoluteSecretsBase, relativePipesPath)), Target = Path.Combine(absoluteContainerSecretsBase, relativePipesPath), Type = "bind", ReadOnly = false, }, new() { - Source = Path.Combine(absoluteSecretsBase, relativeAccessFilesPath, accessFileName), + Source = TranslateToHostPath(Path.Combine(absoluteSecretsBase, relativeAccessFilesPath, accessFileName)), Target = Path.Combine(absoluteContainerSecretsBase, relativeAccessFilesPath, accessFileName), Type = "bind", ReadOnly = true, }, new() { - Source = Path.Combine(absoluteSecretsBase, relativeTombstonePath), + Source = TranslateToHostPath(Path.Combine(absoluteSecretsBase, relativeTombstonePath)), Target = Path.Combine(absoluteContainerSecretsBase, relativeTombstonePath), Type = "bind", ReadOnly = true, @@ -290,7 +291,7 @@ string RelativePipePath(string bufferParameterName) var socketDir = Path.GetDirectoryName(relaySocketPath)!; sidecarContainerParameters.HostConfig.Mounts.Add(new() { - Source = socketDir, + Source = TranslateToHostPath(socketDir), Target = socketDir, Type = "bind", ReadOnly = false, @@ -307,7 +308,7 @@ string RelativePipePath(string bufferParameterName) var dataPlaneSocket = accessUri.AbsolutePath.Split(':')[0]; sidecarContainerParameters.HostConfig.Mounts.Add(new() { - Source = dataPlaneSocket, + Source = TranslateToHostPath(dataPlaneSocket), Target = dataPlaneSocket, Type = "bind", ReadOnly = false, @@ -360,14 +361,14 @@ string RelativePipePath(string bufferParameterName) [ new() { - Source = Path.Combine(absoluteSecretsBase, relativePipesPath), + Source = TranslateToHostPath(Path.Combine(absoluteSecretsBase, relativePipesPath)), Target = Path.Combine(absoluteContainerSecretsBase, relativePipesPath), Type = "bind", ReadOnly = false, }, new() { - Source = Path.Combine(absoluteSecretsBase, relativeTombstonePath), + Source = TranslateToHostPath(Path.Combine(absoluteSecretsBase, relativeTombstonePath)), Target = Path.Combine(absoluteContainerSecretsBase, relativeTombstonePath), Type = "bind", ReadOnly = true, @@ -409,7 +410,7 @@ string RelativePipePath(string bufferParameterName) [ new() { - Source = Path.Combine(absoluteSecretsBase, relativePipesPath), + Source = TranslateToHostPath(Path.Combine(absoluteSecretsBase, relativePipesPath)), Target = Path.Combine(absoluteContainerSecretsBase, relativePipesPath), Type = "bind", ReadOnly = false, @@ -513,7 +514,6 @@ public static string ExpandVariables(string input, IDictionary e public override async Task StartAsync(CancellationToken cancellationToken) { Directory.CreateDirectory(_dockerOptions.RunSecretsPath); - Directory.CreateDirectory(_dockerOptions.EphemeralBuffersPath); await AddPublicSigningKeyToBufferSidecarImage(cancellationToken); } @@ -577,4 +577,23 @@ private static MemoryStream GetPublicPemStream(string path) pemStream.Position = 0; return pemStream; } + + private string TranslateToHostPath(string path) + { + foreach (var (source, dest) in _dockerOptions.HostPathTranslations) + { + if (path.StartsWith(source, StringComparison.Ordinal)) + { + return path.Replace(source, dest); + } + + // source ends with a '/' + if (path.Length + 1 == source.Length && path.Equals(source[..^1], StringComparison.Ordinal)) + { + return dest[..^1]; + } + } + + return path; + } }