-
-
Notifications
You must be signed in to change notification settings - Fork 254
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
patch and rebuild cmd/link to modify the magic value in pclntab
This value is hard-coded in the linker and written in a header. We could rewrite the final binary, like we used to do with import paths, but that would require once again maintaining libraries to do so. Instead, we're now modifying the linker to do what we want. It's not particularly hard, as every Go install has its source code, and rebuilding a slightly modified linker only takes a few seconds at most. Thanks to `go build -overlay`, we only need to copy the files we modify, and right now we're just modifying one file in the toolchain. We use a git patch, as the change is fairly static and small, and the patch is easier to understand and maintain. The other side of this change is in the runtime, as it also hard-codes the magic value when loading information. We modify the code via syntax trees in that case, like `-tiny` does, because the change is tiny (one literal) and the affected lines of code are modified regularly between major Go releases. Since rebuilding a slightly modified linker can take a few seconds, and Go's build cache does not cache linked binaries, we keep our own cached version of the rebuilt binary in `os.UserCacheDir`. The feature isn't perfect, and will be improved in the future. See the TODOs about the added dependency on `git`, or how we are currently only able to cache one linker binary at once. Fixes #622.
- Loading branch information
Showing
11 changed files
with
434 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,250 @@ | ||
| // Copyright (c) 2022, The Garble Authors. | ||
| // See LICENSE for licensing information. | ||
|
|
||
| package linker | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "crypto/sha256" | ||
| "embed" | ||
| "encoding/base64" | ||
| "encoding/json" | ||
| "fmt" | ||
| "io" | ||
| "io/fs" | ||
| "os" | ||
| "os/exec" | ||
| "path/filepath" | ||
| "runtime" | ||
| "strings" | ||
|
|
||
| "github.com/bluekeyes/go-gitdiff/gitdiff" | ||
| "github.com/rogpeppe/go-internal/lockedfile" | ||
| ) | ||
|
|
||
| const ( | ||
| MagicValueEnv = "GARBLE_LINKER_MAGIC" | ||
|
|
||
| cacheDirName = "garble" | ||
| versionExt = ".version" | ||
| garbleCacheDir = "GARBLE_CACHE_DIR" | ||
| baseSrcSubdir = "src" | ||
| ) | ||
|
|
||
| var ( | ||
| //go:embed patches/*.patch | ||
| linkerPatchesFS embed.FS | ||
| ) | ||
|
|
||
| func loadLinkerPatches() (string, map[string]string, error) { | ||
| versionHash := sha256.New() | ||
| patches := make(map[string]string) | ||
| err := fs.WalkDir(linkerPatchesFS, ".", func(path string, d fs.DirEntry, err error) error { | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if d.IsDir() { | ||
| return nil | ||
| } | ||
|
|
||
| patchBytes, err := linkerPatchesFS.ReadFile(path) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if _, err := versionHash.Write(patchBytes); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| files, _, err := gitdiff.Parse(bytes.NewReader(patchBytes)) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| for _, file := range files { | ||
| if file.IsNew || file.IsDelete || file.IsCopy || file.IsRename { | ||
| panic("only modification patch is supported") | ||
| } | ||
| patches[file.OldName] = string(patchBytes) | ||
| } | ||
| return nil | ||
| }) | ||
|
|
||
| if err != nil { | ||
| return "", nil, err | ||
| } | ||
| return base64.RawStdEncoding.EncodeToString(versionHash.Sum(nil)), patches, nil | ||
| } | ||
|
|
||
| // TODO(pagran): Remove git dependency in future | ||
| // more information in README.md | ||
| func applyPatch(workingDir, patch string) error { | ||
| cmd := exec.Command("git", "-C", workingDir, "apply") | ||
| cmd.Stdin = strings.NewReader(patch) | ||
| return cmd.Run() | ||
| } | ||
|
|
||
| func copyFile(src, target string) error { | ||
| targetDir := filepath.Dir(target) | ||
| if err := os.MkdirAll(targetDir, 0o777); err != nil { | ||
| return err | ||
| } | ||
| srcFile, err := os.Open(src) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer srcFile.Close() | ||
|
|
||
| targetFile, err := os.Create(target) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer targetFile.Close() | ||
| _, err = io.Copy(targetFile, srcFile) | ||
| return err | ||
| } | ||
|
|
||
| func fileExists(path string) bool { | ||
| stat, err := os.Stat(path) | ||
| if err != nil { | ||
| return false | ||
| } | ||
| return !stat.IsDir() | ||
| } | ||
|
|
||
| func applyPatches(srcDir, workingDir string, patches map[string]string) (map[string]string, error) { | ||
| mod := make(map[string]string) | ||
| for fileName, patch := range patches { | ||
| oldPath := filepath.Join(srcDir, fileName) | ||
| newPath := filepath.Join(workingDir, fileName) | ||
| mod[oldPath] = newPath | ||
|
|
||
| if err := copyFile(oldPath, newPath); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| if err := applyPatch(workingDir, patch); err != nil { | ||
| return nil, fmt.Errorf("apply patch for %s failed: %v", fileName, err) | ||
| } | ||
| } | ||
| return mod, nil | ||
| } | ||
|
|
||
| func cachePath(goExe string) (string, error) { | ||
| var cacheDir string | ||
| if val, ok := os.LookupEnv(garbleCacheDir); ok { | ||
| cacheDir = val | ||
| } else { | ||
| userCacheDir, err := os.UserCacheDir() | ||
| if err != nil { | ||
| panic(fmt.Errorf("cannot retreive user cache directory: %v", err)) | ||
| } | ||
| cacheDir = userCacheDir | ||
| } | ||
|
|
||
| cacheDir = filepath.Join(cacheDir, cacheDirName) | ||
| if err := os.MkdirAll(cacheDir, 0o777); err != nil { | ||
| return "", err | ||
| } | ||
|
|
||
| // Note that we only keep one patched and built linker in the cache. | ||
| // If the user switches between Go versions or garble versions often, | ||
| // this may result in rebuilds since we don't keep multiple binaries in the cache. | ||
| // We can consider keeping multiple versions of the binary in our cache in the future, | ||
| // similar to how GOCACHE works with multiple built versions of the same package. | ||
| return filepath.Join(cacheDir, "link"+goExe), nil | ||
| } | ||
|
|
||
| func getCurrentVersion(goVersion, patchesVer string) string { | ||
| return goVersion + " " + patchesVer | ||
| } | ||
|
|
||
| func checkVersion(linkerPath, goVersion, patchesVer string) (bool, error) { | ||
| versionPath := linkerPath + versionExt | ||
| version, err := os.ReadFile(versionPath) | ||
| if os.IsNotExist(err) { | ||
| return false, nil | ||
| } | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| return string(version) == getCurrentVersion(goVersion, patchesVer), nil | ||
| } | ||
|
|
||
| func writeVersion(linkerPath, goVersion, patchesVer string) error { | ||
| versionPath := linkerPath + versionExt | ||
| return os.WriteFile(versionPath, []byte(getCurrentVersion(goVersion, patchesVer)), 0o777) | ||
| } | ||
|
|
||
| func buildLinker(workingDir string, overlay map[string]string, outputLinkPath string) error { | ||
| file, err := json.Marshal(&struct{ Replace map[string]string }{overlay}) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| overlayPath := filepath.Join(workingDir, "overlay.json") | ||
| if err := os.WriteFile(overlayPath, file, 0o777); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| cmd := exec.Command("go", "build", "-overlay", overlayPath, "-o", outputLinkPath, "cmd/link") | ||
| // Explicitly setting GOOS and GOARCH variables prevents conflicts during cross-build | ||
| cmd.Env = append(os.Environ(), "GOOS="+runtime.GOOS, "GOARCH="+runtime.GOARCH) | ||
|
|
||
| out, err := cmd.CombinedOutput() | ||
| if err != nil { | ||
| return fmt.Errorf("compiler compile error: %v\n\n%s", err, string(out)) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func PatchLinker(goRoot, goVersion, goExe, tempDir string) (string, func(), error) { | ||
| patchesVer, patches, err := loadLinkerPatches() | ||
| if err != nil { | ||
| panic(fmt.Errorf("cannot retrieve linker patches: %v", err)) | ||
| } | ||
|
|
||
| outputLinkPath, err := cachePath(goExe) | ||
| if err != nil { | ||
| return "", nil, err | ||
| } | ||
|
|
||
| mutex := lockedfile.MutexAt(outputLinkPath + ".lock") | ||
| unlock, err := mutex.Lock() | ||
| if err != nil { | ||
| return "", nil, err | ||
| } | ||
|
|
||
| // If build is successful, mutex unlocking must be on the caller's side | ||
| successBuild := false | ||
| defer func() { | ||
| if !successBuild { | ||
| unlock() | ||
| } | ||
| }() | ||
|
|
||
| isCorrectVer, err := checkVersion(outputLinkPath, goVersion, patchesVer) | ||
| if err != nil { | ||
| return "", nil, err | ||
| } | ||
| if isCorrectVer && fileExists(outputLinkPath) { | ||
| successBuild = true | ||
| return outputLinkPath, unlock, nil | ||
| } | ||
|
|
||
| srcDir := filepath.Join(goRoot, baseSrcSubdir) | ||
| workingDir := filepath.Join(tempDir, "linker-src") | ||
|
|
||
| overlay, err := applyPatches(srcDir, workingDir, patches) | ||
| if err != nil { | ||
| return "", nil, err | ||
| } | ||
| if err := buildLinker(workingDir, overlay, outputLinkPath); err != nil { | ||
| return "", nil, err | ||
| } | ||
| if err := writeVersion(outputLinkPath, goVersion, patchesVer); err != nil { | ||
| return "", nil, err | ||
| } | ||
| successBuild = true | ||
| return outputLinkPath, unlock, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| From de93a968f1bb3500088b30cbdce439e6a0d95e58 Mon Sep 17 00:00:00 2001 | ||
| From: pagran <pagran@protonmail.com> | ||
| Date: Sun, 8 Jan 2023 14:12:51 +0100 | ||
| Subject: [PATCH 1/1] add custom magic value | ||
|
|
||
| --- | ||
| cmd/link/internal/ld/pcln.go | 13 +++++++++++++ | ||
| 1 file changed, 13 insertions(+) | ||
|
|
||
| diff --git a/cmd/link/internal/ld/pcln.go b/cmd/link/internal/ld/pcln.go | ||
| index 34ab86cf12..1ec237ffc8 100644 | ||
| --- a/cmd/link/internal/ld/pcln.go | ||
| +++ b/cmd/link/internal/ld/pcln.go | ||
| @@ -249,6 +249,19 @@ func (state *pclntab) generatePCHeader(ctxt *Link) { | ||
| if off != size { | ||
| panic(fmt.Sprintf("pcHeader size: %d != %d", off, size)) | ||
| } | ||
| + | ||
| + // Use garble prefix in variable names to minimize collision risk | ||
| + garbleMagicStr := os.Getenv("GARBLE_LINKER_MAGIC") | ||
| + if garbleMagicStr == "" { | ||
| + panic("[garble] magic value must be set") | ||
| + } | ||
| + var garbleMagicVal uint32 | ||
| + // Use fmt package instead of strconv to avoid importing a new package | ||
| + if _, err := fmt.Sscan(garbleMagicStr, &garbleMagicVal); err != nil { | ||
| + panic(fmt.Errorf("[garble] invalid magic value %s: %v", garbleMagicStr, err)) | ||
| + } | ||
| + | ||
| + header.SetUint32(ctxt.Arch, 0, garbleMagicVal) | ||
| } | ||
|
|
||
| state.pcheader = state.addGeneratedSym(ctxt, "runtime.pcheader", size, writeHeader) | ||
| -- | ||
| 2.38.1.windows.1 | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.