diff --git a/cmd/internal/pkgsite/server.go b/cmd/internal/pkgsite/server.go new file mode 100644 index 000000000..aa57f404a --- /dev/null +++ b/cmd/internal/pkgsite/server.go @@ -0,0 +1,338 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgsite + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/fs" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "time" + + "github.com/google/safehtml/template" + "golang.org/x/pkgsite/internal" + "golang.org/x/pkgsite/internal/fetch" + "golang.org/x/pkgsite/internal/fetchdatasource" + "golang.org/x/pkgsite/internal/frontend" + "golang.org/x/pkgsite/internal/log" + "golang.org/x/pkgsite/internal/proxy" + "golang.org/x/pkgsite/internal/source" + "golang.org/x/pkgsite/static" + thirdparty "golang.org/x/pkgsite/third_party" +) + +// ServerConfig provides configuration for BuildServer. +type ServerConfig struct { + Paths []string + GOPATHMode bool + UseCache bool + CacheDir string + UseListedMods bool + UseLocalStdlib bool + DevMode bool + DevModeStaticDir string + GoRepoPath string + + Proxy *proxy.Client // client, or nil; controlled by the -proxy flag +} + +// BuildServer builds a *frontend.Server using the given configuration. +func BuildServer(ctx context.Context, serverCfg ServerConfig) (*frontend.Server, error) { + if len(serverCfg.Paths) == 0 && !serverCfg.UseCache && serverCfg.Proxy == nil { + serverCfg.Paths = []string{"."} + } + + cfg := getterConfig{ + all: serverCfg.UseListedMods, + proxy: serverCfg.Proxy, + goRepoPath: serverCfg.GoRepoPath, + } + + // By default, the requested Paths are interpreted as directories. However, + // if -gopath_mode is set, they are interpreted as relative Paths to modules + // in a GOPATH directory. + if serverCfg.GOPATHMode { + var err error + cfg.dirs, err = getGOPATHModuleDirs(ctx, serverCfg.Paths) + if err != nil { + return nil, fmt.Errorf("searching GOPATH: %v", err) + } + } else { + var err error + cfg.dirs, err = getModuleDirs(ctx, serverCfg.Paths) + if err != nil { + return nil, fmt.Errorf("searching GOPATH: %v", err) + } + } + + if serverCfg.UseCache { + cfg.modCacheDir = serverCfg.CacheDir + if cfg.modCacheDir == "" { + var err error + cfg.modCacheDir, err = defaultCacheDir() + if err != nil { + return nil, err + } + if cfg.modCacheDir == "" { + return nil, fmt.Errorf("empty value for GOMODCACHE") + } + } + } + + if serverCfg.UseLocalStdlib { + cfg.useLocalStdlib = true + } + + getters, err := buildGetters(ctx, cfg) + if err != nil { + return nil, err + } + + // Collect unique module Paths served by this server. + seenModules := make(map[frontend.LocalModule]bool) + var allModules []frontend.LocalModule + for _, modules := range cfg.dirs { + for _, m := range modules { + if seenModules[m] { + continue + } + seenModules[m] = true + allModules = append(allModules, m) + } + } + sort.Slice(allModules, func(i, j int) bool { + return allModules[i].ModulePath < allModules[j].ModulePath + }) + + return newServer(getters, allModules, cfg.proxy, serverCfg.DevMode, serverCfg.DevModeStaticDir) +} + +// getModuleDirs returns the set of workspace modules for each directory, +// determined by running go list -m. +// +// An error is returned if any operations failed unexpectedly, or if no +// requested directories contain any valid modules. +func getModuleDirs(ctx context.Context, dirs []string) (map[string][]frontend.LocalModule, error) { + dirModules := make(map[string][]frontend.LocalModule) + for _, dir := range dirs { + output, err := runGo(dir, "list", "-m", "-json") + if err != nil { + return nil, fmt.Errorf("listing modules in %s: %v", dir, err) + } + var modules []frontend.LocalModule + decoder := json.NewDecoder(bytes.NewBuffer(output)) + for decoder.More() { + var m frontend.LocalModule + if err := decoder.Decode(&m); err != nil { + return nil, err + } + if m.ModulePath != "command-line-arguments" { + modules = append(modules, m) + } + } + if len(modules) > 0 { + dirModules[dir] = modules + } + } + if len(dirs) > 0 && len(dirModules) == 0 { + return nil, fmt.Errorf("no modules in any of the requested directories") + } + return dirModules, nil +} + +// getGOPATHModuleDirs returns local module information for directories in +// GOPATH corresponding to the requested module Paths. +// +// An error is returned if any operations failed unexpectedly, or if no modules +// were resolved. If individual module Paths are not found, an error is logged +// and the path skipped. +func getGOPATHModuleDirs(ctx context.Context, modulePaths []string) (map[string][]frontend.LocalModule, error) { + gopath, err := runGo("", "env", "GOPATH") + if err != nil { + return nil, err + } + gopaths := filepath.SplitList(strings.TrimSpace(string(gopath))) + + dirs := make(map[string][]frontend.LocalModule) + for _, path := range modulePaths { + dir := "" + for _, gopath := range gopaths { + candidate := filepath.Join(gopath, "src", path) + info, err := os.Stat(candidate) + if err == nil && info.IsDir() { + dir = candidate + break + } + if err != nil && !os.IsNotExist(err) { + return nil, err + } + } + if dir == "" { + log.Errorf(ctx, "ERROR: no GOPATH directory contains %q", path) + } else { + dirs[dir] = []frontend.LocalModule{{ModulePath: path, Dir: dir}} + } + } + + if len(modulePaths) > 0 && len(dirs) == 0 { + return nil, fmt.Errorf("no GOPATH directories contain any of the requested module(s)") + } + return dirs, nil +} + +// getterConfig defines the set of getters for the server to use. +// See buildGetters. +type getterConfig struct { + all bool // if set, request "all" instead of ["/..."] + dirs map[string][]frontend.LocalModule // local modules to serve + modCacheDir string // path to module cache, or "" + proxy *proxy.Client // proxy client, or nil + useLocalStdlib bool // use go/packages for the local stdlib + goRepoPath string // repo path for local stdlib +} + +// buildGetters constructs module getters based on the given configuration. +// +// Getters are returned in the following priority order: +// 1. local getters for cfg.dirs, in the given order +// 2. a module cache getter, if cfg.modCacheDir != "" +// 3. a proxy getter, if cfg.proxy != nil +func buildGetters(ctx context.Context, cfg getterConfig) ([]fetch.ModuleGetter, error) { + var getters []fetch.ModuleGetter + + // Load local getters for each directory. + for dir, modules := range cfg.dirs { + var patterns []string + if cfg.all { + patterns = append(patterns, "all") + } else { + for _, m := range modules { + patterns = append(patterns, fmt.Sprintf("%s/...", m)) + } + } + mg, err := fetch.NewGoPackagesModuleGetter(ctx, dir, patterns...) + if err != nil { + log.Errorf(ctx, "Loading packages from %s: %v", dir, err) + } else { + getters = append(getters, mg) + } + } + if len(getters) == 0 && len(cfg.dirs) > 0 { + return nil, fmt.Errorf("failed to load any module(s) at %v", cfg.dirs) + } + + // Add a getter for the local module cache. + if cfg.modCacheDir != "" { + g, err := fetch.NewModCacheGetter(cfg.modCacheDir) + if err != nil { + return nil, err + } + getters = append(getters, g) + } + + if cfg.useLocalStdlib { + goRepo := cfg.goRepoPath + if goRepo == "" { + goRepo = getGOROOT() + } + if goRepo != "" { // if goRepo == "" we didn't get a *goRepoPath and couldn't find GOROOT. Fall back to the zip files. + mg, err := fetch.NewGoPackagesStdlibModuleGetter(ctx, goRepo) + if err != nil { + log.Errorf(ctx, "loading packages from stdlib: %v", err) + } else { + getters = append(getters, mg) + } + } + } + + // Add a proxy + if cfg.proxy != nil { + getters = append(getters, fetch.NewProxyModuleGetter(cfg.proxy, source.NewClient(&http.Client{Timeout: time.Second}))) + } + + getters = append(getters, fetch.NewStdlibZipModuleGetter()) + + return getters, nil +} + +func getGOROOT() string { + if rg := runtime.GOROOT(); rg != "" { + return rg + } + b, err := exec.Command("go", "env", "GOROOT").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + +func newServer(getters []fetch.ModuleGetter, localModules []frontend.LocalModule, prox *proxy.Client, devMode bool, staticFlag string) (*frontend.Server, error) { + lds := fetchdatasource.Options{ + Getters: getters, + ProxyClientForLatest: prox, + BypassLicenseCheck: true, + }.New() + + // In dev mode, use a dirFS to pick up template/JS/CSS changes without + // restarting the server. + var staticFS fs.FS + if devMode { + staticFS = os.DirFS(staticFlag) + } else { + staticFS = static.FS + } + + // Preload local modules to warm the cache. + for _, lm := range localModules { + go lds.GetUnitMeta(context.Background(), "", lm.ModulePath, fetch.LocalVersion) + } + go lds.GetUnitMeta(context.Background(), "", "std", "latest") + + server, err := frontend.NewServer(frontend.ServerConfig{ + DataSourceGetter: func(context.Context) internal.DataSource { return lds }, + TemplateFS: template.TrustedFSFromEmbed(static.FS), + StaticFS: staticFS, + DevMode: devMode, + LocalMode: true, + LocalModules: localModules, + ThirdPartyFS: thirdparty.FS, + }) + if err != nil { + return nil, err + } + for _, g := range getters { + p, fsys := g.SourceFS() + if p != "" { + server.InstallFS(p, fsys) + } + } + return server, nil +} + +func defaultCacheDir() (string, error) { + out, err := runGo("", "env", "GOMODCACHE") + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} + +func runGo(dir string, args ...string) ([]byte, error) { + cmd := exec.Command("go", args...) + cmd.Dir = dir + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("running go with %q: %v: %s", args, err, out) + } + return out, nil +} diff --git a/cmd/internal/pkgsite/server_test.go b/cmd/internal/pkgsite/server_test.go new file mode 100644 index 000000000..49d09213d --- /dev/null +++ b/cmd/internal/pkgsite/server_test.go @@ -0,0 +1,230 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgsite + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "regexp" + "testing" + + "golang.org/x/net/html" + "golang.org/x/pkgsite/internal/proxy/proxytest" + "golang.org/x/pkgsite/internal/testenv" + "golang.org/x/pkgsite/internal/testing/htmlcheck" + "golang.org/x/pkgsite/internal/testing/testhelper" +) + +var ( + in = htmlcheck.In + hasText = htmlcheck.HasText + attr = htmlcheck.HasAttr + + // href checks for an exact match in an href attribute. + href = func(val string) htmlcheck.Checker { + return attr("href", "^"+regexp.QuoteMeta(val)+"$") + } +) + +func TestServer(t *testing.T) { + testenv.MustHaveExecPath(t, "go") // for local modules + + repoPath := func(fn string) string { return filepath.Join("..", "..", "..", fn) } + + abs := func(dir string) string { + a, err := filepath.Abs(dir) + if err != nil { + t.Fatal(err) + } + return a + } + + localModule, _ := testhelper.WriteTxtarToTempDir(t, ` +-- go.mod -- +module example.com/testmod +-- a.go -- +package a +`) + cacheDir := repoPath("internal/fetch/testdata/modcache") + testModules := proxytest.LoadTestModules(repoPath("internal/proxy/testdata")) + prox, teardown := proxytest.SetupTestClient(t, testModules) + defer teardown() + + cfg := func(modifyDefault func(*ServerConfig)) ServerConfig { + c := ServerConfig{ + Paths: []string{localModule}, + GOPATHMode: false, + UseListedMods: true, + UseCache: true, + CacheDir: cacheDir, + Proxy: prox, + } + if modifyDefault != nil { + modifyDefault(&c) + } + return c + } + + modcacheChecker := in("", + in(".Documentation", hasText("var V = 1")), + sourceLinks(path.Join(filepath.ToSlash(abs(cacheDir)), "modcache.com@v1.0.0"), "a.go")) + + ctx := context.Background() + for _, test := range []struct { + name string + cfg ServerConfig + url string + wantCode int + want htmlcheck.Checker + }{ + { + "local", + cfg(nil), + "example.com/testmod", + http.StatusOK, + in("", + in(".Documentation", hasText("There is no documentation for this package.")), + sourceLinks(path.Join(filepath.ToSlash(abs(localModule)), "example.com/testmod"), "a.go")), + }, + { + "modcache", + cfg(nil), + "modcache.com@v1.0.0", + http.StatusOK, + modcacheChecker, + }, + { + "modcache latest", + cfg(nil), + "modcache.com", + http.StatusOK, + modcacheChecker, + }, + { + "modcache unsupported", + cfg(func(c *ServerConfig) { + c.UseCache = false + }), + "modcache.com", + http.StatusFailedDependency, // TODO(rfindley): should this be 404? + hasText("page is not supported"), + }, + { + "proxy", + cfg(nil), + "example.com/single/pkg", + http.StatusOK, + hasText("G is new in v1.1.0"), + }, + { + "proxy unsupported", + cfg(func(c *ServerConfig) { + c.Proxy = nil + }), + "example.com/single/pkg", + http.StatusFailedDependency, // TODO(rfindley): should this be 404? + hasText("page is not supported"), + }, + { + "search", + cfg(func(c *ServerConfig) { + c.UseLocalStdlib = false + }), + "search?q=a", + http.StatusOK, + in(".SearchResults", + hasText("example.com/testmod"), + ), + }, + { + "no symbol search", + cfg(func(c *ServerConfig) { + c.UseLocalStdlib = false + }), + "search?q=A", // using a capital letter should not cause symbol search + http.StatusOK, + in(".SearchResults", + hasText("example.com/testmod"), + ), + }, + { + "search not found", + cfg(func(c *ServerConfig) { + c.UseLocalStdlib = false + }), + "search?q=zzz", + http.StatusOK, + in(".SearchResults", + hasText("no matches"), + ), + }, + { + "search vulns not found", + cfg(nil), + "search?q=GO-1234-1234", + http.StatusOK, + in(".SearchResults", + hasText("no matches"), + ), + }, + { + "search unsupported", + cfg(func(c *ServerConfig) { + c.Paths = nil + c.UseLocalStdlib = false + }), + "search?q=zzz", + http.StatusFailedDependency, + hasText("page is not supported"), + }, + { + "vulns unsupported", + cfg(nil), + "vuln/", + http.StatusFailedDependency, + hasText("page is not supported"), + }, + // TODO(rfindley): add a test for the standard library once it doesn't go + // through the stdlib package. + // See also golang/go#58923. + } { + t.Run(test.name, func(t *testing.T) { + server, err := BuildServer(ctx, test.cfg) + if err != nil { + t.Fatal(err) + } + mux := http.NewServeMux() + server.Install(mux.Handle, nil, nil) + + w := httptest.NewRecorder() + mux.ServeHTTP(w, httptest.NewRequest("GET", "/"+test.url, nil)) + if w.Code != test.wantCode { + t.Fatalf("got status code = %d, want %d", w.Code, test.wantCode) + } + doc, err := html.Parse(w.Body) + if err != nil { + t.Fatal(err) + } + if err := test.want(doc); err != nil { + if testing.Verbose() { + html.Render(os.Stdout, doc) + } + t.Error(err) + } + }) + } +} + +func sourceLinks(dir, filename string) htmlcheck.Checker { + filesPath := path.Join("/files", dir) + "/" + return in("", + in(".UnitMeta-repo a", href(filesPath)), + in(".UnitFiles-titleLink a", href(filesPath)), + in(".UnitFiles-fileList a", href(filesPath+filename))) +} diff --git a/cmd/pkgsite/main.go b/cmd/pkgsite/main.go index f9eae158b..50b694442 100644 --- a/cmd/pkgsite/main.go +++ b/cmd/pkgsite/main.go @@ -49,35 +49,21 @@ package main import ( - "bytes" "context" - "encoding/json" "flag" "fmt" - "io/fs" "net" "net/http" "os" - "os/exec" - "path/filepath" - "runtime" - "sort" "strings" "time" - "github.com/google/safehtml/template" - "golang.org/x/pkgsite/internal" + "golang.org/x/pkgsite/cmd/internal/pkgsite" "golang.org/x/pkgsite/internal/browser" - "golang.org/x/pkgsite/internal/fetch" - "golang.org/x/pkgsite/internal/fetchdatasource" - "golang.org/x/pkgsite/internal/frontend" "golang.org/x/pkgsite/internal/log" "golang.org/x/pkgsite/internal/middleware/timeout" "golang.org/x/pkgsite/internal/proxy" - "golang.org/x/pkgsite/internal/source" "golang.org/x/pkgsite/internal/stdlib" - "golang.org/x/pkgsite/static" - thirdparty "golang.org/x/pkgsite/third_party" ) const defaultAddr = "localhost:8080" // default webserver address @@ -86,31 +72,21 @@ var ( httpAddr = flag.String("http", defaultAddr, "HTTP service address to listen for incoming requests on") goRepoPath = flag.String("gorepo", "", "path to Go repo on local filesystem") useProxy = flag.Bool("proxy", false, "fetch from GOPROXY if not found locally") - devMode = flag.Bool("dev", false, "enable developer mode (reload templates on each page load, serve non-minified JS/CSS, etc.)") - staticFlag = flag.String("static", "static", "path to folder containing static files served") openFlag = flag.Bool("open", false, "open a browser window to the server's address") - // other flags are bound to serverConfig below + // other flags are bound to ServerConfig below ) -type serverConfig struct { - paths []string - gopathMode bool - useCache bool - cacheDir string - useListedMods bool - useLocalStdlib bool - - proxy *proxy.Client // client, or nil; controlled by the -proxy flag -} - func main() { - var serverCfg serverConfig + var serverCfg pkgsite.ServerConfig - flag.BoolVar(&serverCfg.gopathMode, "gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src") - flag.BoolVar(&serverCfg.useCache, "cache", false, "fetch from the module cache") - flag.StringVar(&serverCfg.cacheDir, "cachedir", "", "module cache directory (defaults to `go env GOMODCACHE`)") - flag.BoolVar(&serverCfg.useListedMods, "list", true, "for each path, serve all modules in build list") - serverCfg.useLocalStdlib = true + flag.BoolVar(&serverCfg.GOPATHMode, "gopath_mode", false, "assume that local modules' Paths are relative to GOPATH/src") + flag.BoolVar(&serverCfg.UseCache, "cache", false, "fetch from the module cache") + flag.StringVar(&serverCfg.CacheDir, "cachedir", "", "module cache directory (defaults to `go env GOMODCACHE`)") + flag.BoolVar(&serverCfg.UseListedMods, "list", true, "for each path, serve all modules in build list") + flag.BoolVar(&serverCfg.DevMode, "dev", false, "enable developer mode (reload templates on each page load, serve non-minified JS/CSS, etc.)") + flag.StringVar(&serverCfg.DevModeStaticDir, "static", "static", "path to folder containing static files served") + serverCfg.UseLocalStdlib = true + serverCfg.GoRepoPath = *goRepoPath flag.Usage = func() { out := flag.CommandLine.Output() @@ -120,9 +96,9 @@ func main() { flag.PrintDefaults() } flag.Parse() - serverCfg.paths = collectPaths(flag.Args()) + serverCfg.Paths = collectPaths(flag.Args()) - if serverCfg.useCache || *useProxy { + if serverCfg.UseCache || *useProxy { fmt.Fprintf(os.Stderr, "BYPASSING LICENSE CHECKING: MAY DISPLAY NON-REDISTRIBUTABLE INFORMATION\n") } @@ -132,7 +108,7 @@ func main() { die("GOPROXY environment variable is not set") } var err error - serverCfg.proxy, err = proxy.New(url, nil) + serverCfg.Proxy, err = proxy.New(url, nil) if err != nil { die("connecting to proxy: %s", err) } @@ -143,7 +119,7 @@ func main() { } ctx := context.Background() - server, err := buildServer(ctx, serverCfg) + server, err := pkgsite.BuildServer(ctx, serverCfg) if err != nil { die(err.Error()) } @@ -182,75 +158,6 @@ func die(format string, args ...any) { os.Exit(1) } -func buildServer(ctx context.Context, serverCfg serverConfig) (*frontend.Server, error) { - if len(serverCfg.paths) == 0 && !serverCfg.useCache && serverCfg.proxy == nil { - serverCfg.paths = []string{"."} - } - - cfg := getterConfig{ - all: serverCfg.useListedMods, - proxy: serverCfg.proxy, - } - - // By default, the requested paths are interpreted as directories. However, - // if -gopath_mode is set, they are interpreted as relative paths to modules - // in a GOPATH directory. - if serverCfg.gopathMode { - var err error - cfg.dirs, err = getGOPATHModuleDirs(ctx, serverCfg.paths) - if err != nil { - return nil, fmt.Errorf("searching GOPATH: %v", err) - } - } else { - var err error - cfg.dirs, err = getModuleDirs(ctx, serverCfg.paths) - if err != nil { - return nil, fmt.Errorf("searching GOPATH: %v", err) - } - } - - if serverCfg.useCache { - cfg.modCacheDir = serverCfg.cacheDir - if cfg.modCacheDir == "" { - var err error - cfg.modCacheDir, err = defaultCacheDir() - if err != nil { - return nil, err - } - if cfg.modCacheDir == "" { - return nil, fmt.Errorf("empty value for GOMODCACHE") - } - } - } - - if serverCfg.useLocalStdlib { - cfg.useLocalStdlib = true - } - - getters, err := buildGetters(ctx, cfg) - if err != nil { - return nil, err - } - - // Collect unique module paths served by this server. - seenModules := make(map[frontend.LocalModule]bool) - var allModules []frontend.LocalModule - for _, modules := range cfg.dirs { - for _, m := range modules { - if seenModules[m] { - continue - } - seenModules[m] = true - allModules = append(allModules, m) - } - } - sort.Slice(allModules, func(i, j int) bool { - return allModules[i].ModulePath < allModules[j].ModulePath - }) - - return newServer(getters, allModules, cfg.proxy) -} - func collectPaths(args []string) []string { var paths []string for _, arg := range args { @@ -258,222 +165,3 @@ func collectPaths(args []string) []string { } return paths } - -// getGOPATHModuleDirs returns the set of workspace modules for each directory, -// determined by running go list -m. -// -// An error is returned if any operations failed unexpectedly, or if no -// requested directories contain any valid modules. -func getModuleDirs(ctx context.Context, dirs []string) (map[string][]frontend.LocalModule, error) { - dirModules := make(map[string][]frontend.LocalModule) - for _, dir := range dirs { - output, err := runGo(dir, "list", "-m", "-json") - if err != nil { - return nil, fmt.Errorf("listing modules in %s: %v", dir, err) - } - var modules []frontend.LocalModule - decoder := json.NewDecoder(bytes.NewBuffer(output)) - for decoder.More() { - var m frontend.LocalModule - if err := decoder.Decode(&m); err != nil { - return nil, err - } - if m.ModulePath != "command-line-arguments" { - modules = append(modules, m) - } - } - if len(modules) > 0 { - dirModules[dir] = modules - } - } - if len(dirs) > 0 && len(dirModules) == 0 { - return nil, fmt.Errorf("no modules in any of the requested directories") - } - return dirModules, nil -} - -// getGOPATHModuleDirs returns local module information for directories in -// GOPATH corresponding to the requested module paths. -// -// An error is returned if any operations failed unexpectedly, or if no modules -// were resolved. If individual module paths are not found, an error is logged -// and the path skipped. -func getGOPATHModuleDirs(ctx context.Context, modulePaths []string) (map[string][]frontend.LocalModule, error) { - gopath, err := runGo("", "env", "GOPATH") - if err != nil { - return nil, err - } - gopaths := filepath.SplitList(strings.TrimSpace(string(gopath))) - - dirs := make(map[string][]frontend.LocalModule) - for _, path := range modulePaths { - dir := "" - for _, gopath := range gopaths { - candidate := filepath.Join(gopath, "src", path) - info, err := os.Stat(candidate) - if err == nil && info.IsDir() { - dir = candidate - break - } - if err != nil && !os.IsNotExist(err) { - return nil, err - } - } - if dir == "" { - log.Errorf(ctx, "ERROR: no GOPATH directory contains %q", path) - } else { - dirs[dir] = []frontend.LocalModule{{ModulePath: path, Dir: dir}} - } - } - - if len(modulePaths) > 0 && len(dirs) == 0 { - return nil, fmt.Errorf("no GOPATH directories contain any of the requested module(s)") - } - return dirs, nil -} - -// getterConfig defines the set of getters for the server to use. -// See buildGetters. -type getterConfig struct { - all bool // if set, request "all" instead of ["/..."] - dirs map[string][]frontend.LocalModule // local modules to serve - modCacheDir string // path to module cache, or "" - proxy *proxy.Client // proxy client, or nil - useLocalStdlib bool // use go/packages for the local stdlib -} - -// buildGetters constructs module getters based on the given configuration. -// -// Getters are returned in the following priority order: -// 1. local getters for cfg.dirs, in the given order -// 2. a module cache getter, if cfg.modCacheDir != "" -// 3. a proxy getter, if cfg.proxy != nil -func buildGetters(ctx context.Context, cfg getterConfig) ([]fetch.ModuleGetter, error) { - var getters []fetch.ModuleGetter - - // Load local getters for each directory. - for dir, modules := range cfg.dirs { - var patterns []string - if cfg.all { - patterns = append(patterns, "all") - } else { - for _, m := range modules { - patterns = append(patterns, fmt.Sprintf("%s/...", m)) - } - } - mg, err := fetch.NewGoPackagesModuleGetter(ctx, dir, patterns...) - if err != nil { - log.Errorf(ctx, "Loading packages from %s: %v", dir, err) - } else { - getters = append(getters, mg) - } - } - if len(getters) == 0 && len(cfg.dirs) > 0 { - return nil, fmt.Errorf("failed to load any module(s) at %v", cfg.dirs) - } - - // Add a getter for the local module cache. - if cfg.modCacheDir != "" { - g, err := fetch.NewModCacheGetter(cfg.modCacheDir) - if err != nil { - return nil, err - } - getters = append(getters, g) - } - - if cfg.useLocalStdlib { - goRepo := *goRepoPath - if goRepo == "" { - goRepo = getGOROOT() - } - if goRepo != "" { // if goRepo == "" we didn't get a *goRepoPath and couldn't find GOROOT. Fall back to the zip files. - mg, err := fetch.NewGoPackagesStdlibModuleGetter(ctx, goRepo) - if err != nil { - log.Errorf(ctx, "loading packages from stdlib: %v", err) - } else { - getters = append(getters, mg) - } - } - } - - // Add a proxy - if cfg.proxy != nil { - getters = append(getters, fetch.NewProxyModuleGetter(cfg.proxy, source.NewClient(&http.Client{Timeout: time.Second}))) - } - - getters = append(getters, fetch.NewStdlibZipModuleGetter()) - - return getters, nil -} - -func getGOROOT() string { - if rg := runtime.GOROOT(); rg != "" { - return rg - } - b, err := exec.Command("go", "env", "GOROOT").Output() - if err != nil { - return "" - } - return strings.TrimSpace(string(b)) -} - -func newServer(getters []fetch.ModuleGetter, localModules []frontend.LocalModule, prox *proxy.Client) (*frontend.Server, error) { - lds := fetchdatasource.Options{ - Getters: getters, - ProxyClientForLatest: prox, - BypassLicenseCheck: true, - }.New() - - // In dev mode, use a dirFS to pick up template/JS/CSS changes without - // restarting the server. - var staticFS fs.FS - if *devMode { - staticFS = os.DirFS(*staticFlag) - } else { - staticFS = static.FS - } - - // Preload local modules to warm the cache. - for _, lm := range localModules { - go lds.GetUnitMeta(context.Background(), "", lm.ModulePath, fetch.LocalVersion) - } - go lds.GetUnitMeta(context.Background(), "", "std", "latest") - - server, err := frontend.NewServer(frontend.ServerConfig{ - DataSourceGetter: func(context.Context) internal.DataSource { return lds }, - TemplateFS: template.TrustedFSFromEmbed(static.FS), - StaticFS: staticFS, - DevMode: *devMode, - LocalMode: true, - LocalModules: localModules, - ThirdPartyFS: thirdparty.FS, - }) - if err != nil { - return nil, err - } - for _, g := range getters { - p, fsys := g.SourceFS() - if p != "" { - server.InstallFS(p, fsys) - } - } - return server, nil -} - -func defaultCacheDir() (string, error) { - out, err := runGo("", "env", "GOMODCACHE") - if err != nil { - return "", err - } - return strings.TrimSpace(string(out)), nil -} - -func runGo(dir string, args ...string) ([]byte, error) { - cmd := exec.Command("go", args...) - cmd.Dir = dir - out, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("running go with %q: %v: %s", args, err, out) - } - return out, nil -} diff --git a/cmd/pkgsite/main_test.go b/cmd/pkgsite/main_test.go index 704a425e5..85026c156 100644 --- a/cmd/pkgsite/main_test.go +++ b/cmd/pkgsite/main_test.go @@ -5,231 +5,11 @@ package main import ( - "context" - "net/http" - "net/http/httptest" - "os" - "path" - "path/filepath" - "regexp" "testing" "github.com/google/go-cmp/cmp" - "golang.org/x/net/html" - "golang.org/x/pkgsite/internal/proxy/proxytest" - "golang.org/x/pkgsite/internal/testenv" - "golang.org/x/pkgsite/internal/testing/htmlcheck" - "golang.org/x/pkgsite/internal/testing/testhelper" ) -var ( - in = htmlcheck.In - hasText = htmlcheck.HasText - attr = htmlcheck.HasAttr - - // href checks for an exact match in an href attribute. - href = func(val string) htmlcheck.Checker { - return attr("href", "^"+regexp.QuoteMeta(val)+"$") - } -) - -func TestServer(t *testing.T) { - testenv.MustHaveExecPath(t, "go") // for local modules - - repoPath := func(fn string) string { return filepath.Join("..", "..", fn) } - - abs := func(dir string) string { - a, err := filepath.Abs(dir) - if err != nil { - t.Fatal(err) - } - return a - } - - localModule, _ := testhelper.WriteTxtarToTempDir(t, ` --- go.mod -- -module example.com/testmod --- a.go -- -package a -`) - cacheDir := repoPath("internal/fetch/testdata/modcache") - testModules := proxytest.LoadTestModules(repoPath("internal/proxy/testdata")) - prox, teardown := proxytest.SetupTestClient(t, testModules) - defer teardown() - - cfg := func(modifyDefault func(*serverConfig)) serverConfig { - c := serverConfig{ - paths: []string{localModule}, - gopathMode: false, - useListedMods: true, - useCache: true, - cacheDir: cacheDir, - proxy: prox, - } - if modifyDefault != nil { - modifyDefault(&c) - } - return c - } - - modcacheChecker := in("", - in(".Documentation", hasText("var V = 1")), - sourceLinks(path.Join(filepath.ToSlash(abs(cacheDir)), "modcache.com@v1.0.0"), "a.go")) - - ctx := context.Background() - for _, test := range []struct { - name string - cfg serverConfig - url string - wantCode int - want htmlcheck.Checker - }{ - { - "local", - cfg(nil), - "example.com/testmod", - http.StatusOK, - in("", - in(".Documentation", hasText("There is no documentation for this package.")), - sourceLinks(path.Join(filepath.ToSlash(abs(localModule)), "example.com/testmod"), "a.go")), - }, - { - "modcache", - cfg(nil), - "modcache.com@v1.0.0", - http.StatusOK, - modcacheChecker, - }, - { - "modcache latest", - cfg(nil), - "modcache.com", - http.StatusOK, - modcacheChecker, - }, - { - "modcache unsupported", - cfg(func(c *serverConfig) { - c.useCache = false - }), - "modcache.com", - http.StatusFailedDependency, // TODO(rfindley): should this be 404? - hasText("page is not supported"), - }, - { - "proxy", - cfg(nil), - "example.com/single/pkg", - http.StatusOK, - hasText("G is new in v1.1.0"), - }, - { - "proxy unsupported", - cfg(func(c *serverConfig) { - c.proxy = nil - }), - "example.com/single/pkg", - http.StatusFailedDependency, // TODO(rfindley): should this be 404? - hasText("page is not supported"), - }, - { - "search", - cfg(func(c *serverConfig) { - c.useLocalStdlib = false - }), - "search?q=a", - http.StatusOK, - in(".SearchResults", - hasText("example.com/testmod"), - ), - }, - { - "no symbol search", - cfg(func(c *serverConfig) { - c.useLocalStdlib = false - }), - "search?q=A", // using a capital letter should not cause symbol search - http.StatusOK, - in(".SearchResults", - hasText("example.com/testmod"), - ), - }, - { - "search not found", - cfg(func(c *serverConfig) { - c.useLocalStdlib = false - }), - "search?q=zzz", - http.StatusOK, - in(".SearchResults", - hasText("no matches"), - ), - }, - { - "search vulns not found", - cfg(nil), - "search?q=GO-1234-1234", - http.StatusOK, - in(".SearchResults", - hasText("no matches"), - ), - }, - { - "search unsupported", - cfg(func(c *serverConfig) { - c.paths = nil - c.useLocalStdlib = false - }), - "search?q=zzz", - http.StatusFailedDependency, - hasText("page is not supported"), - }, - { - "vulns unsupported", - cfg(nil), - "vuln/", - http.StatusFailedDependency, - hasText("page is not supported"), - }, - // TODO(rfindley): add a test for the standard library once it doesn't go - // through the stdlib package. - // See also golang/go#58923. - } { - t.Run(test.name, func(t *testing.T) { - server, err := buildServer(ctx, test.cfg) - if err != nil { - t.Fatal(err) - } - mux := http.NewServeMux() - server.Install(mux.Handle, nil, nil) - - w := httptest.NewRecorder() - mux.ServeHTTP(w, httptest.NewRequest("GET", "/"+test.url, nil)) - if w.Code != test.wantCode { - t.Fatalf("got status code = %d, want %d", w.Code, test.wantCode) - } - doc, err := html.Parse(w.Body) - if err != nil { - t.Fatal(err) - } - if err := test.want(doc); err != nil { - if testing.Verbose() { - html.Render(os.Stdout, doc) - } - t.Error(err) - } - }) - } -} - -func sourceLinks(dir, filename string) htmlcheck.Checker { - filesPath := path.Join("/files", dir) + "/" - return in("", - in(".UnitMeta-repo a", href(filesPath)), - in(".UnitFiles-titleLink a", href(filesPath)), - in(".UnitFiles-fileList a", href(filesPath+filename))) -} - func TestCollectPaths(t *testing.T) { got := collectPaths([]string{"a", "b,c2,d3", "e4", "f,g"}) want := []string{"a", "b", "c2", "d3", "e4", "f", "g"}