feat(internal/godocfx): add support for other modules (#4290)

I chose to process all modules at the same time, rather than in separate
invocations of godocfx, so we wouldn't have to make duplicate requests
to the module index.

The docs.metadata generation will probably need to be updated in a
future PR to support different final URL destinations.

internal/godocfx/index.go

@@ -28,7 +28,7 @@ import (
 
 // indexer gets a limited list of entries from index.golang.org.
 type indexer interface {
-	get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error)
+	get(prefixes []string, since time.Time) (entries []indexEntry, last time.Time, err error)
 }
 
 // indexClient is used to access index.golang.org.
@@ -49,7 +49,7 @@ type indexEntry struct {
 // versions since the given timestamp.
 //
 // newModules stores the timestamp of the last successful run with tSaver.
-func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) ([]indexEntry, error) {
+func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefixes []string) ([]indexEntry, error) {
 	since, err := tSaver.get(ctx)
 	if err != nil {
 		return nil, err
@@ -61,7 +61,7 @@ func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string)
 	for {
 		count++
 		var cur []indexEntry
-		cur, since, err = i.get(prefix, since)
+		cur, since, err = i.get(prefixes, since)
 		if err != nil {
 			return nil, err
 		}
@@ -80,7 +80,7 @@ func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string)
 
 // get fetches a single chronological page of modules from
 // index.golang.org/index.
-func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time, error) {
+func (indexClient) get(prefixes []string, since time.Time) ([]indexEntry, time.Time, error) {
 	entries := []indexEntry{}
 	sinceString := since.Format(time.RFC3339)
 	resp, err := http.Get("https://index.golang.org/index?since=" + sinceString)
@@ -96,7 +96,7 @@ func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time,
 			return nil, time.Time{}, err
 		}
 		last = e.Timestamp // Always update the last timestamp.
-		if !strings.HasPrefix(e.Path, prefix) ||
+		if !hasPrefix(e.Path, prefixes) ||
 			strings.Contains(e.Path, "internal") ||
 			strings.Contains(e.Path, "third_party") ||
 			strings.Contains(e.Version, "-") { // Filter out pseudo-versions.

internal/godocfx/index_test.go

@@ -26,7 +26,7 @@ const wantEntries = 5
 
 type fakeIC struct{}
 
-func (f fakeIC) get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) {
+func (f fakeIC) get(prefixes []string, since time.Time) (entries []indexEntry, last time.Time, err error) {
 	e := indexEntry{Timestamp: since.Add(24 * time.Hour)}
 	return []indexEntry{e}, e.Timestamp, nil
 }
@@ -49,7 +49,7 @@ func (f *fakeTS) put(context.Context, time.Time) error {
 func TestNewModules(t *testing.T) {
 	ic := fakeIC{}
 	ts := &fakeTS{}
-	entries, err := newModules(context.Background(), ic, ts, "cloud.google.com")
+	entries, err := newModules(context.Background(), ic, ts, []string{"cloud.google.com"})
 	if err != nil {
 		t.Fatalf("newModules got err: %v", err)
 	}

internal/godocfx/main.go

@@ -58,42 +58,43 @@ func main() {
 	outDir := flag.String("out", "obj/api", "Output directory (default obj/api)")
 	projectID := flag.String("project", "", "Project ID to use. Required when using -new-modules.")
 	newMods := flag.Bool("new-modules", false, "Process all new modules with the given prefix. Uses timestamp in Datastore. Stores results in $out/$mod.")
+	// TODO: flag to set output URL path
 
 	log.SetPrefix("[godocfx] ")
 
 	flag.Parse()
-	if flag.NArg() != 1 {
+	if flag.NArg() == 0 {
 		log.Fatalf("%s missing required argument: module path/prefix", os.Args[0])
 	}
 
-	mod := flag.Arg(0)
+	modNames := flag.Args()
 	var mods []indexEntry
 	if *newMods {
 		if *projectID == "" {
 			log.Fatal("Must set -project when using -new-modules")
 		}
 		var err error
-		mods, err = newModules(context.Background(), indexClient{}, &dsTimeSaver{projectID: *projectID}, mod)
+		mods, err = newModules(context.Background(), indexClient{}, &dsTimeSaver{projectID: *projectID}, modNames)
 		if err != nil {
 			log.Fatal(err)
 		}
 	} else {
-		modPath := mod
-		version := "latest"
-		if strings.Contains(mod, "@") {
-			parts := strings.Split(mod, "@")
-			if len(parts) != 2 {
-				log.Fatal("module arg expected only one '@'")
+		for _, mod := range modNames {
+			modPath := mod
+			version := "latest"
+			if strings.Contains(mod, "@") {
+				parts := strings.Split(mod, "@")
+				if len(parts) != 2 {
+					log.Fatal("module arg expected only one '@'")
+				}
+				modPath = parts[0]
+				version = parts[1]
 			}
-			modPath = parts[0]
-			version = parts[1]
-		}
-		modPath = strings.TrimSuffix(modPath, "/...") // No /... needed.
-		mods = []indexEntry{
-			{
+			modPath = strings.TrimSuffix(modPath, "/...") // No /... needed.
+			mods = append(mods, indexEntry{
 				Path:    modPath,
 				Version: version,
-			},
+			})
 		}
 	}
 
@@ -105,21 +106,19 @@ func main() {
 		return
 	}
 	// Create a temp module so we can get the exact version asked for.
-	tempDir, err := ioutil.TempDir("", "godocfx-*")
+	workingDir, err := ioutil.TempDir("", "godocfx-*")
 	if err != nil {
 		log.Fatalf("ioutil.TempDir: %v", err)
 	}
-	runCmd(tempDir, "go", "mod", "init", "cloud.google.com/go/lets-build-some-docs")
+	// Use a fake module that doesn't start with cloud.google.com/go.
+	runCmd(workingDir, "go", "mod", "init", "cloud.google.com/lets-build-some-docs")
 	for _, m := range mods {
 		log.Printf("Processing %s@%s", m.Path, m.Version)
 
-		path := *outDir
-		// If we have more than one module, we need a more specific out path.
-		if len(mods) > 1 {
-			path = filepath.Join(path, fmt.Sprintf("%s@%s", m.Path, m.Version))
-		}
-		if err := process(m, tempDir, path, *print); err != nil {
-			log.Printf("Failed to process %v", m)
+		// Always output to specific directory.
+		path := filepath.Join(*outDir, fmt.Sprintf("%s@%s", m.Path, m.Version))
+		if err := process(m, workingDir, path, *print); err != nil {
+			log.Printf("Failed to process %v: %v", m, err)
 		}
 		log.Printf("Done with %s@%s", m.Path, m.Version)
 	}
@@ -138,22 +137,23 @@ func runCmd(dir, name string, args ...string) error {
 	return nil
 }
 
-func process(mod indexEntry, tempDir, outDir string, print bool) error {
+func process(mod indexEntry, workingDir, outDir string, print bool) error {
 	// Be sure to get the module and run the module loader in the tempDir.
-	if err := runCmd(tempDir, "go", "mod", "tidy"); err != nil {
-		return err
+	if err := runCmd(workingDir, "go", "mod", "tidy"); err != nil {
+		return fmt.Errorf("go mod tidy error: %v", err)
 	}
-	if err := runCmd(tempDir, "go", "get", mod.Path+"@"+mod.Version); err != nil {
-		return err
+	if err := runCmd(workingDir, "go", "get", "-d", "-t", mod.Path+"/...@"+mod.Version); err != nil {
+		return fmt.Errorf("go get %s@%s: %v", mod.Path, mod.Version, err)
 	}
 
+	log.Println("Starting to parse")
 	optionalExtraFiles := []string{}
 	filter := []string{
 		"cloud.google.com/go/analytics",
 		"cloud.google.com/go/area120",
 		"cloud.google.com/go/gsuiteaddons",
 	}
-	r, err := parse(mod.Path+"/...", tempDir, optionalExtraFiles, filter)
+	r, err := parse(mod.Path+"/...", workingDir, optionalExtraFiles, filter)
 	if err != nil {
 		return fmt.Errorf("parse: %v", err)
 	}

internal/godocfx/parse.go

@@ -452,8 +452,6 @@ func (l *linker) linkify(s string) string {
 	return fmt.Sprintf("%s%s.%s", prefix, href(l.toURL(pkgPath, ""), pkg), href(l.toURL(pkgPath, name), name))
 }
 
-// TODO: link to the right baseURL, with the right module name and version
-// pattern.
 func (l *linker) toURL(pkg, name string) string {
 	if pkg == "" {
 		if anchor := l.idToAnchor[""][name]; anchor != "" {
@@ -467,6 +465,7 @@ func (l *linker) toURL(pkg, name string) string {
 			pkgRemainder = pkg[len(mod.Path)+1:] // +1 to skip slash.
 		}
 		// Note: we always link to latest. One day, we'll link to mod.Version.
+		// Also, other packages may have different paths.
 		baseURL := fmt.Sprintf("/go/docs/reference/%v/latest/%v", mod.Path, pkgRemainder)
 		if anchor := l.idToAnchor[pkg][name]; anchor != "" {
 			return fmt.Sprintf("%s#%s", baseURL, anchor)