From 7ce0c8d0fee1437daf8d219a78570d8d92fbaf07 Mon Sep 17 00:00:00 2001 From: noahdietz Date: Wed, 23 Jun 2021 14:10:14 -0700 Subject: [PATCH] chore(internal): gen googleapis-discovery compute genproto --- internal/gapicgen/cmd/genbot/generate.go | 15 ++++++--- internal/gapicgen/cmd/genbot/local.go | 34 +++++++++++--------- internal/gapicgen/cmd/genbot/main.go | 16 +++++----- internal/gapicgen/generator/generator.go | 19 +++++------ internal/gapicgen/generator/genproto.go | 40 +++++++++++++++++------- 5 files changed, 76 insertions(+), 48 deletions(-) diff --git a/internal/gapicgen/cmd/genbot/generate.go b/internal/gapicgen/cmd/genbot/generate.go index ac6903866de6..b4ea82b336c5 100644 --- a/internal/gapicgen/cmd/genbot/generate.go +++ b/internal/gapicgen/cmd/genbot/generate.go @@ -42,6 +42,7 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool log.Printf("working out %s\n", tmpDir) googleapisDir := filepath.Join(tmpDir, "googleapis") + googleapisDiscoDir := filepath.Join(tmpDir, "googleapis-discovery") gocloudDir := filepath.Join(tmpDir, "gocloud") genprotoDir := filepath.Join(tmpDir, "genproto") protoDir := filepath.Join(tmpDir, "proto") @@ -52,6 +53,9 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool grp.Go(func() error { return git.DeepClone("https://github.com/googleapis/googleapis", googleapisDir) }) + grp.Go(func() error { + return git.DeepClone("https://github.com/googleapis/googleapis-discovery", googleapisDiscoDir) + }) grp.Go(func() error { return git.DeepClone("https://github.com/googleapis/go-genproto", genprotoDir) }) @@ -67,11 +71,12 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool // Regen. conf := &generator.Config{ - GoogleapisDir: googleapisDir, - GenprotoDir: genprotoDir, - GapicDir: gocloudDir, - ProtoDir: protoDir, - ForceAll: forceAll, + GoogleapisDir: googleapisDir, + GoogleapisDiscoDir: googleapisDiscoDir, + GenprotoDir: genprotoDir, + GapicDir: gocloudDir, + ProtoDir: protoDir, + ForceAll: forceAll, } changes, err := generator.Generate(ctx, conf) if err != nil { diff --git a/internal/gapicgen/cmd/genbot/local.go b/internal/gapicgen/cmd/genbot/local.go index 0848ef3d5c5a..c7cab8bfaa4d 100644 --- a/internal/gapicgen/cmd/genbot/local.go +++ b/internal/gapicgen/cmd/genbot/local.go @@ -29,13 +29,14 @@ import ( ) type localConfig struct { - googleapisDir string - gocloudDir string - genprotoDir string - protoDir string - gapicToGenerate string - onlyGapics bool - regenOnly bool + googleapisDir string + googleapisDiscoDir string + gocloudDir string + genprotoDir string + protoDir string + gapicToGenerate string + onlyGapics bool + regenOnly bool } func genLocal(ctx context.Context, c localConfig) error { @@ -46,6 +47,7 @@ func genLocal(ctx context.Context, c localConfig) error { } log.Printf("temp dir created at %s\n", tmpDir) tmpGoogleapisDir := filepath.Join(tmpDir, "googleapis") + tmpGoogleapisDiscoDir := filepath.Join(tmpDir, "googleapis-discovery") tmpGenprotoDir := filepath.Join(tmpDir, "genproto") tmpGocloudDir := filepath.Join(tmpDir, "gocloud") tmpProtoDir := filepath.Join(tmpDir, "proto") @@ -53,6 +55,7 @@ func genLocal(ctx context.Context, c localConfig) error { // Clone repositories if needed. grp, _ := errgroup.WithContext(ctx) gitShallowClone(grp, "https://github.com/googleapis/googleapis.git", c.googleapisDir, tmpGoogleapisDir) + gitShallowClone(grp, "https://github.com/googleapis/googleapis-discovery.git", c.googleapisDiscoDir, tmpGoogleapisDiscoDir) gitShallowClone(grp, "https://github.com/googleapis/go-genproto", c.genprotoDir, tmpGenprotoDir) gitShallowClone(grp, "https://github.com/googleapis/google-cloud-go", c.gocloudDir, tmpGocloudDir) gitShallowClone(grp, "https://github.com/protocolbuffers/protobuf", c.protoDir, tmpProtoDir) @@ -62,14 +65,15 @@ func genLocal(ctx context.Context, c localConfig) error { // Regen. conf := &generator.Config{ - GoogleapisDir: deafultDir(tmpGoogleapisDir, c.googleapisDir), - GenprotoDir: deafultDir(tmpGenprotoDir, c.genprotoDir), - GapicDir: deafultDir(tmpGocloudDir, c.gocloudDir), - ProtoDir: deafultDir(tmpProtoDir, c.protoDir), - GapicToGenerate: c.gapicToGenerate, - OnlyGenerateGapic: c.onlyGapics, - LocalMode: true, - RegenOnly: c.regenOnly, + GoogleapisDir: deafultDir(tmpGoogleapisDir, c.googleapisDir), + GoogleapisDiscoDir: deafultDir(tmpGoogleapisDiscoDir, c.googleapisDiscoDir), + GenprotoDir: deafultDir(tmpGenprotoDir, c.genprotoDir), + GapicDir: deafultDir(tmpGocloudDir, c.gocloudDir), + ProtoDir: deafultDir(tmpProtoDir, c.protoDir), + GapicToGenerate: c.gapicToGenerate, + OnlyGenerateGapic: c.onlyGapics, + LocalMode: true, + RegenOnly: c.regenOnly, } if _, err := generator.Generate(ctx, conf); err != nil { log.Printf("Generator ran (and failed) in %s\n", tmpDir) diff --git a/internal/gapicgen/cmd/genbot/main.go b/internal/gapicgen/cmd/genbot/main.go index ddaa4f302e14..89f941959f68 100644 --- a/internal/gapicgen/cmd/genbot/main.go +++ b/internal/gapicgen/cmd/genbot/main.go @@ -45,6 +45,7 @@ func main() { // flags for local mode googleapisDir := flag.String("googleapis-dir", os.Getenv("GOOGLEAPIS_DIR"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") + googleapisDiscoDir := flag.String("googleapis-disco-dir", os.Getenv("GOOGLEAPIS_DISCO_DIR"), "Directory where sources of googleapis/googleapis-discovery resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") gocloudDir := flag.String("gocloud-dir", os.Getenv("GOCLOUD_DIR"), "Directory where sources of googleapis/google-cloud-go resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") genprotoDir := flag.String("genproto-dir", os.Getenv("GENPROTO_DIR"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") protoDir := flag.String("proto-dir", os.Getenv("PROTO_DIR"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") @@ -56,13 +57,14 @@ func main() { if *localMode { if err := genLocal(ctx, localConfig{ - googleapisDir: *googleapisDir, - gocloudDir: *gocloudDir, - genprotoDir: *genprotoDir, - protoDir: *protoDir, - gapicToGenerate: *gapicToGenerate, - onlyGapics: *onlyGapics, - regenOnly: *regenOnly, + googleapisDir: *googleapisDir, + googleapisDiscoDir: *googleapisDiscoDir, + gocloudDir: *gocloudDir, + genprotoDir: *genprotoDir, + protoDir: *protoDir, + gapicToGenerate: *gapicToGenerate, + onlyGapics: *onlyGapics, + regenOnly: *regenOnly, }); err != nil { log.Fatal(err) } diff --git a/internal/gapicgen/generator/generator.go b/internal/gapicgen/generator/generator.go index fa644e43ca46..bea74721bc2c 100644 --- a/internal/gapicgen/generator/generator.go +++ b/internal/gapicgen/generator/generator.go @@ -29,15 +29,16 @@ import ( // Config contains inputs needed to generate sources. type Config struct { - GoogleapisDir string - GenprotoDir string - GapicDir string - ProtoDir string - GapicToGenerate string - OnlyGenerateGapic bool - LocalMode bool - RegenOnly bool - ForceAll bool + GoogleapisDir string + GoogleapisDiscoDir string + GenprotoDir string + GapicDir string + ProtoDir string + GapicToGenerate string + OnlyGenerateGapic bool + LocalMode bool + RegenOnly bool + ForceAll bool } // Generate generates genproto and gapics. diff --git a/internal/gapicgen/generator/genproto.go b/internal/gapicgen/generator/genproto.go index 33b1b7b6390d..2d18bf0dd56a 100644 --- a/internal/gapicgen/generator/genproto.go +++ b/internal/gapicgen/generator/genproto.go @@ -55,19 +55,21 @@ var denylist = map[string]bool{ // GenprotoGenerator is used to generate code for googleapis/go-genproto. type GenprotoGenerator struct { - genprotoDir string - googleapisDir string - protoSrcDir string - forceAll bool + genprotoDir string + googleapisDir string + googleapisDiscoDir string + protoSrcDir string + forceAll bool } // NewGenprotoGenerator creates a new GenprotoGenerator. func NewGenprotoGenerator(c *Config) *GenprotoGenerator { return &GenprotoGenerator{ - genprotoDir: c.GenprotoDir, - googleapisDir: c.GoogleapisDir, - protoSrcDir: filepath.Join(c.ProtoDir, "/src"), - forceAll: c.ForceAll, + genprotoDir: c.GenprotoDir, + googleapisDir: c.GoogleapisDir, + googleapisDiscoDir: c.GoogleapisDiscoDir, + protoSrcDir: filepath.Join(c.ProtoDir, "/src"), + forceAll: c.ForceAll, } } @@ -102,7 +104,7 @@ func (g *GenprotoGenerator) Regen(ctx context.Context) error { log.Println("regenerating genproto") // Create space to put generated .pb.go's. - c := execv.Command("mkdir", "generated") + c := execv.Command("mkdir", "-p", "generated") c.Dir = g.genprotoDir if err := c.Run(); err != nil { return err @@ -114,6 +116,10 @@ func (g *GenprotoGenerator) Regen(ctx context.Context) error { return err } + // TODO(noahdietz): In local mode, since it clones a shallow copy with 1 commit, + // if the last regenerated hash is earlier than the top commit, the git diff-tree + // command fails. This is is a bit of a rough edge. Using my local clone of + // googleapis rectified the issue. pkgFiles, err := g.getUpdatedPackages(string(lastHash)) if err != nil { return err @@ -132,9 +138,15 @@ func (g *GenprotoGenerator) Regen(ctx context.Context) error { fn := fileNames grp.Go(func() error { log.Println("running protoc on", pk) - return g.protoc(fn) + return g.protoc(fn, true /* grpc */) }) } + // TODO(noahdietz): This needs to be generalized to support any proto in googleapis-discovery. + // It's hard because the regen.txt contains the committish from googleapis last used to regen. + grp.Go(func() error { + log.Println("running protoc on compute") + return g.protoc([]string{"google/cloud/compute/v1/compute.proto"}, false /* grpc */) + }) if err := grp.Wait(); err != nil { return err } @@ -178,8 +190,12 @@ func goPkg(fileName string) (string, error) { // protoc executes the "protoc" command on files named in fileNames, and outputs // to "/generated". -func (g *GenprotoGenerator) protoc(fileNames []string) error { - args := []string{"--experimental_allow_proto3_optional", fmt.Sprintf("--go_out=plugins=grpc:%s/generated", g.genprotoDir), "-I", g.googleapisDir, "-I", g.protoSrcDir} +func (g *GenprotoGenerator) protoc(fileNames []string, grpc bool) error { + stubs := fmt.Sprintf("--go_out=%s/generated", g.genprotoDir) + if grpc { + stubs = fmt.Sprintf("--go_out=plugins=grpc:%s/generated", g.genprotoDir) + } + args := []string{"--experimental_allow_proto3_optional", stubs, "-I", g.googleapisDiscoDir, "-I", g.googleapisDir, "-I", g.protoSrcDir} args = append(args, fileNames...) c := execv.Command("protoc", args...) c.Dir = g.genprotoDir