From 17243c8f16fa97755aafaba8b8e90569ad2fade0 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Thu, 16 Nov 2023 09:34:14 +0100 Subject: [PATCH 1/2] use buf for builds and refactor build process --- .drone.yml | 33 ------- .gitignore | 1 + Dockerfile | 17 +--- build.go | 258 ++++++++--------------------------------------------- 4 files changed, 39 insertions(+), 270 deletions(-) delete mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index f708d2f..0000000 --- a/.drone.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -kind: pipeline -type: docker -name: build - -platform: - os: linux - arch: amd64 - -trigger: - branch: - - master - event: - exclude: - - pull_request - - tag - - promote - - rollback - -steps: -- name: publish-docker-image - pull: always - image: plugins/docker - settings: - repo: cs3org/cs3apis - tags: latest - username: - from_secret: dockerhub_username - password: - from_secret: dockerhub_password - custom_dns: - - 128.142.17.5 - - 128.142.16.5 diff --git a/.gitignore b/.gitignore index f0fb447..f68f347 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ cs3apis-build +build diff --git a/Dockerfile b/Dockerfile index bc69bea..ac8efb8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,21 +8,6 @@ RUN apt-get update RUN apt-get install build-essential curl unzip sudo -y RUN apt-get install python3-pip python3-full -y -# deps for protoc -RUN cd /tmp && curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v25.0/protoc-25.0-linux-x86_64.zip -o protoc.zip && unzip -o protoc.zip && sudo cp bin/protoc /usr/local/bin/protoc -RUN cd /tmp && curl -sSL https://github.com/uber/prototool/releases/download/v1.10.0/prototool-Linux-x86_64 -o prototool && sudo cp prototool /usr/local/bin/ && sudo chmod u+x /usr/local/bin/prototool -RUN cd /tmp && curl -sSL https://github.com/nilslice/protolock/releases/download/v0.16.0/protolock.20220302T184110Z.linux-amd64.tgz -o protolock.tgz && tar -xzf protolock.tgz && sudo cp protolock /usr/local/bin/ -RUN cd /tmp && curl -sSL https://github.com/pseudomuto/protoc-gen-doc/releases/download/v1.5.1/protoc-gen-doc_1.5.1_linux_amd64.tar.gz -o protoc-gen-doc.tar.gz && tar xzfv protoc-gen-doc.tar.gz && sudo cp protoc-gen-doc /usr/local/bin/ -RUN go install github.com/golang/protobuf/protoc-gen-go@v1.5.3 - - -# deps for python -RUN pip install grpcio grpcio-tools --ignore-installed --break-system-packages - -# deps for js -RUN curl -sSL https://github.com/grpc/grpc-web/releases/download/1.5.0/protoc-gen-grpc-web-1.5.0-linux-x86_64 -o /tmp/protoc-gen-grpc-web -RUN sudo mv /tmp/protoc-gen-grpc-web /usr/local/bin/ && sudo chmod u+x /usr/local/bin/protoc-gen-grpc-web - # deps for node.js RUN sudo apt-get update && sudo apt-get install -y ca-certificates curl gnupg RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg @@ -30,7 +15,7 @@ RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesourc RUN apt-get update RUN sudo apt-get install nodejs -y RUN node -v -RUN npm install protoc-gen-grpc -g -unsafe-perm +RUN npm install -g @bufbuild/buf # compile build tool and put it into path ADD . /root/cs3apis-build diff --git a/build.go b/build.go index 5972907..6516104 100644 --- a/build.go +++ b/build.go @@ -19,47 +19,14 @@ var ( gitName = flag.String("git-author-name", "cs3org-bot", "Git author name") gitSSH = flag.Bool("git-ssh", false, "Use git protocol instead of https for cloning repos") - _only_build = flag.Bool("only-build", false, "Build all protos and languages but do not push to language repos") - _all = flag.Bool("all", false, "Compile, build and publish for all available languages, mean to be run in CI platform like Drone") - - _buildProto = flag.Bool("build-proto", false, "Compile Protobuf definitions") - - _buildGo = flag.Bool("build-go", false, "Build Go library") - _pushGo = flag.Bool("push-go", false, "Push Go library to github.com/cs3org/go-cs3apis") - - _buildPython = flag.Bool("build-python", false, "Build Python library") - _pushPython = flag.Bool("push-python", false, "Push Python library to github.com/cs3org/python-cs3apis") - - _buildJs = flag.Bool("build-js", false, "Build Js library") - _pushJs = flag.Bool("push-js", false, "Push Js library to github.com/cs3org/js-cs3apis") - - _buildNode = flag.Bool("build-node", false, "Build Node.js library") - _pushNode = flag.Bool("push-node", false, "Push Node.js library to github.com/cs3org/node-cs3apis") + _pushGo = flag.Bool("push-go", false, "Push Go library to github.com/cs3org/go-cs3apis") + _pushPython = flag.Bool("push-python", false, "Push Python library to github.com/cs3org/python-cs3apis") + _pushJs = flag.Bool("push-js", false, "Push Js library to github.com/cs3org/js-cs3apis") + _pushNode = flag.Bool("push-node", false, "Push Node.js library to github.com/cs3org/node-cs3apis") ) func init() { flag.Parse() - - if *_all { - *_buildProto = true - *_buildGo = true - *_buildPython = true - *_buildJs = true - *_buildNode = true - - *_pushGo = true - *_pushPython = true - *_pushJs = true - *_pushNode = true - } - - if *_only_build { - *_buildProto = true - *_buildGo = true - *_buildPython = true - *_buildJs = true - *_buildNode = true - } } func getProtoOS() string { @@ -280,185 +247,57 @@ func findFolders() []string { return folders } -func buildProto() { - dir := "." - cmd := exec.Command("prototool", "compile", "--walk-timeout", "10s") - cmd.Dir = dir - run(cmd) - - cmd = exec.Command("protolock", "status") - cmd.Dir = dir - run(cmd) - - // lint - cmd = exec.Command("prototool", "format", "-w", "--walk-timeout", "10s") - cmd.Dir = dir - run(cmd) - cmd = exec.Command("prototool", "lint", "--walk-timeout", "10s") - cmd.Dir = dir - run(cmd) - cmd = exec.Command("go", "run", "tools/check-license/check-license.go") - cmd.Dir = dir - run(cmd) - - os.RemoveAll("docs") - os.MkdirAll("docs", 0755) - - files := findProtos() - fmt.Println(files) - - args := []string{"--doc_out=./docs", "--doc_opt=html,index.html", "-I=.", "-I=./third_party"} - args = append(args, files...) - cmd = exec.Command("protoc", args...) - run(cmd) -} - -func buildGo() { - - // Remove build dir - os.RemoveAll("build/go-cs3apis") - os.MkdirAll("build", 0755) - - // Clone Go repo and set branch to current branch - clone("cs3org/go-cs3apis", "build") - protoBranch := getGitBranch(".") - goBranch := getGitBranch("build/go-cs3apis") - fmt.Printf("Proto branch: %s\nGo branch: %s\n", protoBranch, goBranch) - - if goBranch != protoBranch { - checkout(protoBranch, "build/go-cs3apis") +func generate() { + cwd, err := os.Getwd() + if err != nil { + panic(err) } + fmt.Printf("current working directory: %s\n", cwd) - // remove leftovers (existing defs) - os.RemoveAll("build/go-cs3apis/cs3") - - cmd := exec.Command("prototool", "generate", "--walk-timeout", "10s") + cmd := exec.Command("git", "config", "--global", "--add", "safe.directory", cwd) run(cmd) - sed("build/go-cs3apis", ".go", "github.com/cs3org/go-cs3apis/build/go-cs3apis/cs3/", "github.com/cs3org/go-cs3apis/cs3/") - - if !isRepoDirty("build/go-cs3apis") { - fmt.Println("Repo is clean, nothing to do") - } - - // get proto repo commit id - hash := getCommitID(".") - repo := "build/go-cs3apis" - msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash - commit(repo, msg) -} - -func buildPython() { - // Remove build dir - os.RemoveAll("build/python-cs3apis") + os.RemoveAll("build") os.MkdirAll("build", 0755) - // Clone Go repo and set branch to current branch - clone("cs3org/python-cs3apis", "build") - protoBranch := getGitBranch(".") - buildBranch := getGitBranch("build/python-cs3apis") - fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch) - - if buildBranch != protoBranch { - checkout(protoBranch, "build/python-cs3apis") - } - - // remove leftovers (existing defs) - os.RemoveAll("build/python-cs3apis/cs3") - - files := findProtos() - - args := []string{"-m", "grpc_tools.protoc", "--python_out=./build/python-cs3apis", "-I.", "-I./third_party", "--grpc_python_out=./build/python-cs3apis"} - args = append(args, files...) - cmd := exec.Command("python3", args...) - run(cmd) - - modules := findFolders() - - var initFiles []string - for _, f := range modules { - initPy := fmt.Sprintf("%s/%s/%s", "build/python-cs3apis", f, "__init__.py") - initFiles = append(initFiles, initPy) - } + languages := []string{"go", "js", "node", "python"} - cmd = exec.Command("touch", initFiles...) - run(cmd) + // prepare language git repos + for _, l := range languages { + target := fmt.Sprintf("%s-cs3apis", l) - // get proto repo commit id - hash := getCommitID(".") - repo := "build/python-cs3apis" - msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash - commit(repo, msg) -} + // Clone Go repo and set branch to current branch + clone("cs3org/"+target, "build") + protoBranch := getGitBranch(".") + targetBranch := getGitBranch("build/" + target) + fmt.Printf("Proto branch: %s\n%s branch: %s\n", l, protoBranch, targetBranch) -func buildJS() { - // Remove build dir - os.RemoveAll("build/js-cs3apis") - os.MkdirAll("build", 0755) + if targetBranch != protoBranch { + checkout(protoBranch, "build/"+target) + } - // Clone repo and set branch to current branch - clone("cs3org/js-cs3apis", "build") - protoBranch := getGitBranch(".") - buildBranch := getGitBranch("build/js-cs3apis") - fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch) + // remove leftovers (existing defs) + os.RemoveAll(fmt.Sprintf("build/%s/cs3", target)) - if buildBranch != protoBranch { - checkout(protoBranch, "build/js-cs3apis") } - // remove leftovers (existing defs) - os.RemoveAll("build/js-cs3apis/cs3") - - files := findProtos() - - args := []string{"--js_out=import_style=commonjs:./build/js-cs3apis", "--grpc-web_out=import_style=commonjs,mode=grpcwebtext:./build/js-cs3apis/", "-I.", "-I./third_party"} - args = append(args, files...) - cmd := exec.Command("protoc", args...) + cmd = exec.Command("buf", "generate") run(cmd) - // get proto repo commit id - hash := getCommitID(".") - repo := "build/js-cs3apis" - msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash - commit(repo, msg) -} + for _, l := range languages { + target := fmt.Sprintf("%s-cs3apis", l) -func buildNode() { - // Remove build dir - os.RemoveAll("build/node-cs3apis") - os.MkdirAll("build", 0755) - - // Clone repo and set branch to current branch - clone("cs3org/node-cs3apis", "build") - protoBranch := getGitBranch(".") - buildBranch := getGitBranch("build/node-cs3apis") - fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch) + if !isRepoDirty("build/" + target) { + fmt.Println("Repo is clean, nothing to do") + } - if buildBranch != protoBranch { - checkout(protoBranch, "build/node-cs3apis") + // get proto repo commit id + hash := getCommitID(".") + repo := "build/" + target + msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash + commit(repo, msg) } - - // remove leftovers (existing defs) - os.RemoveAll("build/node-cs3apis/cs3") - - files := findProtos() - - args1 := []string{"--ts_out=grpc_js:./build/node-cs3apis", "--proto_path=.", "--proto_path=./third_party"} - args1 = append(args1, files...) - cmd1 := exec.Command("protoc-gen-grpc-ts", args1...) - run(cmd1) - - args2 := []string{"--js_out=import_style=commonjs,binary:./build/node-cs3apis", "--grpc_out=grpc_js:./build/node-cs3apis/", "--proto_path=.", "--proto_path=./third_party"} - args2 = append(args2, files...) - cmd2 := exec.Command("protoc-gen-grpc", args2...) - run(cmd2) - - // get proto repo commit id - hash := getCommitID(".") - repo := "build/node-cs3apis" - msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash - commit(repo, msg) } func pushPython() { @@ -478,46 +317,23 @@ func pushNode() { } func main() { - if *_buildProto { - fmt.Println("Compiling and linting protobufs ...") - buildProto() - } - - if *_buildGo { - fmt.Println("Building Go ...") - buildGo() - } + generate() if *_pushGo { fmt.Println("Pushing Go ...") pushGo() } - if *_buildPython { - fmt.Println("Building Python ...") - buildPython() - } - if *_pushPython { fmt.Println("Pushing Python ...") pushPython() } - if *_buildJs { - fmt.Println("Building JS ...") - buildJS() - } - if *_pushJs { fmt.Println("Pushing Js ...") pushJS() } - if *_buildNode { - fmt.Println("Building Node.js ...") - buildNode() - } - if *_pushNode { fmt.Println("Pushing Node.js ...") pushNode() From 92022f9d4ba9abcb9432f217f7899dc7084bb14a Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Thu, 16 Nov 2023 09:59:52 +0100 Subject: [PATCH 2/2] add verbosity to build chain --- build.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.go b/build.go index 6516104..8d111e2 100644 --- a/build.go +++ b/build.go @@ -252,6 +252,7 @@ func generate() { if err != nil { panic(err) } + fmt.Println("Starting generation of protobuf language bindings ...") fmt.Printf("current working directory: %s\n", cwd) cmd := exec.Command("git", "config", "--global", "--add", "safe.directory", cwd) @@ -266,6 +267,7 @@ func generate() { // prepare language git repos for _, l := range languages { target := fmt.Sprintf("%s-cs3apis", l) + fmt.Println("cloning repo for " + target) // Clone Go repo and set branch to current branch clone("cs3org/"+target, "build") @@ -282,11 +284,13 @@ func generate() { } + fmt.Println("Generating ...") cmd = exec.Command("buf", "generate") run(cmd) for _, l := range languages { target := fmt.Sprintf("%s-cs3apis", l) + fmt.Println("Commiting changes for " + target) if !isRepoDirty("build/" + target) { fmt.Println("Repo is clean, nothing to do") @@ -298,6 +302,7 @@ func generate() { msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash commit(repo, msg) } + fmt.Println("Generation done!") } func pushPython() {