Skip to content

Commit

Permalink
Merge pull request #191 from ulucinar/cp-sr
Browse files Browse the repository at this point in the history
[Backport standard-runners]: Backports PRs #187 #188 #189 #190
  • Loading branch information
ulucinar authored Mar 12, 2024
2 parents ae8df92 + 5dae6fd commit fd8ec1e
Show file tree
Hide file tree
Showing 10 changed files with 235 additions and 13 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ on:

env:
# Common versions
GO_VERSION: '1.19'
GO_VERSION: '1.21'
GOLANGCI_VERSION: 'v1.54.2'
DOCKER_BUILDX_VERSION: 'v0.8.2'

Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/pr-comment-trigger.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ on:
default: 'provider'
required: false
type: string
cleanup-disk:
description: "If set to true, an initial step will be run to reclaim some extra disk space for the uptest job in this workflow"
required: false
type: boolean
default: false
update-test-parameter:
description: 'Input parameter to use during update step. If this field
is empty, then the update step will be skipped.
Expand Down Expand Up @@ -113,6 +118,7 @@ jobs:
steps:
- name: Cleanup Disk
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
if: ${{ inputs.cleanup-disk }}
with:
android: true
dotnet: true
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/promote.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ on:

env:
# Common versions
GO_VERSION: '1.19'
GO_VERSION: '1.21'

# Common users. We can't run a step 'if secrets.XXX != ""' but we can run
# a step 'if env.XXX' != ""', so we copy these to succinctly test whether
Expand Down
58 changes: 57 additions & 1 deletion .github/workflows/provider-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ on:
workflow_call:
inputs:
upjet-based-provider:
description: 'Should be set to true if the repo running the workflow is an upjet-based provider repo'
required: false
type: boolean
default: true
Expand All @@ -12,11 +13,21 @@ on:
default: '1.20'
required: false
type: string
cleanup-disk:
description: "If set to true, an initial step will be run to reclaim some extra disk space for the build/test jobs in this workflow"
required: false
type: boolean
default: false
golangci-version:
description: 'The version string to be used with the golangci-lint action'
default: 'v1.55.2'
required: false
type: string
golangci-skip:
description: 'If set to true, the lint job will be skipped'
default: false
required: false
type: boolean
secrets:
UPBOUND_MARKETPLACE_PUSH_ROBOT_USR:
required: true
Expand Down Expand Up @@ -75,9 +86,20 @@ jobs:
lint:
runs-on: ubuntu-22.04
needs: detect-noop
if: needs.detect-noop.outputs.noop != 'true'
if: needs.detect-noop.outputs.noop != 'true' && !inputs.golangci-skip

steps:
- name: Cleanup Disk
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
if: ${{ inputs.cleanup-disk }}
with:
android: true
dotnet: true
haskell: true
tool-cache: true
large-packages: false
swap-storage: false

- name: Checkout
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
with:
Expand Down Expand Up @@ -131,6 +153,17 @@ jobs:
if: needs.detect-noop.outputs.noop != 'true'

steps:
- name: Cleanup Disk
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
if: ${{ inputs.cleanup-disk }}
with:
android: true
dotnet: true
haskell: true
tool-cache: true
large-packages: false
swap-storage: false

- name: Checkout
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
with:
Expand Down Expand Up @@ -197,6 +230,17 @@ jobs:
if: needs.detect-noop.outputs.noop != 'true'

steps:
- name: Cleanup Disk
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
if: ${{ inputs.cleanup-disk }}
with:
android: true
dotnet: true
haskell: true
tool-cache: true
large-packages: false
swap-storage: false

- name: Checkout
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
with:
Expand Down Expand Up @@ -248,6 +292,17 @@ jobs:
if: needs.detect-noop.outputs.noop != 'true'

steps:
- name: Cleanup Disk
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
if: ${{ inputs.cleanup-disk }}
with:
android: true
dotnet: true
haskell: true
tool-cache: true
large-packages: false
swap-storage: false

- name: Checkout
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
with:
Expand Down Expand Up @@ -295,6 +350,7 @@ jobs:
steps:
- name: Cleanup Disk
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
if: ${{ inputs.cleanup-disk }}
with:
android: true
dotnet: true
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/provider-publish-service-artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ env:
# Common versions
GOLANGCI_VERSION: 'v1.54.2'
DOCKER_BUILDX_VERSION: 'v0.8.2'
UP_VERSION: 'v0.17.0'
UP_VERSION: 'v0.24.2'

# Common users. We can't run a step 'if secrets.XXX != ""' but we can run a
# step 'if env.XXX' != ""', so we copy these to succinctly test whether
Expand Down Expand Up @@ -87,11 +87,11 @@ jobs:
- name: Login to Upbound
if: env.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR != ''
env:
UP_PASSWORD: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }}
UP_TOKEN: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }}
run: |
curl -fsSLo /tmp/up --create-dirs 'https://cli.upbound.io/stable/${{ env.UP_VERSION }}/bin/linux_amd64/up' && \
chmod +x /tmp/up && \
/tmp/up login -u ${{ env.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }}
/tmp/up login
- name: Checkout
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
Expand Down
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@ S3_BUCKET ?= upbound.official-providers-ci.releases

# ====================================================================================
# Setup Go
GO_REQUIRED_VERSION = 1.19
GO_REQUIRED_VERSION = 1.21
# GOLANGCILINT_VERSION is inherited from build submodule by default.
# Uncomment below if you need to override the version.
# GOLANGCILINT_VERSION ?= 1.54.0

GO_STATIC_PACKAGES = $(GO_PROJECT)/cmd/uptest $(GO_PROJECT)/cmd/crddiff $(GO_PROJECT)/cmd/updoc $(GO_PROJECT)/cmd/ttr $(GO_PROJECT)/cmd/perf $(GO_PROJECT)/cmd/linter/lint-provider-family
GO_STATIC_PACKAGES = $(GO_PROJECT)/cmd/uptest $(GO_PROJECT)/cmd/crddiff $(GO_PROJECT)/cmd/buildtagger $(GO_PROJECT)/cmd/updoc $(GO_PROJECT)/cmd/ttr $(GO_PROJECT)/cmd/perf $(GO_PROJECT)/cmd/linter/lint-provider-family
GO_LDFLAGS += -X $(GO_PROJECT)/internal/version.Version=$(VERSION)
GO_SUBDIRS += cmd internal
GO111MODULE = on
Expand Down
145 changes: 145 additions & 0 deletions cmd/buildtagger/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
// Copyright 2024 Upbound Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// main package for the buildtagger application which is used in the
// official provider repositories to generate build tags for the
// provider families. Each family's source modules can be tagged using
// the buildtagger tool.
package main

import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"

"github.com/pkg/errors"
"gopkg.in/alecthomas/kingpin.v2"
)

var (
app = kingpin.New("buildtagger", "A tool for generating build tags (constraints) for the source modules of the official provider families.").DefaultEnvars()
)

var (
parentDir = app.Flag("parent-dir", "Parent directory which will be recursively walked to find the Go source files whose relative path to this parent matches the specified regular expression. The files found will be tagged using the specified build tag.").Default("./").String()
regex = app.Flag("regex", `The regular expression against which a discovered Go source file's relative path or name will be matched. This expression must contain one and only one group whose value will be substituted in the given tag format string. An example is "(.+)/.+/.+\.go"`).Default(".*").String()
tagFormat = app.Flag("tag-format", `A Printf format string to construct the build tag. An example is "(%s || all) && !ignore_autogenerated", where the "%s" format specifier can be replaced by a family resource provider group name.`+
`There should be a string format specifier for each of the capturing groups specified in the "regex".`).Default("!ignore_autogenerated").String()
mode = app.Flag("mode", `If "file", the file name of the discovered Go source is matched against the given regular expression. If "dir", the relative path of the source file is matched.`).Default("dir").Enum("file", "dir")
deleteTags = app.Flag("delete", `If set, the build tags are removed from the discovered Go sources, instead of being added.`).Default("false").Bool()
)

// addOrUpdateBuildTag traverses directories from the parent,
// updating or adding build tags in Go files. If a build tag already exists,
// it's replaced with the computed tags.
func addOrUpdateBuildTag(parent, regex, tagFormat, mode string, deleteTags bool) error {
re, err := regexp.Compile(regex)
kingpin.FatalIfError(err, "Failed to compile the given regular expression: %s", regex)
matched := false
err = filepath.Walk(parent, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}

if !info.IsDir() {
matchString, err := getMatchString(parent, path, mode)
if err != nil {
return err
}
matches := re.FindStringSubmatch(matchString)
if len(matches) == 0 {
return nil
}
matched = true
args := make([]any, len(matches)-1)
for i, a := range matches[1:] {
args[i] = a
}
tag := fmt.Sprintf("//go:build "+strings.TrimSpace(tagFormat), args...)
err = updateFileWithBuildTag(path, tag, deleteTags)
if err != nil {
return errors.Wrap(err, "failed to update the source file")
}
}
return nil
})
if err != nil {
return err
}
if matched {
return nil
}
return errors.Errorf("no Go source files under %s matched the regular expression %q", parent, regex)
}

func getMatchString(parent, path, mode string) (string, error) {
switch mode {
case "file":
// regex is matched against the filename
return filepath.Base(path), nil
case "dir":
// regex is matched against the relative path
matchString, err := filepath.Rel(parent, path)
return matchString, errors.Wrapf(err, "failed to determine the relative path of %s wrt to %s", path, parent)
default:
return "", errors.Errorf("unknown match mode %q", mode)
}
}

// updateFileWithBuildTag reads a Go file and updates or inserts the specified build tag.
func updateFileWithBuildTag(filePath, buildTag string, deleteTag bool) error {
content, err := os.ReadFile(filepath.Clean(filePath))
if err != nil {
return errors.Wrapf(err, "failed to read the source file at path %s", filePath)
}

lines := strings.Split(string(content), "\n")
if len(lines) < 1 {
return nil
}
var updatedLines []string
index, tagExists, emptyLineFollows := getLineStartIndex(lines, deleteTag)
updatedLines = lines[index+1:]
if !deleteTag {
addedLines := [2]string{buildTag}
trimIndex := 2
if emptyLineFollows && tagExists {
trimIndex = 1
}
updatedLines = append(addedLines[:trimIndex], updatedLines...)
}
// Write the updated content back to the file
return errors.Wrapf(os.WriteFile(filePath, []byte(strings.Join(updatedLines, "\n")), 0600), "failed to write the source file at path %s", filePath)
}

func getLineStartIndex(lines []string, deleteTag bool) (int, bool, bool) {
index := -1
tagExists := false
if strings.HasPrefix(lines[0], "//go:build") {
tagExists = true
index++
}
emptyLineFollows := len(lines) > 1 && strings.TrimSpace(lines[1]) == ""
if deleteTag && emptyLineFollows && tagExists {
index++
}
return index, tagExists, emptyLineFollows
}

func main() {
kingpin.MustParse(app.Parse(os.Args[1:]))
kingpin.FatalIfError(addOrUpdateBuildTag(*parentDir, *regex, *tagFormat, *mode, *deleteTags), "Failed to run the buildtagger...")
}
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/upbound/uptest

go 1.19
go 1.21

require (
cloud.google.com/go/storage v1.27.0
Expand Down
Loading

0 comments on commit fd8ec1e

Please sign in to comment.