From 8a7b245fb42169aa8bd9b588878c2109b7961fab Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 21 Jul 2019 20:51:40 +0200 Subject: [PATCH 1/5] Add release note generation tool * no external dependencies * inspects PRs by version label * generates structured release notes in asciidoc grouped by type label --- dev-tools/release-notes/main/main.go | 265 +++++++++++++++++++++ dev-tools/release-notes/main/main_test.go | 271 ++++++++++++++++++++++ 2 files changed, 536 insertions(+) create mode 100644 dev-tools/release-notes/main/main.go create mode 100644 dev-tools/release-notes/main/main_test.go diff --git a/dev-tools/release-notes/main/main.go b/dev-tools/release-notes/main/main.go new file mode 100644 index 0000000000..59ed56d414 --- /dev/null +++ b/dev-tools/release-notes/main/main.go @@ -0,0 +1,265 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "html/template" + "io" + "net/http" + "os" + "regexp" + "sort" + "strconv" + "strings" +) + +const ( + baseURL = "https://api.github.com/repos/" + repo = "elastic/cloud-on-k8s/" + releaseNoteTemplate = `:issue: https://github.com/{{.Repo}}issues/ +:pull: https://github.com/{{.Repo}}pull/ + +[[release-notes-{{.Version}}]] +== {n} version {{.Version}} +{{range $group, $prs := .Groups}} +[[{{- id $group -}}-{{$.Version}}]] +[float] +=== {{index $.GroupLabels $group}} +{{range $prs}} +* {{.Title}} {pull}{{.Number}}[#{{.Number}}]{{with .RelatedIssues -}} +{{$length := len .}} (issue{{if gt $length 1}}s{{end}}: {{range $idx, $el := .}}{{if $idx}}, {{end}}{issue}{{$el}}[#{{$el}}]{{end}}) +{{- end}} +{{- end}} +{{end}} +` +) + +var ( + groupLabels = map[string]string{ + ">breaking": "Breaking changes", + ">deprecation": "Deprecations", + ">feature": "New features", + ">enhancement": "Enhancements", + ">bug": "Bug fixes", + "nogroup": "Misc", + } + + ignore = map[string]bool{ + ">non-issue": true, + ">refactoring": true, + ">docs": true, + ">test": true, + ":ci": true, + "backport": true, + } +) + +func fetch(url string, out interface{}) (*string, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + + nextLink := extractNextLink(resp.Header) + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, errors.New(fmt.Sprintf("%s: %d %s ", url, resp.StatusCode, resp.Status)) + } + + defer resp.Body.Close() + if err = json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, err + } + return nextLink, nil + +} + +func extractNextLink(headers http.Header) *string { + var nextLink *string + nextRe := regexp.MustCompile(`<([^>]+)>; rel="next"`) + links := headers["Link"] + for _, lnk := range links { + matches := nextRe.FindAllStringSubmatch(lnk, 1) + if matches != nil && matches[0][1] != "" { + nextLink = &matches[0][1] + break + } + } + return nextLink +} + +type Label struct { + Name string `json:"name"` +} + +func fetchVersionLabels() ([]string, error) { + var versionLabels []string + url := fmt.Sprintf("%s%slabels?page=1", baseURL, repo) +FETCH: + var labels []Label + next, err := fetch(url, &labels) + if err != nil { + return nil, err + } + for _, l := range labels { + if strings.HasPrefix(l.Name, "v") { + versionLabels = append(versionLabels, l.Name) + } + } + if next != nil { + url = *next + goto FETCH + } + + return versionLabels, nil +} + +type Issue struct { + Labels []Label `json:"labels"` + Body string `json:"body"` + Title string `json:"title"` + Number int `json:"number"` + PullRequest map[string]string `json:"pull_request,omitempty"` + RelatedIssues []int +} + +type GroupedIssues = map[string][]Issue + +func fetchIssues(version string) (GroupedIssues, error) { + url := fmt.Sprintf("%s%sissues?labels=%s&pagesize=100&state=all&page=1", baseURL, repo, version) + var prs []Issue +FETCH: + var tranche []Issue + next, err := fetch(url, &tranche) + if err != nil { + return nil, err + } + for _, issue := range tranche { + // only look at PRs + if issue.PullRequest != nil { + prs = append(prs, issue) + } + } + if next != nil { + url = *next + goto FETCH + } + result := make(GroupedIssues) + noGroup := "nogroup" +PR: + for _, pr := range prs { + prLabels := make(map[string]bool) + for _, lbl := range pr.Labels { + // remove PRs that have labels to be ignored + if ignore[lbl.Name] { + continue PR + } + // build a lookup table of all labels for this PR + prLabels[lbl.Name] = true + } + + // extract related issues from PR body + if err := extractRelatedIssues(&pr); err != nil { + return nil, err + } + + // group PRs by type label + for typeLabel := range groupLabels { + if prLabels[typeLabel] { + result[typeLabel] = append(result[typeLabel], pr) + continue PR + } + } + // or fall back to a default group + result[noGroup] = append(result[noGroup], pr) + } + return result, nil +} + +func extractRelatedIssues(issue *Issue) error { + re := regexp.MustCompile(fmt.Sprintf(`https://github.com/%sissues/(\d+)`, repo)) + matches := re.FindAllStringSubmatch(issue.Body, -1) + issues := map[int]struct{}{} + for _, capture := range matches { + issueNum, err := strconv.Atoi(capture[1]) + if err != nil { + return err + } + issues[issueNum] = struct{}{} + + } + for rel := range issues { + issue.RelatedIssues = append(issue.RelatedIssues, rel) + } + sort.Ints(issue.RelatedIssues) + return nil +} + +type TemplateParams struct { + Version string + Repo string + GroupLabels map[string]string + Groups GroupedIssues +} + +func dumpIssues(params TemplateParams, out io.Writer) { + funcs := template.FuncMap{ + "id": func(s string) string { + return strings.TrimPrefix(s, ">") + }, + } + tpl := template.Must(template.New("release_notes").Funcs(funcs).Parse(releaseNoteTemplate)) + err := tpl.Execute(out, params) + if err != nil { + println(err) + } +} + +func main() { + labels, err := fetchVersionLabels() + if err != nil { + panic(err) + } + + if len(os.Args) != 2 { + usage(labels) + } + + version := os.Args[1] + found := false + for _, l := range labels { + if l == version { + found = true + } + } + if !found { + usage(labels) + } + + groupedIssues, err := fetchIssues(version) + if err != nil { + panic(err) + } + dumpIssues(TemplateParams{ + Version: strings.TrimPrefix(version, "v"), + Repo: repo, + GroupLabels: groupLabels, + Groups: groupedIssues, + }, os.Stdout) + +} + +func usage(labels []string) { + println(fmt.Sprintf("USAGE: %s version > outfile", os.Args[0])) + println("Known versions:") + sort.Strings(labels) + for _, l := range labels { + println(l) + } + os.Exit(1) +} diff --git a/dev-tools/release-notes/main/main_test.go b/dev-tools/release-notes/main/main_test.go new file mode 100644 index 0000000000..507c67c40a --- /dev/null +++ b/dev-tools/release-notes/main/main_test.go @@ -0,0 +1,271 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "bytes" + "net/http" + "reflect" + "testing" +) + +func Test_dumpIssues(t *testing.T) { + type args struct { + params TemplateParams + } + tests := []struct { + name string + args args + wantOut string + }{ + { + name: "two issues--no related", + args: args{ + params: TemplateParams{ + Version: "0.9.0", + Repo: "me/my-repo/", + GroupLabels: map[string]string{ + ">bugs": "Bug Fixes", + }, + Groups: GroupedIssues{ + ">bugs": []Issue{ + { + Labels: nil, + Body: "body", + Title: "title", + Number: 123, + PullRequest: nil, + RelatedIssues: nil, + }, + { + Labels: nil, + Body: "body2", + Title: "title2", + Number: 456, + PullRequest: nil, + RelatedIssues: nil, + }, + }, + }, + }, + }, + wantOut: `:issue: https://github.com/me/my-repo/issues/ +:pull: https://github.com/me/my-repo/pull/ + +[[release-notes-0.9.0]] +== {n} version 0.9.0 + +[[bugs-0.9.0]] +[float] +=== Bug Fixes + +* title {pull}123[#123] +* title2 {pull}456[#456] + +`, + }, + { + name: "single issue with related", + args: args{ + params: TemplateParams{ + Version: "0.9.0", + Repo: "me/my-repo/", + GroupLabels: map[string]string{ + ">bugs": "Bug Fixes", + }, + Groups: GroupedIssues{ + ">bugs": []Issue{ + { + Labels: nil, + Body: "body", + Title: "title", + Number: 123, + PullRequest: nil, + RelatedIssues: []int{456}, + }, + }, + }, + }, + }, + wantOut: `:issue: https://github.com/me/my-repo/issues/ +:pull: https://github.com/me/my-repo/pull/ + +[[release-notes-0.9.0]] +== {n} version 0.9.0 + +[[bugs-0.9.0]] +[float] +=== Bug Fixes + +* title {pull}123[#123] (issue: {issue}456[#456]) + +`, + }, + { + name: "single issue--two related", + args: args{ + params: TemplateParams{ + Version: "0.9.0", + Repo: "me/my-repo/", + GroupLabels: map[string]string{ + ">bugs": "Bug Fixes", + }, + Groups: GroupedIssues{ + ">bugs": []Issue{ + { + Labels: nil, + Body: "body", + Title: "title", + Number: 123, + PullRequest: nil, + RelatedIssues: []int{456, 789}, + }, + }, + }, + }, + }, + wantOut: `:issue: https://github.com/me/my-repo/issues/ +:pull: https://github.com/me/my-repo/pull/ + +[[release-notes-0.9.0]] +== {n} version 0.9.0 + +[[bugs-0.9.0]] +[float] +=== Bug Fixes + +* title {pull}123[#123] (issues: {issue}456[#456], {issue}789[#789]) + +`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := &bytes.Buffer{} + dumpIssues(tt.args.params, out) + if gotOut := out.String(); gotOut != tt.wantOut { + t.Errorf("dumpIssues() = %v, want %v", gotOut, tt.wantOut) + } + }) + } +} + +func Test_extractRelatedIssues(t *testing.T) { + type args struct { + issue *Issue + } + tests := []struct { + name string + args args + want []int + wantErr bool + }{ + { + name: "single issue", + args: args{ + issue: &Issue{ + Body: "Resolves https://github.com/elastic/cloud-on-k8s/issues/1241\r\n\r\n* If there is no existing annotation on a resource", + }, + }, + want: []int{1241}, + wantErr: false, + }, + { + name: "multi issue", + args: args{ + issue: &Issue{ + Body: "Resolves https://github.com/elastic/cloud-on-k8s/issues/1241\r\n\r\nRelated https://github.com/elastic/cloud-on-k8s/issues/1245\r\n\r\n", + }, + }, + want: []int{1241, 1245}, + wantErr: false, + }, + { + name: "non issue", + args: args{ + issue: &Issue{ + Body: "Resolves https://github.com/elastic/cloud-on-k8s/issues/1241\r\n\r\nSee all issues https://github.com/elastic/cloud-on-k8s/issues/\r\n\r\n", + }, + }, + want: []int{1241}, + wantErr: false, + }, + { + name: "duplicate issue", + args: args{ + issue: &Issue{ + Body: "Resolves https://github.com/elastic/cloud-on-k8s/issues/1241\r\n\r\nRelated https://github.com/elastic/cloud-on-k8s/issues/1241\r\n\r\n", + }, + }, + want: []int{1241}, + wantErr: false, + }, + { + name: "ordered", + args: args{ + issue: &Issue{ + Body: "Resolves https://github.com/elastic/cloud-on-k8s/issues/1245\r\n\r\nRelated https://github.com/elastic/cloud-on-k8s/issues/1241\r\n\r\n", + }, + }, + want: []int{1241, 1245}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := extractRelatedIssues(tt.args.issue); (err != nil) != tt.wantErr { + t.Errorf("extractRelatedIssues() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(tt.want, tt.args.issue.RelatedIssues) { + t.Errorf("extractRelatedIssues() got = %v, want %v", tt.args.issue.RelatedIssues, tt.want) + } + }) + } +} + +func Test_extractNextLink(t *testing.T) { + testFixture := "https://api.github.com/repositories/155368246/issues?page=2" + type args struct { + headers http.Header + } + tests := []struct { + name string + args args + want *string + }{ + { + name: "no link", + args: args{ + headers: http.Header{}, + }, + want: nil, + }, + { + name: "with next link", + args: args{ + headers: http.Header{ + "Link": []string{ + `; rel="next", ; rel="last"`, + }, + }, + }, + want: &testFixture, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := extractNextLink(tt.args.headers); got != tt.want { + if got != nil && tt.want != nil { + if *got != *tt.want { + t.Errorf("extractNextLink() = %v, want %v", *got, *tt.want) + } + } else { + t.Errorf("extractNextLink() = %v, want %v", got, tt.want) + } + + } + }) + } +} From 4b92e3b80984c0f5945f1669b9d7c5b12c690517 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 21 Jul 2019 21:03:04 +0200 Subject: [PATCH 2/5] remove string pointer --- dev-tools/release-notes/main/main.go | 68 ++++++++++++----------- dev-tools/release-notes/main/main_test.go | 16 ++---- 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/dev-tools/release-notes/main/main.go b/dev-tools/release-notes/main/main.go index 59ed56d414..e2968245e1 100644 --- a/dev-tools/release-notes/main/main.go +++ b/dev-tools/release-notes/main/main.go @@ -59,44 +59,64 @@ var ( } ) -func fetch(url string, out interface{}) (*string, error) { +// Label models a subset of a GitHub label. +type Label struct { + Name string `json:"name"` +} + +// Issue models a subset of a Github issue. +type Issue struct { + Labels []Label `json:"labels"` + Body string `json:"body"` + Title string `json:"title"` + Number int `json:"number"` + PullRequest map[string]string `json:"pull_request,omitempty"` + RelatedIssues []int +} + +type GroupedIssues = map[string][]Issue + +type TemplateParams struct { + Version string + Repo string + GroupLabels map[string]string + Groups GroupedIssues +} + +func fetch(url string, out interface{}) (string, error) { resp, err := http.Get(url) if err != nil { - return nil, err + return "", err } nextLink := extractNextLink(resp.Header) if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, errors.New(fmt.Sprintf("%s: %d %s ", url, resp.StatusCode, resp.Status)) + return "", errors.New(fmt.Sprintf("%s: %d %s ", url, resp.StatusCode, resp.Status)) } defer resp.Body.Close() if err = json.NewDecoder(resp.Body).Decode(&out); err != nil { - return nil, err + return "", err } return nextLink, nil } -func extractNextLink(headers http.Header) *string { - var nextLink *string +func extractNextLink(headers http.Header) string { + var nextLink string nextRe := regexp.MustCompile(`<([^>]+)>; rel="next"`) links := headers["Link"] for _, lnk := range links { matches := nextRe.FindAllStringSubmatch(lnk, 1) if matches != nil && matches[0][1] != "" { - nextLink = &matches[0][1] + nextLink = matches[0][1] break } } return nextLink } -type Label struct { - Name string `json:"name"` -} - func fetchVersionLabels() ([]string, error) { var versionLabels []string url := fmt.Sprintf("%s%slabels?page=1", baseURL, repo) @@ -111,25 +131,14 @@ FETCH: versionLabels = append(versionLabels, l.Name) } } - if next != nil { - url = *next + if next != "" { + url = next goto FETCH } return versionLabels, nil } -type Issue struct { - Labels []Label `json:"labels"` - Body string `json:"body"` - Title string `json:"title"` - Number int `json:"number"` - PullRequest map[string]string `json:"pull_request,omitempty"` - RelatedIssues []int -} - -type GroupedIssues = map[string][]Issue - func fetchIssues(version string) (GroupedIssues, error) { url := fmt.Sprintf("%s%sissues?labels=%s&pagesize=100&state=all&page=1", baseURL, repo, version) var prs []Issue @@ -145,8 +154,8 @@ FETCH: prs = append(prs, issue) } } - if next != nil { - url = *next + if next != "" { + url = next goto FETCH } result := make(GroupedIssues) @@ -200,13 +209,6 @@ func extractRelatedIssues(issue *Issue) error { return nil } -type TemplateParams struct { - Version string - Repo string - GroupLabels map[string]string - Groups GroupedIssues -} - func dumpIssues(params TemplateParams, out io.Writer) { funcs := template.FuncMap{ "id": func(s string) string { diff --git a/dev-tools/release-notes/main/main_test.go b/dev-tools/release-notes/main/main_test.go index 507c67c40a..e7bf002206 100644 --- a/dev-tools/release-notes/main/main_test.go +++ b/dev-tools/release-notes/main/main_test.go @@ -226,21 +226,20 @@ func Test_extractRelatedIssues(t *testing.T) { } func Test_extractNextLink(t *testing.T) { - testFixture := "https://api.github.com/repositories/155368246/issues?page=2" type args struct { headers http.Header } tests := []struct { name string args args - want *string + want string }{ { name: "no link", args: args{ headers: http.Header{}, }, - want: nil, + want: "", }, { name: "with next link", @@ -251,20 +250,13 @@ func Test_extractNextLink(t *testing.T) { }, }, }, - want: &testFixture, + want: "https://api.github.com/repositories/155368246/issues?page=2", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := extractNextLink(tt.args.headers); got != tt.want { - if got != nil && tt.want != nil { - if *got != *tt.want { - t.Errorf("extractNextLink() = %v, want %v", *got, *tt.want) - } - } else { - t.Errorf("extractNextLink() = %v, want %v", got, tt.want) - } - + t.Errorf("extractNextLink() = %v, want %v", got, tt.want) } }) } From d34a1cab9861ad0c48e62046662d01fef82249e4 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 21 Jul 2019 21:36:05 +0200 Subject: [PATCH 3/5] remove empty line --- dev-tools/release-notes/main/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/dev-tools/release-notes/main/main.go b/dev-tools/release-notes/main/main.go index e2968245e1..90bc59732f 100644 --- a/dev-tools/release-notes/main/main.go +++ b/dev-tools/release-notes/main/main.go @@ -100,7 +100,6 @@ func fetch(url string, out interface{}) (string, error) { return "", err } return nextLink, nil - } func extractNextLink(headers http.Header) string { From f196b1de335bd20b699e4a01797c5094593a43eb Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Mon, 22 Jul 2019 11:54:34 +0200 Subject: [PATCH 4/5] always close body --- dev-tools/release-notes/main/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/release-notes/main/main.go b/dev-tools/release-notes/main/main.go index 90bc59732f..f5ef7ad8f7 100644 --- a/dev-tools/release-notes/main/main.go +++ b/dev-tools/release-notes/main/main.go @@ -88,6 +88,7 @@ func fetch(url string, out interface{}) (string, error) { if err != nil { return "", err } + defer resp.Body.Close() nextLink := extractNextLink(resp.Header) @@ -95,7 +96,6 @@ func fetch(url string, out interface{}) (string, error) { return "", errors.New(fmt.Sprintf("%s: %d %s ", url, resp.StatusCode, resp.Status)) } - defer resp.Body.Close() if err = json.NewDecoder(resp.Body).Decode(&out); err != nil { return "", err } From a0bbffbcce3efbfa20bebe0a91390c3ec91f9b98 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Mon, 22 Jul 2019 12:03:21 +0200 Subject: [PATCH 5/5] move to hack --- .../release-notes/main/main.go => operators/hack/release_notes.go | 0 .../main/main_test.go => operators/hack/release_notes_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename dev-tools/release-notes/main/main.go => operators/hack/release_notes.go (100%) rename dev-tools/release-notes/main/main_test.go => operators/hack/release_notes_test.go (100%) diff --git a/dev-tools/release-notes/main/main.go b/operators/hack/release_notes.go similarity index 100% rename from dev-tools/release-notes/main/main.go rename to operators/hack/release_notes.go diff --git a/dev-tools/release-notes/main/main_test.go b/operators/hack/release_notes_test.go similarity index 100% rename from dev-tools/release-notes/main/main_test.go rename to operators/hack/release_notes_test.go