Refactor versioner (Azure#1461)

* Ensures go.mod exists before doing actual work

* Fix wrong module name

* Add a new test scenario

* Some refinement on testcases for versioner

* Update version in version.go

* Only keep one line of tag in version.go

* Minor fix

* No need to print tag to stdout

* New test scenario

* Fix linter problem

* Return error from filepath.Abs

* Add test for update version.go file

* Remove method that ensures existence of go.mod file, and some redaudent
test cases

* Formatting code after versioning done

* Refine tests

* Fix a bug that when update applied to v2, the go.mod file does not
update as v2

* Clean up

* Purely file rename, nothing else changed

* Add functionality for run tool in root folder

* Add root command again

* Fix format

* Typo fix

* Add test for list all stage folders

* Add two functions for programmatically call

* Fix CI failure

* Add a new test case for more realistic

* Fix errors in the new test case

* Fix gofmt issues

* Add a new struct to cover this scenario

* Add test for go vet

* Add new test case for one line import

* Format stage folder first to avoid un-expected changes in code

* Fix a bug in regex used for multiline import statement

* Move getPkgs method to an internal package and exported

* Fix broken tests

* Add comment to silence golint error

* Init command

* Format change

* More minor tweak to satisfy golint

tools/internal/pkgs/pkgs.go

@@ -46,20 +46,12 @@ func (p Pkg) GetAPIVersion() (string, error) {
 	if p.IsARMPkg() {
 		// management-plane
 		regex := regexp.MustCompile(`mgmt/(.+)/`)
-		groups := regex.FindStringSubmatch(dest)
-		if len(groups) < 2 {
-			return "", fmt.Errorf("cannot find api version in %s", dest)
-		}
-		versionString := groups[1]
+		versionString := regex.FindStringSubmatch(dest)[1]
 		return versionString, nil
 	}
 	// data-plane
 	regex := regexp.MustCompile(`/(\d{4}-\d{2}.*|v?\d+(\.\d+)?)/`)
-	groups := regex.FindStringSubmatch(dest)
-	if len(groups) < 2 {
-		return "", fmt.Errorf("cannot find api version in %s", dest)
-	}
-	versionString := groups[1]
+	versionString := regex.FindStringSubmatch(dest)[1]
 	if versionString == "" {
 		return "", fmt.Errorf("does not find api version in data plane package %s", dest)
 	}

tools/testdata/README.md

@@ -8,10 +8,4 @@
 | d | Yes | Yes | v1, v2 | Yes |
 | e | Yes | No | v1, v2 | Yes |
 | f | Yes | No | None | Yes |
-| g | Yes | Yes | v1 | Yes |
-| h | Yes | Yes | v1 | Yes |
-| i | No | No | v1 | Yes |
-| j | No | No | v1, v2 | Yes |
-
-* For i, all files in v1 and stage are identical with each other. In this case, no version should be bumped.
-* For j, all files in v2 and stage are identical with each other. In this case, no version should be bumped.
+| g | Yes | Yes | v1 | Yes |

tools/testdata/scenariog/foo/mgmt/2019-10-11/foo/go.mod

@@ -1,3 +1,3 @@
-module github.com/Azure/azure-sdk-for-go/tools/testdata/scenariog/foo/mgmt/2019-10-11/foo
+module github.com/Azure/azure-sdk-for-go/tools/testdata/scenariog/foo
 
 go 1.12

tools/testdata/scenariog/foo/mgmt/2019-10-11/foo/stage/go.mod

@@ -1,3 +1,3 @@
-module github.com/Azure/azure-sdk-for-go/tools/testdata/scenariog/foo/mgmt/2019-10-11/foo
+module github.com/Azure/azure-sdk-for-go/tools/testdata/scenariog/foo
 
 go 1.12

tools/testdata/scenarioh/foo/mgmt/2019-10-11/foo/go.mod

@@ -1,3 +1,3 @@
-module github.com/Azure/azure-sdk-for-go/tools/testdata/scenarioh/foo/mgmt/2019-10-11/foo
+module github.com/Azure/azure-sdk-for-go/tools/testdata/scenarioh/foo
 
 go 1.12

tools/testdata/scenarioh/foo/mgmt/2019-10-11/foo/stage/go.mod

@@ -1,3 +1,3 @@
-module github.com/Azure/azure-sdk-for-go/tools/testdata/scenarioh/foo/mgmt/2019-10-11/foo
+module github.com/Azure/azure-sdk-for-go/tools/testdata/scenarioh/foo
 
 go 1.12

tools/versioner/cmd/init.go

@@ -126,9 +126,6 @@ func theInitCommand(args []string) error {
 		if err := createGoModFile(root, p); err != nil {
 			errs = append(errs, err)
 		}
-		if err := createChangeLogFile(root, p); err != nil {
-			errs = append(errs, err)
-		}
 	}
 	// handle errors
 	if len(errs) == 0 {
@@ -147,8 +144,7 @@ func createVersionFile(root string, p pkgs.Pkg, tagPrefix string) error {
 		return fmt.Errorf("failed to get api version of package %s: %+v", p.Dest, err)
 	}
 	tag := tagPrefix + "/" + startingModVer
-	ver := versionGoRegex.FindString(startingModVer)
-	content := fmt.Sprintf(initialVerGo, p.Package.Name, ver, p.Package.Name, apiVersion, ver, tag)
+	content := fmt.Sprintf(initialVerGo, p.Package.Name, startingModVer, p.Package.Name, apiVersion, startingModVer, tag)
 	err = ioutil.WriteFile(verFilePath, []byte(content), 0755)
 	if err != nil {
 		return fmt.Errorf("failed to write file %s: %+v", verFilePath, err)
@@ -172,17 +168,6 @@ func createGoModFile(root string, p pkgs.Pkg) error {
 	return nil
 }
 
-func createChangeLogFile(root string, p pkgs.Pkg) error {
-	logFilePath := filepath.Join(root, p.Dest, changeLogName)
-	log, err := os.Create(logFilePath)
-	if err != nil {
-		return fmt.Errorf("failed to create %s: %v", changeLogName, err)
-	}
-	defer log.Close()
-	_, err = log.WriteString("No changes to exported content compared to the previous release.\n")
-	return err
-}
-
 func loadExceptions(exceptFile string) (map[string]bool, error) {
 	result := make(map[string]bool)
 	if exceptFile == "" {

tools/versioner/cmd/root_test.go

@@ -46,8 +46,6 @@ func Test_findAllSubDirectories(t *testing.T) {
 		"../../testdata/scenariof/foo/stage",
 		"../../testdata/scenariog/foo/mgmt/2019-10-11/foo/stage",
 		"../../testdata/scenarioh/foo/mgmt/2019-10-11/foo/stage",
-		"../../testdata/scenarioi/foo/mgmt/2019-10-23/foo/stage",
-		"../../testdata/scenarioj/foo/mgmt/2019-10-23/foo/stage",
 	}
 	root, err := filepath.Abs("../../testdata")
 	if err != nil {

tools/versioner/cmd/unstage.go

@@ -58,8 +58,6 @@ var (
 	getTagsHook TagsHookFunc
 )
 
-const changeLogName = "CHANGELOG.md"
-
 // TagsHookFunc is a func used for get tags from remote
 type TagsHookFunc func(string, string) ([]string, error)
 
@@ -101,6 +99,9 @@ func theUnstageCommand(args []string) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("failed to create module info: %v", err)
 	}
+	if err := writeChangelog(stage, mod); err != nil {
+		return "", fmt.Errorf("failed to write changelog: %v", err)
+	}
 	var tag string
 	if mod.BreakingChanges() {
 		tag, err = forSideBySideRelease(stage, mod)
@@ -140,10 +141,6 @@ func forSideBySideRelease(stage string, mod modinfo.Provider) (string, error) {
 	if err := updateVersion(stage, tag); err != nil {
 		return "", fmt.Errorf("failed to update version.go: %v", err)
 	}
-	// write change log
-	if err := writeChangelog(stage, mod); err != nil {
-		return "", fmt.Errorf("failed to write changelog: %v", err)
-	}
 	// move staging to new LMV directory
 	if err = os.Rename(stage, mod.DestDir()); err != nil {
 		return "", fmt.Errorf("failed to rename '%s' to '%s': %v", stage, mod.DestDir(), err)
@@ -181,14 +178,6 @@ func forInplaceUpdate(lmv, stage string, mod modinfo.Provider) (string, error) {
 	if err := updateVersion(stage, tag); err != nil {
 		return "", fmt.Errorf("failed to update version.go: %v", err)
 	}
-	// write changelog
-	if hasChange, err := mod.HasChanges(); err != nil {
-		return "", fmt.Errorf("failed to check changes: %v", err)
-	} else if hasChange {
-		if err := writeChangelog(stage, mod); err != nil {
-			return "", fmt.Errorf("failed to write changelog: %v", err)
-		}
-	}
 	// move staging directory over the LMV by first deleting LMV then renaming stage
 	if modinfo.HasVersionSuffix(lmv) {
 		if err := os.RemoveAll(lmv); err != nil {
@@ -411,6 +400,7 @@ func writeChangelog(stage string, mod modinfo.Provider) error {
 	if mod.NewModule() {
 		return nil
 	}
+	const changeLogName = "CHANGELOG.md"
 	rpt := mod.GenerateReport()
 	log, err := os.Create(filepath.Join(stage, changeLogName))
 	if err != nil {
@@ -494,13 +484,9 @@ func calculateModuleTag(tags []string, mod modinfo.Provider) (string, error) {
 		n := sv.IncMinor()
 		sv = &n
 	} else {
-		// no new exports and has changes, this is a patch update
-		if hasChange, err := mod.HasChanges(); err != nil {
-			return "", err
-		} else if hasChange {
-			n := sv.IncPatch()
-			sv = &n
-		}
+		// no new exports, this is a patch update
+		n := sv.IncPatch()
+		sv = &n
 	}
 	return strings.Replace(tag, v, "v"+sv.String(), 1), nil
 }

tools/versioner/cmd/unstage_test.go

@@ -137,11 +137,6 @@ func (mock mockModInfo) GenerateReport() report.Package {
 	return report.Package{}
 }
 
-func (mock mockModInfo) HasChanges() (bool, error) {
-	// not needed by tests
-	return true, nil
-}
-
 func Test_calculateModuleTagMajorV1(t *testing.T) {
 	pkg := mockModInfo{
 		dir: filepath.Join("work", "src", "github.com", "Azure", "azure-sdk-for-go", "services", "foo"),
@@ -583,11 +578,11 @@ func Test_theCommandNewMgmtMajorV2(t *testing.T) {
 	cleanTestData()
 	defer cleanTestData()
 	getTagsHook = func(root string, prefix string) ([]string, error) {
-		if !strings.HasSuffix(prefix, "testdata/scenariog/foo/mgmt/2019-10-11/foo") {
+		if !strings.HasPrefix(prefix, "/testdata/scenariog/foo") {
 			return nil, fmt.Errorf("bad prefix '%s'", prefix)
 		}
 		return []string{
-			"tools/testdata/scenariog/foo/mgmt/2019-10-11/foo/v1.0.0",
+			"/testdata/scenariog/foo/mgmt/2019-10-11/foo/v1.0.0",
 		}, nil
 	}
 	stage, err := filepath.Abs("../../testdata/scenariog/foo/mgmt/2019-10-11/foo/stage")
@@ -602,7 +597,7 @@ func Test_theCommandNewMgmtMajorV2(t *testing.T) {
 	if tag != expectedTag {
 		t.Fatalf("bad tag, expected '%s' got '%s'", expectedTag, tag)
 	}
-	expectedMod := fmt.Sprintf("module github.com/Azure/azure-sdk-for-go/tools/testdata/scenariog/foo/mgmt/2019-10-11/foo/v2\n\n%s\n", goVersion)
+	expectedMod := fmt.Sprintf("module github.com/Azure/azure-sdk-for-go/tools/testdata/scenariog/foo/v2\n\n%s\n", goVersion)
 	verifyGoMod(t, "../../testdata/scenariog/foo/mgmt/2019-10-11/foo/v2", expectedMod)
 	verifyVersion(t, "../../testdata/scenariog/foo/mgmt/2019-10-11/foo/v2", "2.0.0", tag)
 	verifyChangelog(t, "../../testdata/scenariog/foo/mgmt/2019-10-11/foo/v2")
@@ -614,13 +609,13 @@ func Test_theCommandNewMgmtMajorV2WithOneLineImport(t *testing.T) {
 	cleanTestData()
 	defer cleanTestData()
 	getTagsHook = func(root string, prefix string) ([]string, error) {
-		if !strings.HasSuffix(prefix, "testdata/scenarioh/foo/mgmt/2019-10-11/foo") {
+		if !strings.HasPrefix(prefix, "/testdata/scenarioh/foo") {
 			return nil, fmt.Errorf("bad prefix '%s'", prefix)
 		}
 		return []string{
-			"tools/testdata/scenarioh/foo/mgmt/2019-10-11/foo/v1.0.0",
-			"tools/testdata/scenarioh/foo/mgmt/2019-10-11/foo/v1.1.0",
-			"tools/testdata/scenarioh/foo/mgmt/2019-10-11/foo/v1.2.0",
+			"/testdata/scenarioh/foo/mgmt/2019-10-11/foo/v1.0.0",
+			"/testdata/scenarioh/foo/mgmt/2019-10-11/foo/v1.1.0",
+			"/testdata/scenarioh/foo/mgmt/2019-10-11/foo/v1.2.0",
 		}, nil
 	}
 	stage, err := filepath.Abs("../../testdata/scenarioh/foo/mgmt/2019-10-11/foo/stage")
@@ -635,80 +630,9 @@ func Test_theCommandNewMgmtMajorV2WithOneLineImport(t *testing.T) {
 	if tag != expectedTag {
 		t.Fatalf("bad tag, expected '%s' got '%s'", expectedTag, tag)
 	}
-	expectedMod := fmt.Sprintf("module github.com/Azure/azure-sdk-for-go/tools/testdata/scenarioh/foo/mgmt/2019-10-11/foo/v2\n\n%s\n", goVersion)
+	expectedMod := fmt.Sprintf("module github.com/Azure/azure-sdk-for-go/tools/testdata/scenarioh/foo/v2\n\n%s\n", goVersion)
 	verifyGoMod(t, "../../testdata/scenarioh/foo/mgmt/2019-10-11/foo/v2", expectedMod)
 	verifyVersion(t, "../../testdata/scenarioh/foo/mgmt/2019-10-11/foo/v2", "2.0.0", tag)
 	verifyChangelog(t, "../../testdata/scenarioh/foo/mgmt/2019-10-11/foo/v2")
 	verifyGoVet(t, "../../testdata/scenarioh/foo/mgmt/2019-10-11/foo")
 }
-
-// scenarioi
-func Test_theCommandNoAnyChange(t *testing.T) {
-	cleanTestData()
-	defer cleanTestData()
-	getTagsHook = func(root string, prefix string) ([]string, error) {
-		if !strings.HasSuffix(prefix, "testdata/scenarioi/foo/mgmt/2019-10-23/foo") {
-			return nil, fmt.Errorf("bad prefix '%s'", prefix)
-		}
-		return []string{
-			"tools/testdata/scenarioi/foo/mgmt/2019-10-23/foo/v1.0.0",
-			"tools/testdata/scenarioi/foo/mgmt/2019-10-23/foo/v1.0.1",
-			"tools/testdata/scenarioi/foo/mgmt/2019-10-23/foo/v1.1.0",
-			"tools/testdata/scenarioi/foo/mgmt/2019-10-23/foo/v1.1.1",
-			"tools/testdata/scenarioi/foo/mgmt/2019-10-23/foo/v1.1.2",
-		}, nil
-	}
-	stage, err := filepath.Abs("../../testdata/scenarioi/foo/mgmt/2019-10-23/foo/stage")
-	if err != nil {
-		t.Fatalf("failed: %v", err)
-	}
-	tag, err := theUnstageCommand([]string{stage})
-	if err != nil {
-		t.Fatalf("failed: %v", err)
-	}
-	const expectedTag = "tools/testdata/scenarioi/foo/mgmt/2019-10-23/foo/v1.1.2"
-	if tag != expectedTag {
-		t.Fatalf("bad tag, expected '%s' got '%s'", expectedTag, tag)
-	}
-	expectedMod := fmt.Sprintf("module github.com/Azure/azure-sdk-for-go/tools/testdata/scenarioi/foo/mgmt/2019-10-23/foo\n\n%s\n", goVersion)
-	verifyGoMod(t, "../../testdata/scenarioi/foo/mgmt/2019-10-23/foo", expectedMod)
-	verifyVersion(t, "../../testdata/scenarioi/foo/mgmt/2019-10-23/foo", "1.1.2", tag)
-	verifyNoChangelog(t, "../../testdata/scenarioi/foo/mgmt/2019-10-23/foo")
-	verifyGoVet(t, "../../testdata/scenarioi/foo/mgmt/2019-10-23/foo")
-}
-
-// scenarioj
-func Test_theCommandNoAnyChangeV2(t *testing.T) {
-	cleanTestData()
-	defer cleanTestData()
-	getTagsHook = func(root string, prefix string) ([]string, error) {
-		if !strings.HasSuffix(prefix, "testdata/scenarioj/foo/mgmt/2019-10-23/foo/v2") {
-			return nil, fmt.Errorf("bad prefix '%s'", prefix)
-		}
-		return []string{
-			"tools/testdata/scenarioj/foo/mgmt/2019-10-23/foo/v1.0.0",
-			"tools/testdata/scenarioj/foo/mgmt/2019-10-23/foo/v1.0.1",
-			"tools/testdata/scenarioj/foo/mgmt/2019-10-23/foo/v1.1.0",
-			"tools/testdata/scenarioj/foo/mgmt/2019-10-23/foo/v1.1.1",
-			"tools/testdata/scenarioj/foo/mgmt/2019-10-23/foo/v1.1.2",
-			"tools/testdata/scenarioj/foo/mgmt/2019-10-23/foo/v2.0.0",
-		}, nil
-	}
-	stage, err := filepath.Abs("../../testdata/scenarioj/foo/mgmt/2019-10-23/foo/stage")
-	if err != nil {
-		t.Fatalf("failed: %v", err)
-	}
-	tag, err := theUnstageCommand([]string{stage})
-	if err != nil {
-		t.Fatalf("failed: %v", err)
-	}
-	const expectedTag = "tools/testdata/scenarioj/foo/mgmt/2019-10-23/foo/v2.0.0"
-	if tag != expectedTag {
-		t.Fatalf("bad tag, expected '%s' got '%s'", expectedTag, tag)
-	}
-	expectedMod := fmt.Sprintf("module github.com/Azure/azure-sdk-for-go/tools/testdata/scenarioj/foo/mgmt/2019-10-23/foo/v2\n\n%s\n", goVersion)
-	verifyGoMod(t, "../../testdata/scenarioj/foo/mgmt/2019-10-23/foo/v2", expectedMod)
-	verifyVersion(t, "../../testdata/scenarioj/foo/mgmt/2019-10-23/foo/v2", "2.0.0", tag)
-	verifyNoChangelog(t, "../../testdata/scenarioj/foo/mgmt/2019-10-23/foo/v2")
-	verifyGoVet(t, "../../testdata/scenarioj/foo/mgmt/2019-10-23/foo/v2")
-}