Refactor doctor (#12264)

* Refactor Logger

Refactor Logger to make a logger interface and make it possible to
wrap loggers for specific purposes.

* Refactor Doctor

    Move the gitea doctor functions into its own module.
    Use a logger for its messages instead of returning a results string[]

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Update modules/doctor/misc.go

Co-authored-by: 6543 <6543@obermui.de>

* Update modules/doctor/misc.go

Co-authored-by: 6543 <6543@obermui.de>

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

modules/doctor/authorizedkeys.go

@@ -0,0 +1,95 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package doctor
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+const tplCommentPrefix = `# gitea public key`
+
+func checkAuthorizedKeys(logger log.Logger, autofix bool) error {
+	if setting.SSH.StartBuiltinServer || !setting.SSH.CreateAuthorizedKeysFile {
+		return nil
+	}
+
+	fPath := filepath.Join(setting.SSH.RootPath, "authorized_keys")
+	f, err := os.Open(fPath)
+	if err != nil {
+		if !autofix {
+			logger.Critical("Unable to open authorized_keys file. ERROR: %v", err)
+			return fmt.Errorf("Unable to open authorized_keys file. ERROR: %v", err)
+		}
+		logger.Warn("Unable to open authorized_keys. (ERROR: %v). Attempting to rewrite...", err)
+		if err = models.RewriteAllPublicKeys(); err != nil {
+			logger.Critical("Unable to rewrite authorized_keys file. ERROR: %v", err)
+			return fmt.Errorf("Unable to rewrite authorized_keys file. ERROR: %v", err)
+		}
+	}
+	defer f.Close()
+
+	linesInAuthorizedKeys := map[string]bool{}
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, tplCommentPrefix) {
+			continue
+		}
+		linesInAuthorizedKeys[line] = true
+	}
+	f.Close()
+
+	// now we regenerate and check if there are any lines missing
+	regenerated := &bytes.Buffer{}
+	if err := models.RegeneratePublicKeys(regenerated); err != nil {
+		logger.Critical("Unable to regenerate authorized_keys file. ERROR: %v", err)
+		return fmt.Errorf("Unable to regenerate authorized_keys file. ERROR: %v", err)
+	}
+	scanner = bufio.NewScanner(regenerated)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, tplCommentPrefix) {
+			continue
+		}
+		if ok := linesInAuthorizedKeys[line]; ok {
+			continue
+		}
+		if !autofix {
+			logger.Critical(
+				"authorized_keys file %q is out of date.\nRegenerate it with:\n\t\"%s\"\nor\n\t\"%s\"",
+				fPath,
+				"gitea admin regenerate keys",
+				"gitea doctor --run authorized_keys --fix")
+			return fmt.Errorf(`authorized_keys is out of date and should be regenerated with "gitea admin regenerate keys" or "gitea doctor --run authorized_keys --fix"`)
+		}
+		logger.Warn("authorized_keys is out of date. Attempting rewrite...")
+		err = models.RewriteAllPublicKeys()
+		if err != nil {
+			logger.Critical("Unable to rewrite authorized_keys file. ERROR: %v", err)
+			return fmt.Errorf("Unable to rewrite authorized_keys file. ERROR: %v", err)
+		}
+	}
+	return nil
+}
+
+func init() {
+	Register(&Check{
+		Title:     "Check if OpenSSH authorized_keys file is up-to-date",
+		Name:      "authorized-keys",
+		IsDefault: true,
+		Run:       checkAuthorizedKeys,
+		Priority:  4,
+	})
+}

modules/doctor/dbconsistency.go

@@ -0,0 +1,127 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package doctor
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/models/migrations"
+	"code.gitea.io/gitea/modules/log"
+)
+
+func checkDBConsistency(logger log.Logger, autofix bool) error {
+	// make sure DB version is uptodate
+	if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil {
+		logger.Critical("Model version on the database does not match the current Gitea version. Model consistency will not be checked until the database is upgraded")
+		return err
+	}
+
+	// find labels without existing repo or org
+	count, err := models.CountOrphanedLabels()
+	if err != nil {
+		logger.Critical("Error: %v whilst counting orphaned labels")
+		return err
+	}
+
+	if count > 0 {
+		if autofix {
+			if err = models.DeleteOrphanedLabels(); err != nil {
+				logger.Critical("Error: %v whilst deleting orphaned labels")
+				return err
+			}
+			logger.Info("%d labels without existing repository/organisation deleted", count)
+		} else {
+			logger.Warn("%d labels without existing repository/organisation", count)
+		}
+	}
+
+	// find issues without existing repository
+	count, err = models.CountOrphanedIssues()
+	if err != nil {
+		logger.Critical("Error: %v whilst counting orphaned issues")
+		return err
+	}
+	if count > 0 {
+		if autofix {
+			if err = models.DeleteOrphanedIssues(); err != nil {
+				logger.Critical("Error: %v whilst deleting orphaned issues")
+				return err
+			}
+			logger.Info("%d issues without existing repository deleted", count)
+		} else {
+			logger.Warn("%d issues without existing repository", count)
+		}
+	}
+
+	// find pulls without existing issues
+	count, err = models.CountOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id")
+	if err != nil {
+		logger.Critical("Error: %v whilst counting orphaned objects")
+		return err
+	}
+	if count > 0 {
+		if autofix {
+			if err = models.DeleteOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil {
+				logger.Critical("Error: %v whilst deleting orphaned objects")
+				return err
+			}
+			logger.Info("%d pull requests without existing issue deleted", count)
+		} else {
+			logger.Warn("%d pull requests without existing issue", count)
+		}
+	}
+
+	// find tracked times without existing issues/pulls
+	count, err = models.CountOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id")
+	if err != nil {
+		logger.Critical("Error: %v whilst counting orphaned objects")
+		return err
+	}
+	if count > 0 {
+		if autofix {
+			if err = models.DeleteOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil {
+				logger.Critical("Error: %v whilst deleting orphaned objects")
+				return err
+			}
+			logger.Info("%d tracked times without existing issue deleted", count)
+		} else {
+			logger.Warn("%d tracked times without existing issue", count)
+		}
+	}
+
+	// find null archived repositories
+	count, err = models.CountNullArchivedRepository()
+	if err != nil {
+		logger.Critical("Error: %v whilst counting null archived repositories")
+		return err
+	}
+	if count > 0 {
+		if autofix {
+			updatedCount, err := models.FixNullArchivedRepository()
+			if err != nil {
+				logger.Critical("Error: %v whilst fixing null archived repositories")
+				return err
+			}
+			logger.Info("%d repositories with null is_archived updated", updatedCount)
+		} else {
+			logger.Warn("%d repositories with null is_archived", count)
+		}
+	}
+
+	// TODO: function to recalc all counters
+
+	return nil
+}
+
+func init() {
+	Register(&Check{
+		Title:     "Check consistency of database",
+		Name:      "check-db-consistency",
+		IsDefault: false,
+		Run:       checkDBConsistency,
+		Priority:  3,
+	})
+}

modules/doctor/dbversion.go

@@ -0,0 +1,42 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package doctor
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/models/migrations"
+	"code.gitea.io/gitea/modules/log"
+)
+
+func checkDBVersion(logger log.Logger, autofix bool) error {
+	if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil {
+		if !autofix {
+			logger.Critical("Error: %v during ensure up to date", err)
+			return err
+		}
+		logger.Warn("Got Error: %v during ensure up to date", err)
+		logger.Warn("Attempting to migrate to the latest DB version to fix this.")
+
+		err = models.NewEngine(context.Background(), migrations.Migrate)
+		if err != nil {
+			logger.Critical("Error: %v during migration")
+		}
+		return err
+	}
+	return nil
+}
+
+func init() {
+	Register(&Check{
+		Title:         "Check Database Version",
+		Name:          "check-db-version",
+		IsDefault:     true,
+		Run:           checkDBVersion,
+		AbortIfFailed: false,
+		Priority:      2,
+	})
+}

modules/doctor/doctor.go

@@ -0,0 +1,105 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package doctor
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// Check represents a Doctor check
+type Check struct {
+	Title                      string
+	Name                       string
+	IsDefault                  bool
+	Run                        func(logger log.Logger, autofix bool) error
+	AbortIfFailed              bool
+	SkipDatabaseInitialization bool
+	Priority                   int
+}
+
+type wrappedLevelLogger struct {
+	log.LevelLogger
+}
+
+func (w *wrappedLevelLogger) Log(skip int, level log.Level, format string, v ...interface{}) error {
+	return w.LevelLogger.Log(
+		skip+1,
+		level,
+		" - %s "+format,
+		append(
+			[]interface{}{
+				log.NewColoredValueBytes(
+					fmt.Sprintf("[%s]", strings.ToUpper(level.String()[0:1])),
+					level.Color()),
+			}, v...)...)
+}
+
+func initDBDisableConsole(disableConsole bool) error {
+	setting.NewContext()
+	setting.InitDBConfig()
+
+	setting.NewXORMLogService(disableConsole)
+	if err := models.SetEngine(); err != nil {
+		return fmt.Errorf("models.SetEngine: %v", err)
+	}
+	return nil
+}
+
+// Checks is the list of available commands
+var Checks []*Check
+
+// RunChecks runs the doctor checks for the provided list
+func RunChecks(logger log.Logger, autofix bool, checks []*Check) error {
+	wrappedLogger := log.LevelLoggerLogger{
+		LevelLogger: &wrappedLevelLogger{logger},
+	}
+
+	dbIsInit := false
+	for i, check := range checks {
+		if !dbIsInit && !check.SkipDatabaseInitialization {
+			// Only open database after the most basic configuration check
+			setting.EnableXORMLog = false
+			if err := initDBDisableConsole(true); err != nil {
+				logger.Error("Error whilst initializing the database: %v", err)
+				logger.Error("Check if you are using the right config file. You can use a --config directive to specify one.")
+				return nil
+			}
+			dbIsInit = true
+		}
+		logger.Info("[%d] %s", log.NewColoredIDValue(i+1), check.Title)
+		logger.Flush()
+		if err := check.Run(&wrappedLogger, autofix); err != nil {
+			if check.AbortIfFailed {
+				logger.Critical("FAIL")
+				return err
+			}
+			logger.Error("ERROR")
+		} else {
+			logger.Info("OK")
+			logger.Flush()
+		}
+	}
+	return nil
+}
+
+// Register registers a command with the list
+func Register(command *Check) {
+	Checks = append(Checks, command)
+	sort.SliceStable(Checks, func(i, j int) bool {
+		if Checks[i].Priority == Checks[j].Priority {
+			return Checks[i].Name < Checks[j].Name
+		}
+		if Checks[i].Priority == 0 {
+			return false
+		}
+		return Checks[i].Priority < Checks[j].Priority
+	})
+}