Merge remote-tracking branch 'giteaoffical/main'

* giteaoffical/main:
  Implement sync push mirror on commit (go-gitea#19411)
  Use git.HOME_PATH for Git HOME directory (go-gitea#20114)

custom/conf/app.example.ini

@@ -603,7 +603,10 @@ ROUTER = console
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 ;; The path of git executable. If empty, Gitea searches through the PATH environment.
-PATH =
+;PATH =
+;;
+;; The HOME directory for Git
+;HOME_PATH = %(APP_DATA_PATH)/home
 ;;
 ;; Disables highlight of added and removed changes
 ;DISABLE_DIFF_HIGHLIGHT = false

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -948,6 +948,8 @@ Default templates for project boards:
 ## Git (`git`)
 
 - `PATH`: **""**: The path of Git executable. If empty, Gitea searches through the PATH environment.
+- `HOME_PATH`: **%(APP_DATA_PATH)/home**: The HOME directory for Git.
+	  This directory will be used to contain the `.gitconfig` and possible `.gnupg` directories that Gitea's git calls will use. If you can confirm Gitea is the only application running in this environment, you can set it to the normal home directory for Gitea user.
 - `DISABLE_DIFF_HIGHLIGHT`: **false**: Disables highlight of added and removed changes.
 - `MAX_GIT_DIFF_LINES`: **1000**: Max number of lines allowed of a single file in diff view.
 - `MAX_GIT_DIFF_LINE_CHARACTERS`: **5000**: Max character count per line highlighted in diff view.

docs/content/doc/advanced/signing.en-us.md

@@ -97,10 +97,11 @@ repositories, `SIGNING_KEY=default` could be used to provide different
 signing keys on a per-repository basis. However, this is clearly not an
 ideal UI and therefore subject to change.
 
-**Since 1.17**, Gitea runs git in its own home directory `[repository].ROOT` and uses its own config `{[repository].ROOT}/.gitconfig`.
+**Since 1.17**, Gitea runs git in its own home directory `[git].HOME_PATH` (default to `%(APP_DATA_PATH)/home`)
+and uses its own config `{[git].HOME_PATH}/.gitconfig`.
 If you have your own customized git config for Gitea, you should set these configs in system git config (aka `/etc/gitconfig`)
-or the Gitea internal git config `{[repository].ROOT}/.gitconfig`. 
-Related home files for git command (like `.gnupg`) should also be put in Gitea's git home directory `[repository].ROOT`. 
+or the Gitea internal git config `{[git].HOME_PATH}/.gitconfig`. 
+Related home files for git command (like `.gnupg`) should also be put in Gitea's git home directory `[git].HOME_PATH`. 
 
 
 ### `INITIAL_COMMIT`

models/migrations/migrations.go

@@ -396,6 +396,8 @@ var migrations = []Migration{
 	NewMigration("Alter hook_task table TEXT fields to LONGTEXT", alterHookTaskTextFieldsToLongText),
 	// v218 -> v219
 	NewMigration("Improve Action table indices v2", improveActionTableIndices),
+	// v219 -> v220
+	NewMigration("Add sync_on_commit column to push_mirror table", addSyncOnCommitColForPushMirror),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v219.go

@@ -0,0 +1,30 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"time"
+
+	"code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/timeutil"
+	"xorm.io/xorm"
+)
+
+func addSyncOnCommitColForPushMirror(x *xorm.Engine) error {
+	type PushMirror struct {
+		ID         int64            `xorm:"pk autoincr"`
+		RepoID     int64            `xorm:"INDEX"`
+		Repo       *repo.Repository `xorm:"-"`
+		RemoteName string
+
+		SyncOnCommit   bool `xorm:"NOT NULL DEFAULT true"`
+		Interval       time.Duration
+		CreatedUnix    timeutil.TimeStamp `xorm:"created"`
+		LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
+		LastError      string             `xorm:"text"`
+	}
+
+	return x.Sync2(new(PushMirror))
+}

models/repo/pushmirror.go

@@ -23,6 +23,7 @@ type PushMirror struct {
 	Repo       *Repository `xorm:"-"`
 	RemoteName string
 
+	SyncOnCommit   bool `xorm:"NOT NULL DEFAULT true"`
 	Interval       time.Duration
 	CreatedUnix    timeutil.TimeStamp `xorm:"created"`
 	LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
@@ -93,6 +94,14 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) {
 	return mirrors, db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID).Find(&mirrors)
 }
 
+// GetPushMirrorsSyncedOnCommit returns push-mirrors for this repo that should be updated by new commits
+func GetPushMirrorsSyncedOnCommit(repoID int64) ([]*PushMirror, error) {
+	mirrors := make([]*PushMirror, 0, 10)
+	return mirrors, db.GetEngine(db.DefaultContext).
+		Where("repo_id=? AND sync_on_commit=?", repoID, true).
+		Find(&mirrors)
+}
+
 // PushMirrorsIterate iterates all push-mirror repositories.
 func PushMirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
 	return db.GetEngine(db.DefaultContext).

models/unittest/testdb.go

@@ -107,6 +107,8 @@ func MainTest(m *testing.M, testOpts *TestOptions) {
 
 	setting.Packages.Storage.Path = filepath.Join(setting.AppDataPath, "packages")
 
+	setting.Git.HomePath = filepath.Join(setting.AppDataPath, "home")
+
 	if err = storage.Init(); err != nil {
 		fatalTestError("storage.Init: %v\n", err)
 	}

modules/git/command.go

@@ -105,23 +105,36 @@ type RunOpts struct {
 	PipelineFunc   func(context.Context, context.CancelFunc) error
 }
 
-// CommonGitCmdEnvs returns the common environment variables for a "git" command.
-func CommonGitCmdEnvs() []string {
+func commonBaseEnvs() []string {
 	// at the moment, do not set "GIT_CONFIG_NOSYSTEM", users may have put some configs like "receive.certNonceSeed" in it
-	return []string{
-		fmt.Sprintf("LC_ALL=%s", DefaultLocale),
-		"GIT_TERMINAL_PROMPT=0",    // avoid prompting for credentials interactively, supported since git v2.3
-		"GIT_NO_REPLACE_OBJECTS=1", // ignore replace references (https://git-scm.com/docs/git-replace)
+	envs := []string{
 		"HOME=" + HomeDir(),        // make Gitea use internal git config only, to prevent conflicts with user's git config
+		"GIT_NO_REPLACE_OBJECTS=1", // ignore replace references (https://git-scm.com/docs/git-replace)
+	}
+
+	// some environment variables should be passed to git command
+	passThroughEnvKeys := []string{
+		"GNUPGHOME", // git may call gnupg to do commit signing
+	}
+	for _, key := range passThroughEnvKeys {
+		if val, ok := os.LookupEnv(key); ok {
+			envs = append(envs, key+"="+val)
+		}
 	}
+	return envs
+}
+
+// CommonGitCmdEnvs returns the common environment variables for a "git" command.
+func CommonGitCmdEnvs() []string {
+	return append(commonBaseEnvs(), []string{
+		"LC_ALL=" + DefaultLocale,
+		"GIT_TERMINAL_PROMPT=0", // avoid prompting for credentials interactively, supported since git v2.3
+	}...)
 }
 
 // CommonCmdServEnvs is like CommonGitCmdEnvs but it only returns minimal required environment variables for the "gitea serv" command
 func CommonCmdServEnvs() []string {
-	return []string{
-		"GIT_NO_REPLACE_OBJECTS=1", // ignore replace references (https://git-scm.com/docs/git-replace)
-		"HOME=" + HomeDir(),        // make Gitea use internal git config only, to prevent conflicts with user's git config
-	}
+	return commonBaseEnvs()
 }
 
 // Run runs the command with the RunOpts

modules/git/git.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"strings"
@@ -19,7 +20,6 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-
 	"github.com/hashicorp/go-version"
 )
 
@@ -126,8 +126,8 @@ func VersionInfo() string {
 }
 
 func checkInit() error {
-	if setting.RepoRootPath == "" {
-		return errors.New("can not init Git's HomeDir (RepoRootPath is empty), the setting and git modules are not initialized correctly")
+	if setting.Git.HomePath == "" {
+		return errors.New("unable to init Git's HomeDir, incorrect initialization of the setting and git modules")
 	}
 	if DefaultContext != nil {
 		log.Warn("git module has been initialized already, duplicate init should be fixed")
@@ -137,14 +137,14 @@ func checkInit() error {
 
 // HomeDir is the home dir for git to store the global config file used by Gitea internally
 func HomeDir() string {
-	if setting.RepoRootPath == "" {
+	if setting.Git.HomePath == "" {
 		// strict check, make sure the git module is initialized correctly.
 		// attention: when the git module is called in gitea sub-command (serv/hook), the log module is not able to show messages to users.
 		// for example: if there is gitea git hook code calling git.NewCommand before git.InitXxx, the integration test won't show the real failure reasons.
-		log.Fatal("can not get Git's HomeDir (RepoRootPath is empty), the setting and git modules are not initialized correctly")
+		log.Fatal("Unable to init Git's HomeDir, incorrect initialization of the setting and git modules")
 		return ""
 	}
-	return setting.RepoRootPath
+	return setting.Git.HomePath
 }
 
 // InitSimple initializes git module with a very simple step, no config changes, no global command arguments.
@@ -175,11 +175,15 @@ func InitOnceWithSync(ctx context.Context) (err error) {
 	}
 
 	initOnce.Do(func() {
-		err = InitSimple(ctx)
-		if err != nil {
+		if err = InitSimple(ctx); err != nil {
 			return
 		}
 
+		// when git works with gnupg (commit signing), there should be a stable home for gnupg commands
+		if _, ok := os.LookupEnv("GNUPGHOME"); !ok {
+			_ = os.Setenv("GNUPGHOME", filepath.Join(HomeDir(), ".gnupg"))
+		}
+
 		// Since git wire protocol has been released from git v2.18
 		if setting.Git.EnableAutoGitWireProtocol && CheckGitVersionAtLeast("2.18") == nil {
 			globalCommandArgs = append(globalCommandArgs, "-c", "protocol.version=2")
@@ -206,7 +210,7 @@ func InitOnceWithSync(ctx context.Context) (err error) {
 // syncGitConfig only modifies gitconfig, won't change global variables (otherwise there will be data-race problem)
 func syncGitConfig() (err error) {
 	if err = os.MkdirAll(HomeDir(), os.ModePerm); err != nil {
-		return fmt.Errorf("unable to create directory %s, err: %w", setting.RepoRootPath, err)
+		return fmt.Errorf("unable to prepare git home directory %s, err: %w", HomeDir(), err)
 	}
 
 	// Git requires setting user.name and user.email in order to commit changes - old comment: "if they're not set just add some defaults"

modules/git/git_test.go

@@ -21,12 +21,12 @@ import (
 func testRun(m *testing.M) error {
 	_ = log.NewLogger(1000, "console", "console", `{"level":"trace","stacktracelevel":"NONE","stderr":true}`)
 
-	repoRootPath, err := os.MkdirTemp(os.TempDir(), "repos")
+	gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home")
 	if err != nil {
 		return fmt.Errorf("unable to create temp dir: %w", err)
 	}
-	defer util.RemoveAll(repoRootPath)
-	setting.RepoRootPath = repoRootPath
+	defer util.RemoveAll(gitHomePath)
+	setting.Git.HomePath = gitHomePath
 
 	if err = InitOnceWithSync(context.Background()); err != nil {
 		return fmt.Errorf("failed to call Init: %w", err)

modules/mirror/mirror.go

@@ -0,0 +1,69 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package mirror
+
+import (
+	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/queue"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var mirrorQueue queue.UniqueQueue
+
+// SyncType type of sync request
+type SyncType int
+
+const (
+	// PullMirrorType for pull mirrors
+	PullMirrorType SyncType = iota
+	// PushMirrorType for push mirrors
+	PushMirrorType
+)
+
+// SyncRequest for the mirror queue
+type SyncRequest struct {
+	Type        SyncType
+	ReferenceID int64 // RepoID for pull mirror, MirrorID for push mirror
+}
+
+// StartSyncMirrors starts a go routine to sync the mirrors
+func StartSyncMirrors(queueHandle func(data ...queue.Data) []queue.Data) {
+	if !setting.Mirror.Enabled {
+		return
+	}
+	mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle, new(SyncRequest))
+
+	go graceful.GetManager().RunWithShutdownFns(mirrorQueue.Run)
+}
+
+// AddPullMirrorToQueue adds repoID to mirror queue
+func AddPullMirrorToQueue(repoID int64) {
+	addMirrorToQueue(PullMirrorType, repoID)
+}
+
+// AddPushMirrorToQueue adds the push mirror to the queue
+func AddPushMirrorToQueue(mirrorID int64) {
+	addMirrorToQueue(PushMirrorType, mirrorID)
+}
+
+func addMirrorToQueue(syncType SyncType, referenceID int64) {
+	if !setting.Mirror.Enabled {
+		return
+	}
+	go func() {
+		if err := PushToQueue(syncType, referenceID); err != nil {
+			log.Error("Unable to push sync request for to the queue for pull mirror repo[%d]. Error: %v", referenceID, err)
+		}
+	}()
+}
+
+// PushToQueue adds the sync request to the queue
+func PushToQueue(mirrorType SyncType, referenceID int64) error {
+	return mirrorQueue.Push(&SyncRequest{
+		Type:        mirrorType,
+		ReferenceID: referenceID,
+	})
+}

modules/notification/mirror/mirror.go

@@ -0,0 +1,45 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package mirror
+
+import (
+	repo_model "code.gitea.io/gitea/models/repo"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/log"
+	mirror_module "code.gitea.io/gitea/modules/mirror"
+	"code.gitea.io/gitea/modules/notification/base"
+	"code.gitea.io/gitea/modules/repository"
+)
+
+type mirrorNotifier struct {
+	base.NullNotifier
+}
+
+var _ base.Notifier = &mirrorNotifier{}
+
+// NewNotifier create a new mirrorNotifier notifier
+func NewNotifier() base.Notifier {
+	return &mirrorNotifier{}
+}
+
+func (m *mirrorNotifier) NotifyPushCommits(_ *user_model.User, repo *repo_model.Repository, _ *repository.PushUpdateOptions, _ *repository.PushCommits) {
+	syncPushMirrorWithSyncOnCommit(repo.ID)
+}
+
+func (m *mirrorNotifier) NotifySyncPushCommits(_ *user_model.User, repo *repo_model.Repository, _ *repository.PushUpdateOptions, _ *repository.PushCommits) {
+	syncPushMirrorWithSyncOnCommit(repo.ID)
+}
+
+func syncPushMirrorWithSyncOnCommit(repoID int64) {
+	pushMirrors, err := repo_model.GetPushMirrorsSyncedOnCommit(repoID)
+	if err != nil {
+		log.Error("repo_model.GetPushMirrorsSyncedOnCommit failed: %v", err)
+		return
+	}
+
+	for _, mirror := range pushMirrors {
+		mirror_module.AddPushMirrorToQueue(mirror.ID)
+	}
+}

modules/notification/notification.go

@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/notification/base"
 	"code.gitea.io/gitea/modules/notification/indexer"
 	"code.gitea.io/gitea/modules/notification/mail"
+	"code.gitea.io/gitea/modules/notification/mirror"
 	"code.gitea.io/gitea/modules/notification/ui"
 	"code.gitea.io/gitea/modules/notification/webhook"
 	"code.gitea.io/gitea/modules/repository"
@@ -37,6 +38,7 @@ func NewContext() {
 	RegisterNotifier(indexer.NewNotifier())
 	RegisterNotifier(webhook.NewNotifier())
 	RegisterNotifier(action.NewNotifier())
+	RegisterNotifier(mirror.NewNotifier())
 }
 
 // NotifyCreateIssueComment notifies issue comment related message to notifiers