Skip to content

Commit

Permalink
Make mirror feature more configurable (go-gitea#16957)
Browse files Browse the repository at this point in the history
  • Loading branch information
6543 committed Sep 5, 2021
1 parent 741d01d commit d90b803
Show file tree
Hide file tree
Showing 17 changed files with 126 additions and 40 deletions.
9 changes: 6 additions & 3 deletions custom/conf/app.example.ini
Original file line number Diff line number Diff line change
Expand Up @@ -793,9 +793,6 @@ PATH =
;; Prefix archive files by placing them in a directory named after the repository
;PREFIX_ARCHIVE_FILES = true
;;
;; Disable the creation of new mirrors. Pre-existing mirrors remain valid.
;DISABLE_MIRRORS = false
;;
;; Disable migrating feature.
;DISABLE_MIGRATIONS = false
;;
Expand Down Expand Up @@ -1934,6 +1931,12 @@ PATH =
;[mirror]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Whether gitea's mirror function can be used or not.
;ENABLED = true
;; Disable the creation of **new** mirrors. Pre-existing mirrors remain valid.
;DISABLE_NEW_PULL = false
;; Disable the creation of **new** push mirrors. Pre-existing mirrors remain valid.
;DISABLE_NEW_PUSH = false
;; Default interval as a duration between each check
;DEFAULT_INTERVAL = 8h
;; Min interval as a duration must be > 1m
Expand Down
4 changes: 3 additions & 1 deletion docs/content/doc/advanced/config-cheat-sheet.en-us.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
- `DISABLED_REPO_UNITS`: **_empty_**: Comma separated list of globally disabled repo units. Allowed values: \[repo.issues, repo.ext_issues, repo.pulls, repo.wiki, repo.ext_wiki, repo.projects\]
- `DEFAULT_REPO_UNITS`: **repo.code,repo.releases,repo.issues,repo.pulls,repo.wiki,repo.projects**: Comma separated list of default repo units. Allowed values: \[repo.code, repo.releases, repo.issues, repo.pulls, repo.wiki, repo.projects\]. Note: Code and Releases can currently not be deactivated. If you specify default repo units you should still list them for future compatibility. External wiki and issue tracker can't be enabled by default as it requires additional settings. Disabled repo units will not be added to new repositories regardless if it is in the default list.
- `PREFIX_ARCHIVE_FILES`: **true**: Prefix archive files by placing them in a directory named after the repository.
- `DISABLE_MIRRORS`: **false**: Disable the creation of **new** mirrors. Pre-existing mirrors remain valid.
- `DISABLE_MIGRATIONS`: **false**: Disable migrating feature.
- `DISABLE_STARS`: **false**: Disable stars feature.
- `DEFAULT_BRANCH`: **master**: Default branch name of all repositories.
Expand Down Expand Up @@ -949,6 +948,9 @@ Task queue configuration has been moved to `queue.task`. However, the below conf

## Mirror (`mirror`)

- `ENABLED`: **true**: Whether gitea's mirror function can be used or not.
- `DISABLE_NEW_PULL`: **false**: Disable the creation of **new** mirrors. Pre-existing mirrors remain valid.
- `DISABLE_NEW_PUSH`: **false**: Disable the creation of **new** push mirrors. Pre-existing mirrors remain valid.
- `DEFAULT_INTERVAL`: **8h**: Default interval between each check
- `MIN_INTERVAL`: **10m**: Minimum interval for checking. (Must be >1m).

Expand Down
2 changes: 1 addition & 1 deletion integrations/api_settings_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func TestAPIExposedSettings(t *testing.T) {

DecodeJSON(t, resp, &repo)
assert.EqualValues(t, &api.GeneralRepoSettings{
MirrorsDisabled: setting.Repository.DisableMirrors,
MirrorsDisabled: !setting.Mirror.Enabled,
HTTPGitDisabled: setting.Repository.DisableHTTPGit,
MigrationsDisabled: setting.Repository.DisableMigrations,
TimeTrackingDisabled: false,
Expand Down
5 changes: 5 additions & 0 deletions models/repo_pushmirror.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ package models

import (
"errors"
"fmt"
"time"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"

"xorm.io/xorm"
Expand Down Expand Up @@ -57,6 +59,9 @@ func (m *PushMirror) GetRemoteName() string {

// InsertPushMirror inserts a push-mirror to database
func InsertPushMirror(m *PushMirror) error {
if setting.Mirror.DisableNewPush {
return fmt.Errorf("creation of new push mirror's are disabled")
}
_, err := x.Insert(m)
return err
}
Expand Down
53 changes: 53 additions & 0 deletions modules/setting/mirror.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package setting

import (
"time"

"code.gitea.io/gitea/modules/log"
)

var (
// Mirror settings
Mirror = struct {
Enabled bool
DisableNewPull bool
DisableNewPush bool
DefaultInterval time.Duration
MinInterval time.Duration
}{
Enabled: true,
DisableNewPull: false,
DisableNewPush: false,
MinInterval: 10 * time.Minute,
DefaultInterval: 8 * time.Hour,
}
)

func newMirror() {
if err := Cfg.Section("mirror").MapTo(&Mirror); err != nil {
log.Fatal("Failed to map Mirror settings: %v", err)
}

// fallback to old config repository.DISABLE_MIRRORS
if Cfg.Section("repository").Key("DISABLE_MIRRORS").MustBool(false) {
Mirror.DisableNewPull = true
}

if !Mirror.Enabled {
Mirror.DisableNewPull = true
Mirror.DisableNewPush = true
}

if Mirror.MinInterval.Minutes() < 1 {
log.Warn("Mirror.MinInterval is too low, set to 1 minute")
Mirror.MinInterval = 1 * time.Minute
}
if Mirror.DefaultInterval < Mirror.MinInterval {
log.Warn("Mirror.DefaultInterval is less than Mirror.MinInterval, set to 8 hours")
Mirror.DefaultInterval = time.Hour * 8
}
}
2 changes: 0 additions & 2 deletions modules/setting/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ var (
DisabledRepoUnits []string
DefaultRepoUnits []string
PrefixArchiveFiles bool
DisableMirrors bool
DisableMigrations bool
DisableStars bool `ini:"DISABLE_STARS"`
DefaultBranch string
Expand Down Expand Up @@ -154,7 +153,6 @@ var (
DisabledRepoUnits: []string{},
DefaultRepoUnits: []string{},
PrefixArchiveFiles: true,
DisableMirrors: false,
DisableMigrations: false,
DisableStars: false,
DefaultBranch: "master",
Expand Down
18 changes: 1 addition & 17 deletions modules/setting/setting.go
Original file line number Diff line number Diff line change
Expand Up @@ -347,12 +347,6 @@ var (

ManifestData string

// Mirror settings
Mirror struct {
DefaultInterval time.Duration
MinInterval time.Duration
}

// API settings
API = struct {
EnableSwagger bool
Expand Down Expand Up @@ -936,17 +930,7 @@ func NewContext() {

newGit()

sec = Cfg.Section("mirror")
Mirror.MinInterval = sec.Key("MIN_INTERVAL").MustDuration(10 * time.Minute)
Mirror.DefaultInterval = sec.Key("DEFAULT_INTERVAL").MustDuration(8 * time.Hour)
if Mirror.MinInterval.Minutes() < 1 {
log.Warn("Mirror.MinInterval is too low")
Mirror.MinInterval = 1 * time.Minute
}
if Mirror.DefaultInterval < Mirror.MinInterval {
log.Warn("Mirror.DefaultInterval is less than Mirror.MinInterval")
Mirror.DefaultInterval = time.Hour * 8
}
newMirror()

Langs = Cfg.Section("i18n").Key("LANGS").Strings(",")
if len(Langs) == 0 {
Expand Down
4 changes: 2 additions & 2 deletions routers/api/v1/repo/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ func Migrate(ctx *context.APIContext) {

gitServiceType := convert.ToGitServiceType(form.Service)

if form.Mirror && setting.Repository.DisableMirrors {
ctx.Error(http.StatusForbidden, "MirrorsGlobalDisabled", fmt.Errorf("the site administrator has disabled mirrors"))
if form.Mirror && setting.Mirror.DisableNewPull {
ctx.Error(http.StatusForbidden, "MirrorsGlobalDisabled", fmt.Errorf("the site administrator has disabled new mirror creations"))
return
}

Expand Down
6 changes: 6 additions & 0 deletions routers/api/v1/repo/mirror.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/setting"
mirror_service "code.gitea.io/gitea/services/mirror"
)

Expand Down Expand Up @@ -42,6 +43,11 @@ func MirrorSync(ctx *context.APIContext) {
ctx.Error(http.StatusForbidden, "MirrorSync", "Must have write access")
}

if !setting.Mirror.Enabled {
ctx.Status(http.StatusBadRequest)
return
}

mirror_service.StartToMirror(repo.ID)

ctx.Status(http.StatusOK)
Expand Down
2 changes: 1 addition & 1 deletion routers/api/v1/settings/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func GetGeneralRepoSettings(ctx *context.APIContext) {
// "200":
// "$ref": "#/responses/GeneralRepoSettings"
ctx.JSON(http.StatusOK, api.GeneralRepoSettings{
MirrorsDisabled: setting.Repository.DisableMirrors,
MirrorsDisabled: !setting.Mirror.Enabled,
HTTPGitDisabled: setting.Repository.DisableHTTPGit,
MigrationsDisabled: setting.Repository.DisableMigrations,
StarsDisabled: setting.Repository.DisableStars,
Expand Down
2 changes: 1 addition & 1 deletion routers/web/org/home.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ func Home(ctx *context.Context) {
ctx.Data["Members"] = members
ctx.Data["Teams"] = org.Teams

ctx.Data["DisabledMirrors"] = setting.Repository.DisableMirrors
ctx.Data["DisableNewPullMirrors"] = setting.Mirror.DisableNewPull

pager := context.NewPagination(int(count), setting.UI.User.RepoPagingNum, page, 5)
pager.SetDefaultParams(ctx)
Expand Down
15 changes: 9 additions & 6 deletions routers/web/repo/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,17 +152,20 @@ func MigratePost(ctx *context.Context) {
return
}

serviceType := structs.GitServiceType(form.Service)
if form.Mirror && setting.Mirror.DisableNewPull {
ctx.Error(http.StatusBadRequest, "MigratePost: the site administrator has disabled creation of new mirrors")
return
}

setMigrationContextData(ctx, serviceType)
setMigrationContextData(ctx, form.Service)

ctxUser := checkContextUser(ctx, form.UID)
if ctx.Written() {
return
}
ctx.Data["ContextUser"] = ctxUser

tpl := base.TplName("repo/migrate/" + serviceType.Name())
tpl := base.TplName("repo/migrate/" + form.Service.Name())

if ctx.HasError() {
ctx.HTML(http.StatusOK, tpl)
Expand Down Expand Up @@ -198,12 +201,12 @@ func MigratePost(ctx *context.Context) {

var opts = migrations.MigrateOptions{
OriginalURL: form.CloneAddr,
GitServiceType: serviceType,
GitServiceType: form.Service,
CloneAddr: remoteAddr,
RepoName: form.RepoName,
Description: form.Description,
Private: form.Private || setting.Repository.ForcePrivate,
Mirror: form.Mirror && !setting.Repository.DisableMirrors,
Mirror: form.Mirror,
LFS: form.LFS,
LFSEndpoint: form.LFSEndpoint,
AuthUsername: form.AuthUsername,
Expand Down Expand Up @@ -246,7 +249,7 @@ func setMigrationContextData(ctx *context.Context, serviceType structs.GitServic

ctx.Data["LFSActive"] = setting.LFS.StartServer
ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate
ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors
ctx.Data["DisableNewPullMirrors"] = setting.Mirror.DisableNewPull

// Plain git should be first
ctx.Data["Services"] = append([]structs.GitServiceType{structs.PlainGitService}, structs.SupportedFullGitService...)
Expand Down
23 changes: 20 additions & 3 deletions routers/web/repo/setting.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ func Settings(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("repo.settings")
ctx.Data["PageIsSettingsOptions"] = true
ctx.Data["ForcePrivate"] = setting.Repository.ForcePrivate
ctx.Data["DisabledMirrors"] = setting.Repository.DisableMirrors
ctx.Data["MirrorsEnabled"] = setting.Mirror.Enabled
ctx.Data["DisableNewPushMirrors"] = setting.Mirror.DisableNewPush
ctx.Data["DefaultMirrorInterval"] = setting.Mirror.DefaultInterval

signing, _ := models.SigningKey(ctx.Repo.Repository.RepoPath())
Expand Down Expand Up @@ -144,7 +145,7 @@ func SettingsPost(ctx *context.Context) {
ctx.Redirect(repo.Link() + "/settings")

case "mirror":
if !repo.IsMirror {
if !setting.Mirror.Enabled || !repo.IsMirror {
ctx.NotFound("", nil)
return
}
Expand Down Expand Up @@ -220,7 +221,7 @@ func SettingsPost(ctx *context.Context) {
ctx.Redirect(repo.Link() + "/settings")

case "mirror-sync":
if !repo.IsMirror {
if !setting.Mirror.Enabled || !repo.IsMirror {
ctx.NotFound("", nil)
return
}
Expand All @@ -231,6 +232,11 @@ func SettingsPost(ctx *context.Context) {
ctx.Redirect(repo.Link() + "/settings")

case "push-mirror-sync":
if !setting.Mirror.Enabled {
ctx.NotFound("", nil)
return
}

m, err := selectPushMirrorByForm(form, repo)
if err != nil {
ctx.NotFound("", nil)
Expand All @@ -243,6 +249,11 @@ func SettingsPost(ctx *context.Context) {
ctx.Redirect(repo.Link() + "/settings")

case "push-mirror-remove":
if !setting.Mirror.Enabled {
ctx.NotFound("", nil)
return
}

// This section doesn't require repo_name/RepoName to be set in the form, don't show it
// as an error on the UI for this action
ctx.Data["Err_RepoName"] = nil
Expand All @@ -267,6 +278,11 @@ func SettingsPost(ctx *context.Context) {
ctx.Redirect(repo.Link() + "/settings")

case "push-mirror-add":
if setting.Mirror.DisableNewPush {
ctx.NotFound("", nil)
return
}

// This section doesn't require repo_name/RepoName to be set in the form, don't show it
// as an error on the UI for this action
ctx.Data["Err_RepoName"] = nil
Expand Down Expand Up @@ -698,6 +714,7 @@ func SettingsPost(ctx *context.Context) {

log.Trace("Repository was archived: %s/%s", ctx.Repo.Owner.Name, repo.Name)
ctx.Redirect(ctx.Repo.RepoLink + "/settings")

case "unarchive":
if !ctx.Repo.IsOwner() {
ctx.Error(http.StatusForbidden)
Expand Down
13 changes: 13 additions & 0 deletions services/mirror/mirror.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ var mirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength)

// Update checks and updates mirror repositories.
func Update(ctx context.Context) error {
if !setting.Mirror.Enabled {
log.Warn("Mirror feature disabled, but cron job enabled: skip update")
return nil
}
log.Trace("Doing: Update")

handler := func(idx int, bean interface{}) error {
Expand Down Expand Up @@ -89,15 +93,24 @@ func syncMirrors(ctx context.Context) {

// InitSyncMirrors initializes a go routine to sync the mirrors
func InitSyncMirrors() {
if !setting.Mirror.Enabled {
return
}
go graceful.GetManager().RunWithShutdownContext(syncMirrors)
}

// StartToMirror adds repoID to mirror queue
func StartToMirror(repoID int64) {
if !setting.Mirror.Enabled {
return
}
go mirrorQueue.Add(fmt.Sprintf("pull %d", repoID))
}

// AddPushMirrorToQueue adds the push mirror to the queue
func AddPushMirrorToQueue(mirrorID int64) {
if !setting.Mirror.Enabled {
return
}
go mirrorQueue.Add(fmt.Sprintf("push %d", mirrorID))
}
2 changes: 1 addition & 1 deletion templates/org/home.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
<div class="ui eleven wide column">
{{if .CanCreateOrgRepo}}
<div class="text right">
{{if not .DisabledMirrors}}
{{if not .DisableNewPullMirrors}}
<a class="ui green button" href="{{AppSubUrl}}/repo/migrate?org={{.Org.ID}}&mirror=1">{{.i18n.Tr "new_migrate"}}</a>
{{end}}
<a class="ui green button" href="{{AppSubUrl}}/repo/create?org={{.Org.ID}}">{{.i18n.Tr "new_repo"}}</a>
Expand Down
2 changes: 1 addition & 1 deletion templates/repo/migrate/options.tmpl
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
<div class="ui checkbox">
{{if .DisableMirrors}}
{{if .DisableNewPullMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
<a target=”_blank” href="https://blog.codeberg.org/mirror-repos-easily-created-consuming-resources-forever.html">{{svg "octicon-question"}}</a>
Expand Down
Loading

0 comments on commit d90b803

Please sign in to comment.