Skip to content

Commit

Permalink
Add a new table issue_index to store the max issue index so that issu…
Browse files Browse the repository at this point in the history
…e could be deleted with no duplicated index (#15599)

* Add a new table issue_index to store the max issue index so that issue could be deleted with no duplicated index

* Fix pull index

* Add tests for concurrent creating issues

* Fix lint

* Fix tests

* Fix postgres test

* Add test for migration v180

* Rename wrong test file name

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: Lauris BH <lauris@nix.lv>
  • Loading branch information
3 people authored Jun 14, 2021
1 parent a005265 commit 0393a57
Show file tree
Hide file tree
Showing 14 changed files with 354 additions and 82 deletions.
4 changes: 2 additions & 2 deletions models/fixtures/issue.yml
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@
-
id: 13
repo_id: 50
index: 0
index: 1
poster_id: 2
name: issue in active repo
content: we'll be testing github issue 13171 with this.
Expand All @@ -164,7 +164,7 @@
-
id: 14
repo_id: 51
index: 0
index: 1
poster_id: 2
name: issue in archived repo
content: we'll be testing github issue 13171 with this.
Expand Down
24 changes: 24 additions & 0 deletions models/fixtures/issue_index.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
-
group_id: 1
max_index: 5
-
group_id: 2
max_index: 2
-
group_id: 3
max_index: 2
-
group_id: 10
max_index: 1
-
group_id: 48
max_index: 1
-
group_id: 42
max_index: 1
-
group_id: 50
max_index: 1
-
group_id: 51
max_index: 1
113 changes: 113 additions & 0 deletions models/index.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
"errors"
"fmt"

"code.gitea.io/gitea/modules/setting"
)

// ResourceIndex represents a resource index which could be used as issue/release and others
// We can create different tables i.e. issue_index, release_index and etc.
type ResourceIndex struct {
GroupID int64 `xorm:"unique"`
MaxIndex int64 `xorm:"index"`
}

// IssueIndex represents the issue index table
type IssueIndex ResourceIndex

// upsertResourceIndex the function will not return until it acquires the lock or receives an error.
func upsertResourceIndex(e Engine, tableName string, groupID int64) (err error) {
// An atomic UPSERT operation (INSERT/UPDATE) is the only operation
// that ensures that the key is actually locked.
switch {
case setting.Database.UseSQLite3 || setting.Database.UsePostgreSQL:
_, err = e.Exec(fmt.Sprintf("INSERT INTO %s (group_id, max_index) "+
"VALUES (?,1) ON CONFLICT (group_id) DO UPDATE SET max_index = %s.max_index+1",
tableName, tableName), groupID)
case setting.Database.UseMySQL:
_, err = e.Exec(fmt.Sprintf("INSERT INTO %s (group_id, max_index) "+
"VALUES (?,1) ON DUPLICATE KEY UPDATE max_index = max_index+1", tableName),
groupID)
case setting.Database.UseMSSQL:
// https://weblogs.sqlteam.com/dang/2009/01/31/upsert-race-condition-with-merge/
_, err = e.Exec(fmt.Sprintf("MERGE %s WITH (HOLDLOCK) as target "+
"USING (SELECT ? AS group_id) AS src "+
"ON src.group_id = target.group_id "+
"WHEN MATCHED THEN UPDATE SET target.max_index = target.max_index+1 "+
"WHEN NOT MATCHED THEN INSERT (group_id, max_index) "+
"VALUES (src.group_id, 1);", tableName),
groupID)
default:
return fmt.Errorf("database type not supported")
}
return
}

var (
// ErrResouceOutdated represents an error when request resource outdated
ErrResouceOutdated = errors.New("resource outdated")
// ErrGetResourceIndexFailed represents an error when resource index retries 3 times
ErrGetResourceIndexFailed = errors.New("get resource index failed")
)

const (
maxDupIndexAttempts = 3
)

// GetNextResourceIndex retried 3 times to generate a resource index
func GetNextResourceIndex(tableName string, groupID int64) (int64, error) {
for i := 0; i < maxDupIndexAttempts; i++ {
idx, err := getNextResourceIndex(tableName, groupID)
if err == ErrResouceOutdated {
continue
}
if err != nil {
return 0, err
}
return idx, nil
}
return 0, ErrGetResourceIndexFailed
}

// deleteResouceIndex delete resource index
func deleteResouceIndex(e Engine, tableName string, groupID int64) error {
_, err := e.Exec(fmt.Sprintf("DELETE FROM %s WHERE group_id=?", tableName), groupID)
return err
}

// getNextResourceIndex return the next index
func getNextResourceIndex(tableName string, groupID int64) (int64, error) {
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return 0, err
}
var preIdx int64
_, err := sess.SQL(fmt.Sprintf("SELECT max_index FROM %s WHERE group_id = ?", tableName), groupID).Get(&preIdx)
if err != nil {
return 0, err
}

if err := upsertResourceIndex(sess, tableName, groupID); err != nil {
return 0, err
}

var curIdx int64
has, err := sess.SQL(fmt.Sprintf("SELECT max_index FROM %s WHERE group_id = ? AND max_index=?", tableName), groupID, preIdx+1).Get(&curIdx)
if err != nil {
return 0, err
}
if !has {
return 0, ErrResouceOutdated
}
if err := sess.Commit(); err != nil {
return 0, err
}
return curIdx, nil
}
27 changes: 27 additions & 0 deletions models/index_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
"fmt"
"sync"
"testing"

"github.com/stretchr/testify/assert"
)

func TestResourceIndex(t *testing.T) {
assert.NoError(t, PrepareTestDatabase())

var wg sync.WaitGroup
for i := 0; i < 100; i++ {
wg.Add(1)
go func(i int) {
testInsertIssue(t, fmt.Sprintf("issue %d", i+1), "my issue", 0)
wg.Done()
}(i)
}
wg.Wait()
}
42 changes: 13 additions & 29 deletions models/issue.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,8 @@ var (
)

const (
issueTasksRegexpStr = `(^\s*[-*]\s\[[\sxX]\]\s.)|(\n\s*[-*]\s\[[\sxX]\]\s.)`
issueTasksDoneRegexpStr = `(^\s*[-*]\s\[[xX]\]\s.)|(\n\s*[-*]\s\[[xX]\]\s.)`
issueMaxDupIndexAttempts = 3
issueTasksRegexpStr = `(^\s*[-*]\s\[[\sxX]\]\s.)|(\n\s*[-*]\s\[[\sxX]\]\s.)`
issueTasksDoneRegexpStr = `(^\s*[-*]\s\[[xX]\]\s.)|(\n\s*[-*]\s\[[xX]\]\s.)`
)

func init() {
Expand Down Expand Up @@ -896,21 +895,17 @@ func newIssue(e *xorm.Session, doer *User, opts NewIssueOptions) (err error) {
}
}

// Milestone validation should happen before insert actual object.
if _, err := e.SetExpr("`index`", "coalesce(MAX(`index`),0)+1").
Where("repo_id=?", opts.Issue.RepoID).
Insert(opts.Issue); err != nil {
return ErrNewIssueInsert{err}
if opts.Issue.Index <= 0 {
return fmt.Errorf("no issue index provided")
}
if opts.Issue.ID > 0 {
return fmt.Errorf("issue exist")
}

inserted, err := getIssueByID(e, opts.Issue.ID)
if err != nil {
if _, err := e.Insert(opts.Issue); err != nil {
return err
}

// Patch Index with the value calculated by the database
opts.Issue.Index = inserted.Index

if opts.Issue.MilestoneID > 0 {
if _, err = e.Exec("UPDATE `milestone` SET num_issues=num_issues+1 WHERE id=?", opts.Issue.MilestoneID); err != nil {
return err
Expand Down Expand Up @@ -987,24 +982,13 @@ func newIssue(e *xorm.Session, doer *User, opts NewIssueOptions) (err error) {

// NewIssue creates new issue with labels for repository.
func NewIssue(repo *Repository, issue *Issue, labelIDs []int64, uuids []string) (err error) {
// Retry several times in case INSERT fails due to duplicate key for (repo_id, index); see #7887
i := 0
for {
if err = newIssueAttempt(repo, issue, labelIDs, uuids); err == nil {
return nil
}
if !IsErrNewIssueInsert(err) {
return err
}
if i++; i == issueMaxDupIndexAttempts {
break
}
log.Error("NewIssue: error attempting to insert the new issue; will retry. Original error: %v", err)
idx, err := GetNextResourceIndex("issue_index", repo.ID)
if err != nil {
return fmt.Errorf("generate issue index failed: %v", err)
}
return fmt.Errorf("NewIssue: too many errors attempting to insert the new issue. Last error was: %v", err)
}

func newIssueAttempt(repo *Repository, issue *Issue, labelIDs []int64, uuids []string) (err error) {
issue.Index = idx

sess := x.NewSession()
defer sess.Close()
if err = sess.Begin(); err != nil {
Expand Down
56 changes: 32 additions & 24 deletions models/issue_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -345,37 +345,45 @@ func TestGetRepoIDsForIssuesOptions(t *testing.T) {
}
}

func testInsertIssue(t *testing.T, title, content string) {
repo := AssertExistsAndLoadBean(t, &Repository{ID: 1}).(*Repository)
user := AssertExistsAndLoadBean(t, &User{ID: 2}).(*User)

issue := Issue{
RepoID: repo.ID,
PosterID: user.ID,
Title: title,
Content: content,
}
err := NewIssue(repo, &issue, nil, nil)
assert.NoError(t, err)

func testInsertIssue(t *testing.T, title, content string, expectIndex int64) *Issue {
var newIssue Issue
has, err := x.ID(issue.ID).Get(&newIssue)
assert.NoError(t, err)
assert.True(t, has)
assert.EqualValues(t, issue.Title, newIssue.Title)
assert.EqualValues(t, issue.Content, newIssue.Content)
// there are 5 issues and max index is 5 on repository 1, so this one should 6
assert.EqualValues(t, 6, newIssue.Index)
t.Run(title, func(t *testing.T) {
repo := AssertExistsAndLoadBean(t, &Repository{ID: 1}).(*Repository)
user := AssertExistsAndLoadBean(t, &User{ID: 2}).(*User)

issue := Issue{
RepoID: repo.ID,
PosterID: user.ID,
Title: title,
Content: content,
}
err := NewIssue(repo, &issue, nil, nil)
assert.NoError(t, err)

_, err = x.ID(issue.ID).Delete(new(Issue))
assert.NoError(t, err)
has, err := x.ID(issue.ID).Get(&newIssue)
assert.NoError(t, err)
assert.True(t, has)
assert.EqualValues(t, issue.Title, newIssue.Title)
assert.EqualValues(t, issue.Content, newIssue.Content)
if expectIndex > 0 {
assert.EqualValues(t, expectIndex, newIssue.Index)
}
})
return &newIssue
}

func TestIssue_InsertIssue(t *testing.T) {
assert.NoError(t, PrepareTestDatabase())

testInsertIssue(t, "my issue1", "special issue's comments?")
testInsertIssue(t, `my issue2, this is my son's love \n \r \ `, "special issue's '' comments?")
// there are 5 issues and max index is 5 on repository 1, so this one should 6
issue := testInsertIssue(t, "my issue1", "special issue's comments?", 6)
_, err := x.ID(issue.ID).Delete(new(Issue))
assert.NoError(t, err)

issue = testInsertIssue(t, `my issue2, this is my son's love \n \r \ `, "special issue's '' comments?", 7)
_, err = x.ID(issue.ID).Delete(new(Issue))
assert.NoError(t, err)

}

func TestIssue_ResolveMentions(t *testing.T) {
Expand Down
19 changes: 17 additions & 2 deletions models/issue_xref_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,12 +125,27 @@ func TestXRef_ResolveCrossReferences(t *testing.T) {
func testCreateIssue(t *testing.T, repo, doer int64, title, content string, ispull bool) *Issue {
r := AssertExistsAndLoadBean(t, &Repository{ID: repo}).(*Repository)
d := AssertExistsAndLoadBean(t, &User{ID: doer}).(*User)
i := &Issue{RepoID: r.ID, PosterID: d.ID, Poster: d, Title: title, Content: content, IsPull: ispull}

idx, err := GetNextResourceIndex("issue_index", r.ID)
assert.NoError(t, err)
i := &Issue{
RepoID: r.ID,
PosterID: d.ID,
Poster: d,
Title: title,
Content: content,
IsPull: ispull,
Index: idx,
}

sess := x.NewSession()
defer sess.Close()

assert.NoError(t, sess.Begin())
_, err := sess.SetExpr("`index`", "coalesce(MAX(`index`),0)+1").Where("repo_id=?", repo).Insert(i)
err = newIssue(sess, d, NewIssueOptions{
Repo: r,
Issue: i,
})
assert.NoError(t, err)
i, err = getIssueByID(sess, i.ID)
assert.NoError(t, err)
Expand Down
2 changes: 2 additions & 0 deletions models/migrations/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,8 @@ var migrations = []Migration{
NewMigration("Delete credentials from past migrations", deleteMigrationCredentials),
// v181 -> v182
NewMigration("Always save primary email on email address table", addPrimaryEmail2EmailAddress),
// v182 -> v183
NewMigration("Add issue resource index table", addIssueResourceIndexTable),
}

// GetCurrentDBVersion returns the current db version
Expand Down
Loading

0 comments on commit 0393a57

Please sign in to comment.