Skip to content

Commit

Permalink
Prevent hang in git cat-file if repository is not a valid repository …
Browse files Browse the repository at this point in the history
…and other fixes (#17991)

This PR contains multiple fixes. The most important of which is:

* Prevent hang in git cat-file if the repository is not a valid repository 
    
    Unfortunately it appears that if git cat-file is run in an invalid
    repository it will hang until stdin is closed. This will result in
    deadlocked /pulls pages and dangling git cat-file calls if a broken
    repository is tried to be reviewed or pulls exists for a broken
    repository.

    Fix #14734
    Fix #9271
    Fix #16113

Otherwise there are a few small other fixes included which this PR was initially intending to fix:

* Fix panic on partial compares due to missing PullRequestWorkInProgressPrefixes
* Fix links on pulls pages  due to regression from #17551 - by making most /issues routes match /pulls too - Fix #17983
* Fix links on feeds pages due to another regression from #17551 but also fix issue with syncing tags - Fix #17943
* Add missing locale entries for oauth group claims
* Prevent NPEs if ColorFormat is called on nil users, repos or teams.
  • Loading branch information
zeripath authored Dec 16, 2021
1 parent 6e7d28c commit 8354670
Show file tree
Hide file tree
Showing 18 changed files with 209 additions and 26 deletions.
38 changes: 38 additions & 0 deletions integrations/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,25 @@ func prepareTestEnv(t testing.TB, skip ...int) func() {
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))

assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, ownerDir := range ownerDirs {
if !ownerDir.Type().IsDir() {
continue
}
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, repoDir := range repoDirs {
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0755)
}
}

return deferFn
}
Expand Down Expand Up @@ -532,4 +551,23 @@ func resetFixtures(t *testing.T) {
assert.NoError(t, unittest.LoadFixtures())
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, ownerDir := range ownerDirs {
if !ownerDir.Type().IsDir() {
continue
}
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, repoDir := range repoDirs {
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0755)
}
}
}
19 changes: 19 additions & 0 deletions integrations/migration-test/migration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,25 @@ func initMigrationTest(t *testing.T) func() {
assert.True(t, len(setting.RepoRootPath) != 0)
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, ownerDir := range ownerDirs {
if !ownerDir.Type().IsDir() {
continue
}
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, repoDir := range repoDirs {
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0755)
}
}

git.CheckLFSVersion()
setting.InitDBConfig()
Expand Down
16 changes: 16 additions & 0 deletions models/action.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"

"xorm.io/builder"
)
Expand Down Expand Up @@ -252,6 +253,21 @@ func (a *Action) GetBranch() string {
return strings.TrimPrefix(a.RefName, git.BranchPrefix)
}

// GetRefLink returns the action's ref link.
func (a *Action) GetRefLink() string {
switch {
case strings.HasPrefix(a.RefName, git.BranchPrefix):
return a.GetRepoLink() + "/src/branch/" + util.PathEscapeSegments(strings.TrimPrefix(a.RefName, git.BranchPrefix))
case strings.HasPrefix(a.RefName, git.TagPrefix):
return a.GetRepoLink() + "/src/tag/" + util.PathEscapeSegments(strings.TrimPrefix(a.RefName, git.TagPrefix))
case len(a.RefName) == 40 && git.SHAPattern.MatchString(a.RefName):
return a.GetRepoLink() + "/src/commit/" + a.RefName
default:
// FIXME: we will just assume it's a branch - this was the old way - at some point we may want to enforce that there is always a ref here.
return a.GetRepoLink() + "/src/branch/" + util.PathEscapeSegments(strings.TrimPrefix(a.RefName, git.BranchPrefix))
}
}

// GetTag returns the action's repository tag.
func (a *Action) GetTag() string {
return strings.TrimPrefix(a.RefName, git.TagPrefix)
Expand Down
19 changes: 19 additions & 0 deletions models/migrations/migrations_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,25 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En

assert.NoError(t, com.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"),
setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, ownerDir := range ownerDirs {
if !ownerDir.Type().IsDir() {
continue
}
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, repoDir := range repoDirs {
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0755)
}
}

if err := deleteDB(); err != nil {
t.Errorf("unable to reset database: %v", err)
Expand Down
8 changes: 8 additions & 0 deletions models/org_team.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,14 @@ func SearchTeam(opts *SearchTeamOptions) ([]*Team, int64, error) {

// ColorFormat provides a basic color format for a Team
func (t *Team) ColorFormat(s fmt.State) {
if t == nil {
log.ColorFprintf(s, "%d:%s (OrgID: %d) %-v",
log.NewColoredIDValue(0),
"<nil>",
log.NewColoredIDValue(0),
0)
return
}
log.ColorFprintf(s, "%d:%s (OrgID: %d) %-v",
log.NewColoredIDValue(t.ID),
t.Name,
Expand Down
7 changes: 7 additions & 0 deletions models/repo/repo.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,13 @@ func (repo *Repository) SanitizedOriginalURL() string {

// ColorFormat returns a colored string to represent this repo
func (repo *Repository) ColorFormat(s fmt.State) {
if repo == nil {
log.ColorFprintf(s, "%d:%s/%s",
log.NewColoredIDValue(0),
"<nil>",
"<nil>")
return
}
log.ColorFprintf(s, "%d:%s/%s",
log.NewColoredIDValue(repo.ID),
repo.OwnerName,
Expand Down
37 changes: 37 additions & 0 deletions models/unittest/testdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,26 @@ func MainTest(m *testing.M, pathToGiteaRoot string, fixtureFiles ...string) {
fatalTestError("util.CopyDir: %v\n", err)
}

ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {
fatalTestError("unable to read the new repo root: %v\n", err)
}
for _, ownerDir := range ownerDirs {
if !ownerDir.Type().IsDir() {
continue
}
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
if err != nil {
fatalTestError("unable to read the new repo root: %v\n", err)
}
for _, repoDir := range repoDirs {
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0755)
}
}

exitStatus := m.Run()
if err = util.RemoveAll(repoRootPath); err != nil {
fatalTestError("util.RemoveAll: %v\n", err)
Expand Down Expand Up @@ -152,5 +172,22 @@ func PrepareTestEnv(t testing.TB) {
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
metaPath := filepath.Join(giteaRoot, "integrations", "gitea-repositories-meta")
assert.NoError(t, util.CopyDir(metaPath, setting.RepoRootPath))

ownerDirs, err := os.ReadDir(setting.RepoRootPath)
assert.NoError(t, err)
for _, ownerDir := range ownerDirs {
if !ownerDir.Type().IsDir() {
continue
}
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
assert.NoError(t, err)
for _, repoDir := range repoDirs {
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0755)
}
}

base.SetupGiteaRoot() // Makes sure GITEA_ROOT is set
}
6 changes: 6 additions & 0 deletions models/user/user.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,12 @@ type SearchOrganizationsOptions struct {

// ColorFormat writes a colored string to identify this struct
func (u *User) ColorFormat(s fmt.State) {
if u == nil {
log.ColorFprintf(s, "%d:%s",
log.NewColoredIDValue(0),
log.NewColoredValue("<nil>"))
return
}
log.ColorFprintf(s, "%d:%s",
log.NewColoredIDValue(u.ID),
log.NewColoredValue(u.Name))
Expand Down
14 changes: 14 additions & 0 deletions modules/git/batch_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,20 @@ type WriteCloserError interface {
CloseWithError(err error) error
}

// EnsureValidGitRepository runs git rev-parse in the repository path - thus ensuring that the repository is a valid repository.
// Run before opening git cat-file.
// This is needed otherwise the git cat-file will hang for invalid repositories.
func EnsureValidGitRepository(ctx context.Context, repoPath string) error {
stderr := strings.Builder{}
err := NewCommandContext(ctx, "rev-parse").
SetDescription(fmt.Sprintf("%s rev-parse [repo_path: %s]", GitExecutable, repoPath)).
RunInDirFullPipeline(repoPath, nil, &stderr, nil)
if err != nil {
return ConcatenateError(err, (&stderr).String())
}
return nil
}

// CatFileBatchCheck opens git cat-file --batch-check in the provided repo and returns a stdin pipe, a stdout reader and cancel function
func CatFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) {
batchStdinReader, batchStdinWriter := io.Pipe()
Expand Down
5 changes: 5 additions & 0 deletions modules/git/repo_base_nogogit.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ func OpenRepositoryCtx(ctx context.Context, repoPath string) (*Repository, error
return nil, errors.New("no such file or directory")
}

// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
if err := EnsureValidGitRepository(ctx, repoPath); err != nil {
return nil, err
}

repo := &Repository{
Path: repoPath,
tagCache: newObjectCache(),
Expand Down
5 changes: 4 additions & 1 deletion modules/git/repo_commit_nogogit.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ func (repo *Repository) ResolveReference(name string) (string, error) {
func (repo *Repository) GetRefCommitID(name string) (string, error) {
wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx)
defer cancel()
_, _ = wr.Write([]byte(name + "\n"))
_, err := wr.Write([]byte(name + "\n"))
if err != nil {
return "", err
}
shaBs, _, _, err := ReadBatchLine(rd)
if IsErrNotExist(err) {
return "", ErrNotExist{name, ""}
Expand Down
6 changes: 6 additions & 0 deletions modules/indexer/code/bleve.go
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,12 @@ func (b *BleveIndexer) Index(repo *repo_model.Repository, sha string, changes *r
batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
if len(changes.Updates) > 0 {

// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
if err := git.EnsureValidGitRepository(git.DefaultContext, repo.RepoPath()); err != nil {
log.Error("Unable to open git repo: %s for %-v: %v", repo.RepoPath(), repo, err)
return err
}

batchWriter, batchReader, cancel := git.CatFileBatch(git.DefaultContext, repo.RepoPath())
defer cancel()

Expand Down
5 changes: 5 additions & 0 deletions modules/indexer/code/elastic_search.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,11 @@ func (b *ElasticSearchIndexer) addDelete(filename string, repo *repo_model.Repos
func (b *ElasticSearchIndexer) Index(repo *repo_model.Repository, sha string, changes *repoChanges) error {
reqs := make([]elastic.BulkableRequest, 0)
if len(changes.Updates) > 0 {
// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
if err := git.EnsureValidGitRepository(git.DefaultContext, repo.RepoPath()); err != nil {
log.Error("Unable to open git repo: %s for %-v: %v", repo.RepoPath(), repo, err)
return err
}

batchWriter, batchReader, cancel := git.CatFileBatch(git.DefaultContext, repo.RepoPath())
defer cancel()
Expand Down
3 changes: 3 additions & 0 deletions options/locale/locale_en-US.ini
Original file line number Diff line number Diff line change
Expand Up @@ -2532,6 +2532,9 @@ auths.oauth2_required_claim_name = Required Claim Name
auths.oauth2_required_claim_name_helper = Set this name to restrict login from this source to users with a claim with this name
auths.oauth2_required_claim_value = Required Claim Value
auths.oauth2_required_claim_value_helper = Set this value to restrict login from this source to users with a claim with this name and value
auths.oauth2_group_claim_name = Claim name providing group names for this source. (Optional)
auths.oauth2_admin_group = Group Claim value for administrator users. (Optional - requires claim name above)
auths.oauth2_restricted_group = Group Claim value for restricted users. (Optional - requires claim name above)
auths.enable_auto_register = Enable Auto Registration
auths.sspi_auto_create_users = Automatically create users
auths.sspi_auto_create_users_helper = Allow SSPI auth method to automatically create new accounts for users that login for the first time
Expand Down
2 changes: 1 addition & 1 deletion routers/web/repo/compare.go
Original file line number Diff line number Diff line change
Expand Up @@ -685,6 +685,7 @@ func CompareDiff(ctx *context.Context) {
return
}

ctx.Data["PullRequestWorkInProgressPrefixes"] = setting.Repository.PullRequest.WorkInProgressPrefixes
ctx.Data["DirectComparison"] = ci.DirectComparison
ctx.Data["OtherCompareSeparator"] = ".."
ctx.Data["CompareSeparator"] = "..."
Expand Down Expand Up @@ -762,7 +763,6 @@ func CompareDiff(ctx *context.Context) {
ctx.Data["IsDiffCompare"] = true
ctx.Data["RequireTribute"] = true
ctx.Data["RequireEasyMDE"] = true
ctx.Data["PullRequestWorkInProgressPrefixes"] = setting.Repository.PullRequest.WorkInProgressPrefixes
setTemplateIfExists(ctx, pullRequestTemplateKey, nil, pullRequestTemplateCandidates)
ctx.Data["IsAttachmentEnabled"] = setting.Attachment.Enabled
upload.AddUploadContext(ctx, "comment")
Expand Down
2 changes: 1 addition & 1 deletion routers/web/web.go
Original file line number Diff line number Diff line change
Expand Up @@ -718,7 +718,7 @@ func RegisterRoutes(m *web.Route) {
}, context.RepoMustNotBeArchived(), reqRepoIssueReader)
// FIXME: should use different URLs but mostly same logic for comments of issue and pull request.
// So they can apply their own enable/disable logic on routers.
m.Group("/issues", func() {
m.Group("/{type:issues|pulls}", func() {
m.Group("/{index}", func() {
m.Post("/title", repo.UpdateIssueTitle)
m.Post("/content", repo.UpdateIssueContent)
Expand Down
3 changes: 2 additions & 1 deletion services/pull/pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -746,7 +746,8 @@ func GetIssuesLastCommitStatus(issues models.IssueList) (map[int64]*models.Commi
if !ok {
gitRepo, err = git.OpenRepository(issue.Repo.RepoPath())
if err != nil {
return nil, err
log.Error("Cannot open git repository %-v for issue #%d[%d]. Error: %v", issue.Repo, issue.Index, issue.ID, err)
continue
}
gitRepos[issue.RepoID] = gitRepo
}
Expand Down
Loading

0 comments on commit 8354670

Please sign in to comment.