diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 5af41f9a21b73..a29064f9cd3a3 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -313,6 +313,7 @@ USER = root ;DB_TYPE = sqlite3 ;PATH= ; defaults to data/gitea.db ;SQLITE_TIMEOUT = ; Query timeout defaults to: 500 +;SQLITE_JOURNAL_MODE = ; defaults to sqlite database default (often DELETE), can be used to enable WAL mode. https://www.sqlite.org/pragma.html#pragma_journal_mode ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; @@ -878,6 +879,9 @@ ROUTER = console ;; Allow deletion of unadopted repositories ;ALLOW_DELETION_OF_UNADOPTED_REPOSITORIES = false +;; Don't allow download source archive files from UI +;DISABLE_DOWNLOAD_SOURCE_ARCHIVES = false + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;[repository.editor] diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index dae7fff5d4983..2f37318e20dd7 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -78,6 +78,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`. - `DEFAULT_BRANCH`: **main**: Default branch name of all repositories. - `ALLOW_ADOPTION_OF_UNADOPTED_REPOSITORIES`: **false**: Allow non-admin users to adopt unadopted repositories - `ALLOW_DELETION_OF_UNADOPTED_REPOSITORIES`: **false**: Allow non-admin users to delete unadopted repositories +- `DISABLE_DOWNLOAD_SOURCE_ARCHIVES`: **false**: Don't allow download source archive files from UI ### Repository - Editor (`repository.editor`) @@ -382,6 +383,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a - `verify-ca`: Enable TLS with verification of the database server certificate against its root certificate. - `verify-full`: Enable TLS and verify the database server name matches the given certificate in either the `Common Name` or `Subject Alternative Name` fields. - `SQLITE_TIMEOUT`: **500**: Query timeout for SQLite3 only. +- `SQLITE_JOURNAL_MODE`: **""**: Change journal mode for SQlite3. Can be used to enable [WAL mode](https://www.sqlite.org/wal.html) when high load causes write congestion. See [SQlite3 docs](https://www.sqlite.org/pragma.html#pragma_journal_mode) for possible values. Defaults to the default for the database file, often DELETE. - `ITERATE_BUFFER_SIZE`: **50**: Internal buffer size for iterating. - `CHARSET`: **utf8mb4**: For MySQL only, either "utf8" or "utf8mb4". NOTICE: for "utf8mb4" you must use MySQL InnoDB > 5.6. Gitea is unable to check this. - `PATH`: **data/gitea.db**: For SQLite3 only, the database file path. diff --git a/docs/content/doc/packages/composer.en-us.md b/docs/content/doc/packages/composer.en-us.md index 2502ee45b509b..47b03781f06e5 100644 --- a/docs/content/doc/packages/composer.en-us.md +++ b/docs/content/doc/packages/composer.en-us.md @@ -60,6 +60,8 @@ curl --user your_username:your_password_or_token \ https://gitea.example.com/api/packages/testuser/composer?version=1.0.3 ``` +If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. + The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | diff --git a/docs/content/doc/packages/conan.en-us.md b/docs/content/doc/packages/conan.en-us.md index c650e9d7ea491..fb104f34f43b2 100644 --- a/docs/content/doc/packages/conan.en-us.md +++ b/docs/content/doc/packages/conan.en-us.md @@ -37,7 +37,7 @@ conan user --remote {remote} --password {password} {username} | -----------| ----------- | | `remote` | The remote name. | | `username` | Your Gitea username. | -| `password` | Your Gitea password or a personal access token. | +| `password` | Your Gitea password. If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. | | `owner` | The owner of the package. | For example: diff --git a/docs/content/doc/packages/container.en-us.md b/docs/content/doc/packages/container.en-us.md index 28559eb22b8d5..77dbbafd02cb5 100644 --- a/docs/content/doc/packages/container.en-us.md +++ b/docs/content/doc/packages/container.en-us.md @@ -34,6 +34,8 @@ To push an image or if the image is in a private registry, you have to authentic docker login gitea.example.com ``` +If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. + ## Image naming convention Images must follow this naming convention: diff --git a/docs/content/doc/packages/generic.en-us.md b/docs/content/doc/packages/generic.en-us.md index 9d4a2dd82d8a6..a82058da8ac5c 100644 --- a/docs/content/doc/packages/generic.en-us.md +++ b/docs/content/doc/packages/generic.en-us.md @@ -48,6 +48,8 @@ curl --user your_username:your_password_or_token \ https://gitea.example.com/api/packages/testuser/generic/test_package/1.0.0/file.bin ``` +If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. + The server reponds with the following HTTP Status codes. | HTTP Status Code | Meaning | diff --git a/docs/content/doc/packages/helm.en-us.md b/docs/content/doc/packages/helm.en-us.md index 9c43b08bf4298..89b929f9bc848 100644 --- a/docs/content/doc/packages/helm.en-us.md +++ b/docs/content/doc/packages/helm.en-us.md @@ -42,7 +42,7 @@ helm cm-push ./{chart_file}.tgz {repo} | Parameter | Description | | ------------ | ----------- | | `username` | Your Gitea username. | -| `password` | Your Gitea password or a personal access token. | +| `password` | Your Gitea password. If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. | | `repo` | The name for the repository. | | `chart_file` | The Helm Chart archive. | | `owner` | The owner of the package. | diff --git a/docs/content/doc/packages/nuget.en-us.md b/docs/content/doc/packages/nuget.en-us.md index 421faf9ee6503..a4435fa99f01a 100644 --- a/docs/content/doc/packages/nuget.en-us.md +++ b/docs/content/doc/packages/nuget.en-us.md @@ -38,7 +38,7 @@ dotnet nuget add source --name {source_name} --username {username} --password {p | ------------- | ----------- | | `source_name` | The desired source name. | | `username` | Your Gitea username. | -| `password` | Your Gitea password or a personal access token. | +| `password` | Your Gitea password. If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. | | `owner` | The owner of the package. | For example: diff --git a/docs/content/doc/packages/pypi.en-us.md b/docs/content/doc/packages/pypi.en-us.md index 1482575072378..af17fe83101c9 100644 --- a/docs/content/doc/packages/pypi.en-us.md +++ b/docs/content/doc/packages/pypi.en-us.md @@ -42,7 +42,7 @@ password = {password} | ------------ | ----------- | | `owner` | The owner of the package. | | `username` | Your Gitea username. | -| `password` | Your Gitea password or a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). | +| `password` | Your Gitea password. If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. | ## Publish a package diff --git a/docs/content/doc/packages/rubygems.en-us.md b/docs/content/doc/packages/rubygems.en-us.md index 98b3feafcd3d8..dd7ac9ee7ed0d 100644 --- a/docs/content/doc/packages/rubygems.en-us.md +++ b/docs/content/doc/packages/rubygems.en-us.md @@ -36,7 +36,7 @@ https://gitea.example.com/api/packages/{owner}/rubygems: Bearer {token} | Parameter | Description | | ------------- | ----------- | | `owner` | The owner of the package. | -| `token` | Your personal access token. | +| `token` | Your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). | For example: diff --git a/integrations/mirror_push_test.go b/integrations/mirror_push_test.go index a73b69e7869dd..3a22b0075423b 100644 --- a/integrations/mirror_push_test.go +++ b/integrations/mirror_push_test.go @@ -13,6 +13,7 @@ import ( "testing" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" @@ -47,7 +48,7 @@ func testMirrorPush(t *testing.T, u *url.URL) { doCreatePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword)(t) - mirrors, err := repo_model.GetPushMirrorsByRepoID(srcRepo.ID) + mirrors, _, err := repo_model.GetPushMirrorsByRepoID(db.DefaultContext, srcRepo.ID, db.ListOptions{}) assert.NoError(t, err) assert.Len(t, mirrors, 1) @@ -72,7 +73,7 @@ func testMirrorPush(t *testing.T, u *url.URL) { // Cleanup doRemovePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword, int(mirrors[0].ID))(t) - mirrors, err = repo_model.GetPushMirrorsByRepoID(srcRepo.ID) + mirrors, _, err = repo_model.GetPushMirrorsByRepoID(db.DefaultContext, srcRepo.ID, db.ListOptions{}) assert.NoError(t, err) assert.Len(t, mirrors, 0) } diff --git a/models/auth/webauthn.go b/models/auth/webauthn.go index 2dc3043780101..d3062342f545b 100644 --- a/models/auth/webauthn.go +++ b/models/auth/webauthn.go @@ -6,7 +6,6 @@ package auth import ( "context" - "encoding/base32" "fmt" "strings" @@ -20,14 +19,14 @@ import ( // ErrWebAuthnCredentialNotExist represents a "ErrWebAuthnCRedentialNotExist" kind of error. type ErrWebAuthnCredentialNotExist struct { ID int64 - CredentialID string + CredentialID []byte } func (err ErrWebAuthnCredentialNotExist) Error() string { - if err.CredentialID == "" { + if len(err.CredentialID) == 0 { return fmt.Sprintf("WebAuthn credential does not exist [id: %d]", err.ID) } - return fmt.Sprintf("WebAuthn credential does not exist [credential_id: %s]", err.CredentialID) + return fmt.Sprintf("WebAuthn credential does not exist [credential_id: %x]", err.CredentialID) } // IsErrWebAuthnCredentialNotExist checks if an error is a ErrWebAuthnCredentialNotExist. @@ -43,7 +42,7 @@ type WebAuthnCredential struct { Name string LowerName string `xorm:"unique(s)"` UserID int64 `xorm:"INDEX unique(s)"` - CredentialID string `xorm:"INDEX VARCHAR(410)"` + CredentialID []byte `xorm:"INDEX VARBINARY(1024)"` PublicKey []byte AttestationType string AAGUID []byte @@ -94,9 +93,8 @@ type WebAuthnCredentialList []*WebAuthnCredential func (list WebAuthnCredentialList) ToCredentials() []webauthn.Credential { creds := make([]webauthn.Credential, 0, len(list)) for _, cred := range list { - credID, _ := base32.HexEncoding.DecodeString(cred.CredentialID) creds = append(creds, webauthn.Credential{ - ID: credID, + ID: cred.CredentialID, PublicKey: cred.PublicKey, AttestationType: cred.AttestationType, Authenticator: webauthn.Authenticator{ @@ -164,11 +162,11 @@ func HasWebAuthnRegistrationsByUID(uid int64) (bool, error) { } // GetWebAuthnCredentialByCredID returns WebAuthn credential by credential ID -func GetWebAuthnCredentialByCredID(userID int64, credID string) (*WebAuthnCredential, error) { +func GetWebAuthnCredentialByCredID(userID int64, credID []byte) (*WebAuthnCredential, error) { return getWebAuthnCredentialByCredID(db.DefaultContext, userID, credID) } -func getWebAuthnCredentialByCredID(ctx context.Context, userID int64, credID string) (*WebAuthnCredential, error) { +func getWebAuthnCredentialByCredID(ctx context.Context, userID int64, credID []byte) (*WebAuthnCredential, error) { cred := new(WebAuthnCredential) if found, err := db.GetEngine(ctx).Where("user_id = ? AND credential_id = ?", userID, credID).Get(cred); err != nil { return nil, err @@ -187,7 +185,7 @@ func createCredential(ctx context.Context, userID int64, name string, cred *weba c := &WebAuthnCredential{ UserID: userID, Name: name, - CredentialID: base32.HexEncoding.EncodeToString(cred.ID), + CredentialID: cred.ID, PublicKey: cred.PublicKey, AttestationType: cred.AttestationType, AAGUID: cred.Authenticator.AAGUID, diff --git a/models/auth/webauthn_test.go b/models/auth/webauthn_test.go index 216bf110806eb..cc39691ce2d36 100644 --- a/models/auth/webauthn_test.go +++ b/models/auth/webauthn_test.go @@ -5,7 +5,6 @@ package auth import ( - "encoding/base32" "testing" "code.gitea.io/gitea/models/unittest" @@ -61,9 +60,7 @@ func TestCreateCredential(t *testing.T) { res, err := CreateCredential(1, "WebAuthn Created Credential", &webauthn.Credential{ID: []byte("Test")}) assert.NoError(t, err) assert.Equal(t, "WebAuthn Created Credential", res.Name) - bs, err := base32.HexEncoding.DecodeString(res.CredentialID) - assert.NoError(t, err) - assert.Equal(t, []byte("Test"), bs) + assert.Equal(t, []byte("Test"), res.CredentialID) unittest.AssertExistsIf(t, true, &WebAuthnCredential{Name: "WebAuthn Created Credential", UserID: 1}) } diff --git a/models/migrations/fixtures/Test_storeWebauthnCredentialIDAsBytes/expected_webauthn_credential.yml b/models/migrations/fixtures/Test_storeWebauthnCredentialIDAsBytes/expected_webauthn_credential.yml new file mode 100644 index 0000000000000..55a237a0d633d --- /dev/null +++ b/models/migrations/fixtures/Test_storeWebauthnCredentialIDAsBytes/expected_webauthn_credential.yml @@ -0,0 +1,9 @@ +- + id: 1 + credential_id: "TVHE44TOH7DF7V48SEAIT3EMMJ7TGBOQ289E5AQB34S98LFCUFJ7U2NAVI8RJG6K2F4TC8AQ8KBNO7AGEOQOL9NE43GR63HTEHJSLOG=" +- + id: 2 + credential_id: "051CLMMKB62S6M9M2A4H54K7MMCQALFJ36G4TGB2S9A47APLTILU6C6744CEBG4EKCGV357N21BSLH8JD33GQMFAR6DQ70S76P34J6FR=" +- + id: 4 + credential_id: "APU4B1NDTEVTEM60V4T0FRL7SRJMO9KIE2AKFQ8JDGTQ7VHFI41FDEFTDLBVQEAE4ER49QV2GTGVFDNBO31BPOA3OQN6879OT6MTU3G=" diff --git a/models/migrations/fixtures/Test_storeWebauthnCredentialIDAsBytes/webauthn_credential.yml b/models/migrations/fixtures/Test_storeWebauthnCredentialIDAsBytes/webauthn_credential.yml new file mode 100644 index 0000000000000..c02a76e3742a9 --- /dev/null +++ b/models/migrations/fixtures/Test_storeWebauthnCredentialIDAsBytes/webauthn_credential.yml @@ -0,0 +1,31 @@ +- + id: 1 + lower_name: "u2fkey-correctly-migrated" + name: "u2fkey-correctly-migrated" + user_id: 1 + credential_id: "TVHE44TOH7DF7V48SEAIT3EMMJ7TGBOQ289E5AQB34S98LFCUFJ7U2NAVI8RJG6K2F4TC8AQ8KBNO7AGEOQOL9NE43GR63HTEHJSLOG=" + public_key: 0x040d0967a2cad045011631187576492a0beb5b377954b4f694c5afc8bdf25270f87f09a9ab6ce9c282f447ba71b2f2bae2105b32b847e0704f310f48644e3eddf2 + attestation_type: 'fido-u2f' + sign_count: 1 + clone_warning: false +- + id: 2 + lower_name: "non-u2f-key" + name: "non-u2f-key" + user_id: 1 + credential_id: "051CLMMKB62S6M9M2A4H54K7MMCQALFJ36G4TGB2S9A47APLTILU6C6744CEBG4EKCGV357N21BSLH8JD33GQMFAR6DQ70S76P34J6FR" + public_key: 0x040d0967a2cad045011631187576492a0beb5b377954b4f694c5afc8bdf25270f87f09a9ab6ce9c282f447ba71b2f2bae2105b32b847e0704f310f48644e3eddf2 + attestation_type: 'none' + sign_count: 1 + clone_warning: false +- + id: 4 + lower_name: "packed-key" + name: "packed-key" + user_id: 1 + credential_id: "APU4B1NDTEVTEM60V4T0FRL7SRJMO9KIE2AKFQ8JDGTQ7VHFI41FDEFTDLBVQEAE4ER49QV2GTGVFDNBO31BPOA3OQN6879OT6MTU3G=" + public_key: 0x040d0967a2cad045011631187576492a0beb5b377954b4f694c5afc8bdf25270f87f09a9ab6ce9c282f447ba71b2f2bae2105b32b847e0704f310f48644e3eddf2 + attestation_type: 'fido-u2f' + sign_count: 1 + clone_warning: false + diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index beeba866dc70f..2719f45efbbbc 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -400,6 +400,12 @@ var migrations = []Migration{ NewMigration("Add sync_on_commit column to push_mirror table", addSyncOnCommitColForPushMirror), // v220 -> v221 NewMigration("Add container repository property", addContainerRepositoryProperty), + // v221 -> v222 + NewMigration("Store WebAuthentication CredentialID as bytes and increase size to at least 1024", storeWebauthnCredentialIDAsBytes), + // v222 -> v223 + NewMigration("Drop old CredentialID column", dropOldCredentialIDColumn), + // v223 -> v224 + NewMigration("Rename CredentialIDBytes column to CredentialID", renameCredentialIDBytes), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v221.go b/models/migrations/v221.go new file mode 100644 index 0000000000000..f3bcfcdf1de20 --- /dev/null +++ b/models/migrations/v221.go @@ -0,0 +1,75 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "encoding/base32" + "fmt" + + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func storeWebauthnCredentialIDAsBytes(x *xorm.Engine) error { + // Create webauthnCredential table + type webauthnCredential struct { + ID int64 `xorm:"pk autoincr"` + Name string + LowerName string `xorm:"unique(s)"` + UserID int64 `xorm:"INDEX unique(s)"` + CredentialID string `xorm:"INDEX VARCHAR(410)"` + // Note the lack of INDEX here - these will be created once the column is renamed in v223.go + CredentialIDBytes []byte `xorm:"VARBINARY(1024)"` // CredentialID is at most 1023 bytes as per spec released 20 July 2022 + PublicKey []byte + AttestationType string + AAGUID []byte + SignCount uint32 `xorm:"BIGINT"` + CloneWarning bool + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + } + if err := x.Sync2(&webauthnCredential{}); err != nil { + return err + } + + var start int + creds := make([]*webauthnCredential, 0, 50) + for { + err := x.Select("id, credential_id").OrderBy("id").Limit(50, start).Find(&creds) + if err != nil { + return err + } + + err = func() error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("unable to allow start session. Error: %w", err) + } + for _, cred := range creds { + cred.CredentialIDBytes, err = base32.HexEncoding.DecodeString(cred.CredentialID) + if err != nil { + return fmt.Errorf("unable to parse credential id %s for credential[%d]: %w", cred.CredentialID, cred.ID, err) + } + count, err := sess.ID(cred.ID).Cols("credential_id_bytes").Update(cred) + if count != 1 || err != nil { + return fmt.Errorf("unable to update credential id bytes for credential[%d]: %d,%w", cred.ID, count, err) + } + } + return sess.Commit() + }() + if err != nil { + return err + } + + if len(creds) < 50 { + break + } + start += 50 + creds = creds[:0] + } + return nil +} diff --git a/models/migrations/v221_test.go b/models/migrations/v221_test.go new file mode 100644 index 0000000000000..c50ca5c873291 --- /dev/null +++ b/models/migrations/v221_test.go @@ -0,0 +1,65 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "encoding/base32" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_storeWebauthnCredentialIDAsBytes(t *testing.T) { + // Create webauthnCredential table + type WebauthnCredential struct { + ID int64 `xorm:"pk autoincr"` + Name string + LowerName string `xorm:"unique(s)"` + UserID int64 `xorm:"INDEX unique(s)"` + CredentialID string `xorm:"INDEX VARCHAR(410)"` + PublicKey []byte + AttestationType string + AAGUID []byte + SignCount uint32 `xorm:"BIGINT"` + CloneWarning bool + } + + type ExpectedWebauthnCredential struct { + ID int64 `xorm:"pk autoincr"` + CredentialID string // CredentialID is at most 1023 bytes as per spec released 20 July 2022 + } + + type ConvertedWebauthnCredential struct { + ID int64 `xorm:"pk autoincr"` + CredentialIDBytes []byte `xorm:"VARBINARY(1024)"` // CredentialID is at most 1023 bytes as per spec released 20 July 2022 + } + + // Prepare and load the testing database + x, deferable := prepareTestEnv(t, 0, new(WebauthnCredential), new(ExpectedWebauthnCredential)) + defer deferable() + if x == nil || t.Failed() { + return + } + + if err := storeWebauthnCredentialIDAsBytes(x); err != nil { + assert.NoError(t, err) + return + } + + expected := []ExpectedWebauthnCredential{} + if err := x.Table("expected_webauthn_credential").Asc("id").Find(&expected); !assert.NoError(t, err) { + return + } + + got := []ConvertedWebauthnCredential{} + if err := x.Table("webauthn_credential").Select("id, credential_id_bytes").Asc("id").Find(&got); !assert.NoError(t, err) { + return + } + + for i, e := range expected { + credIDBytes, _ := base32.HexEncoding.DecodeString(e.CredentialID) + assert.Equal(t, credIDBytes, got[i].CredentialIDBytes) + } +} diff --git a/models/migrations/v222.go b/models/migrations/v222.go new file mode 100644 index 0000000000000..99acdfd20608a --- /dev/null +++ b/models/migrations/v222.go @@ -0,0 +1,64 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "context" + "fmt" + + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func dropOldCredentialIDColumn(x *xorm.Engine) error { + // This migration maybe rerun so that we should check if it has been run + credentialIDExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "webauthn_credential", "credential_id") + if err != nil { + return err + } + if !credentialIDExist { + // Column is already non-extant + return nil + } + credentialIDBytesExists, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "webauthn_credential", "credential_id_bytes") + if err != nil { + return err + } + if !credentialIDBytesExists { + // looks like 221 hasn't properly run + return fmt.Errorf("webauthn_credential does not have a credential_id_bytes column... it is not safe to run this migration") + } + + // Create webauthnCredential table + type webauthnCredential struct { + ID int64 `xorm:"pk autoincr"` + Name string + LowerName string `xorm:"unique(s)"` + UserID int64 `xorm:"INDEX unique(s)"` + CredentialID string `xorm:"INDEX VARCHAR(410)"` + // Note the lack of the INDEX on CredentialIDBytes - we will add this in v223.go + CredentialIDBytes []byte `xorm:"VARBINARY(1024)"` // CredentialID is at most 1023 bytes as per spec released 20 July 2022 + PublicKey []byte + AttestationType string + AAGUID []byte + SignCount uint32 `xorm:"BIGINT"` + CloneWarning bool + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + } + if err := x.Sync2(&webauthnCredential{}); err != nil { + return err + } + + // Drop the old credential ID + sess := x.NewSession() + defer sess.Close() + + if err := dropTableColumns(sess, "webauthn_credential", "credential_id"); err != nil { + return fmt.Errorf("unable to drop old credentialID column: %w", err) + } + return sess.Commit() +} diff --git a/models/migrations/v223.go b/models/migrations/v223.go new file mode 100644 index 0000000000000..d7ee4812b8264 --- /dev/null +++ b/models/migrations/v223.go @@ -0,0 +1,103 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "context" + "fmt" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func renameCredentialIDBytes(x *xorm.Engine) error { + // This migration maybe rerun so that we should check if it has been run + credentialIDExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "webauthn_credential", "credential_id") + if err != nil { + return err + } + if credentialIDExist { + credentialIDBytesExists, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "webauthn_credential", "credential_id_bytes") + if err != nil { + return err + } + if !credentialIDBytesExists { + return nil + } + } + + err = func() error { + // webauthnCredential table + type webauthnCredential struct { + ID int64 `xorm:"pk autoincr"` + Name string + LowerName string `xorm:"unique(s)"` + UserID int64 `xorm:"INDEX unique(s)"` + // Note the lack of INDEX here + CredentialIDBytes []byte `xorm:"VARBINARY(1024)"` // CredentialID is at most 1023 bytes as per spec released 20 July 2022 + PublicKey []byte + AttestationType string + AAGUID []byte + SignCount uint32 `xorm:"BIGINT"` + CloneWarning bool + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + } + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + + if err := sess.Sync2(new(webauthnCredential)); err != nil { + return fmt.Errorf("error on Sync2: %v", err) + } + + if credentialIDExist { + // if both errors and message exist, drop message at first + if err := dropTableColumns(sess, "webauthn_credential", "credential_id"); err != nil { + return err + } + } + + switch { + case setting.Database.UseMySQL: + if _, err := sess.Exec("ALTER TABLE `webauthn_credential` CHANGE credential_id_bytes credential_id VARBINARY(1024)"); err != nil { + return err + } + case setting.Database.UseMSSQL: + if _, err := sess.Exec("sp_rename 'webauthn_credential.credential_id_bytes', 'credential_id', 'COLUMN'"); err != nil { + return err + } + default: + if _, err := sess.Exec("ALTER TABLE `webauthn_credential` RENAME COLUMN credential_id_bytes TO credential_id"); err != nil { + return err + } + } + return sess.Commit() + }() + if err != nil { + return err + } + + // Create webauthnCredential table + type webauthnCredential struct { + ID int64 `xorm:"pk autoincr"` + Name string + LowerName string `xorm:"unique(s)"` + UserID int64 `xorm:"INDEX unique(s)"` + CredentialID []byte `xorm:"INDEX VARBINARY(1024)"` // CredentialID is at most 1023 bytes as per spec released 20 July 2022 + PublicKey []byte + AttestationType string + AAGUID []byte + SignCount uint32 `xorm:"BIGINT"` + CloneWarning bool + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + } + return x.Sync2(&webauthnCredential{}) +} diff --git a/models/repo/pushmirror.go b/models/repo/pushmirror.go index 0a7dea79c93b3..02ffcbee76a50 100644 --- a/models/repo/pushmirror.go +++ b/models/repo/pushmirror.go @@ -5,12 +5,15 @@ package repo import ( + "context" "errors" "time" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" ) // ErrPushMirrorNotExist mirror does not exist error @@ -29,6 +32,25 @@ type PushMirror struct { LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` LastError string `xorm:"text"` } +type PushMirrorOptions struct { + ID int64 + RepoID int64 + RemoteName string +} + +func (opts *PushMirrorOptions) toConds() builder.Cond { + cond := builder.NewCond() + if opts.RepoID > 0 { + cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) + } + if opts.RemoteName != "" { + cond = cond.And(builder.Eq{"remote_name": opts.RemoteName}) + } + if opts.ID > 0 { + cond = cond.And(builder.Eq{"id": opts.ID}) + } + return cond +} func init() { db.RegisterModel(new(PushMirror)) @@ -53,45 +75,48 @@ func (m *PushMirror) GetRemoteName() string { } // InsertPushMirror inserts a push-mirror to database -func InsertPushMirror(m *PushMirror) error { - _, err := db.GetEngine(db.DefaultContext).Insert(m) +func InsertPushMirror(ctx context.Context, m *PushMirror) error { + _, err := db.GetEngine(ctx).Insert(m) return err } // UpdatePushMirror updates the push-mirror -func UpdatePushMirror(m *PushMirror) error { - _, err := db.GetEngine(db.DefaultContext).ID(m.ID).AllCols().Update(m) - return err -} - -// DeletePushMirrorByID deletes a push-mirrors by ID -func DeletePushMirrorByID(ID int64) error { - _, err := db.GetEngine(db.DefaultContext).ID(ID).Delete(&PushMirror{}) +func UpdatePushMirror(ctx context.Context, m *PushMirror) error { + _, err := db.GetEngine(ctx).ID(m.ID).AllCols().Update(m) return err } -// DeletePushMirrorsByRepoID deletes all push-mirrors by repoID -func DeletePushMirrorsByRepoID(repoID int64) error { - _, err := db.GetEngine(db.DefaultContext).Delete(&PushMirror{RepoID: repoID}) - return err +func DeletePushMirrors(ctx context.Context, opts PushMirrorOptions) error { + if opts.RepoID > 0 { + _, err := db.GetEngine(ctx).Where(opts.toConds()).Delete(&PushMirror{}) + return err + } + return errors.New("repoID required and must be set") } -// GetPushMirrorByID returns push-mirror information. -func GetPushMirrorByID(ID int64) (*PushMirror, error) { - m := &PushMirror{} - has, err := db.GetEngine(db.DefaultContext).ID(ID).Get(m) +func GetPushMirror(ctx context.Context, opts PushMirrorOptions) (*PushMirror, error) { + mirror := &PushMirror{} + exist, err := db.GetEngine(ctx).Where(opts.toConds()).Get(mirror) if err != nil { return nil, err - } else if !has { + } else if !exist { return nil, ErrPushMirrorNotExist } - return m, nil + return mirror, nil } // GetPushMirrorsByRepoID returns push-mirror information of a repository. -func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) { +func GetPushMirrorsByRepoID(ctx context.Context, repoID int64, listOptions db.ListOptions) ([]*PushMirror, int64, error) { + sess := db.GetEngine(ctx).Where("repo_id = ?", repoID) + if listOptions.Page != 0 { + sess = db.SetSessionPagination(sess, &listOptions) + mirrors := make([]*PushMirror, 0, listOptions.PageSize) + count, err := sess.FindAndCount(&mirrors) + return mirrors, count, err + } mirrors := make([]*PushMirror, 0, 10) - return mirrors, db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID).Find(&mirrors) + count, err := sess.FindAndCount(&mirrors) + return mirrors, count, err } // GetPushMirrorsSyncedOnCommit returns push-mirrors for this repo that should be updated by new commits @@ -103,8 +128,8 @@ func GetPushMirrorsSyncedOnCommit(repoID int64) ([]*PushMirror, error) { } // PushMirrorsIterate iterates all push-mirror repositories. -func PushMirrorsIterate(limit int, f func(idx int, bean interface{}) error) error { - return db.GetEngine(db.DefaultContext). +func PushMirrorsIterate(ctx context.Context, limit int, f func(idx int, bean interface{}) error) error { + return db.GetEngine(ctx). Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()). And("`interval` != 0"). OrderBy("last_update ASC"). diff --git a/models/repo/pushmirror_test.go b/models/repo/pushmirror_test.go index d36a48547e1c6..5087e30095773 100644 --- a/models/repo/pushmirror_test.go +++ b/models/repo/pushmirror_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/timeutil" @@ -20,20 +21,20 @@ func TestPushMirrorsIterate(t *testing.T) { now := timeutil.TimeStampNow() - repo_model.InsertPushMirror(&repo_model.PushMirror{ + repo_model.InsertPushMirror(db.DefaultContext, &repo_model.PushMirror{ RemoteName: "test-1", LastUpdateUnix: now, Interval: 1, }) long, _ := time.ParseDuration("24h") - repo_model.InsertPushMirror(&repo_model.PushMirror{ + repo_model.InsertPushMirror(db.DefaultContext, &repo_model.PushMirror{ RemoteName: "test-2", LastUpdateUnix: now, Interval: long, }) - repo_model.InsertPushMirror(&repo_model.PushMirror{ + repo_model.InsertPushMirror(db.DefaultContext, &repo_model.PushMirror{ RemoteName: "test-3", LastUpdateUnix: now, Interval: 0, @@ -41,7 +42,7 @@ func TestPushMirrorsIterate(t *testing.T) { time.Sleep(1 * time.Millisecond) - repo_model.PushMirrorsIterate(1, func(idx int, bean interface{}) error { + repo_model.PushMirrorsIterate(db.DefaultContext, 1, func(idx int, bean interface{}) error { m, ok := bean.(*repo_model.PushMirror) assert.True(t, ok) assert.Equal(t, "test-1", m.RemoteName) diff --git a/modules/context/repo.go b/modules/context/repo.go index 1d9f98158e19b..ea40542069991 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -393,7 +393,7 @@ func repoAssignment(ctx *Context, repo *repo_model.Repository) { } } - pushMirrors, err := repo_model.GetPushMirrorsByRepoID(repo.ID) + pushMirrors, _, err := repo_model.GetPushMirrorsByRepoID(ctx, repo.ID, db.ListOptions{}) if err != nil { ctx.ServerError("GetPushMirrorsByRepoID", err) return diff --git a/modules/convert/mirror.go b/modules/convert/mirror.go new file mode 100644 index 0000000000000..b2414f46774cc --- /dev/null +++ b/modules/convert/mirror.go @@ -0,0 +1,39 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package convert + +import ( + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + api "code.gitea.io/gitea/modules/structs" +) + +// ToPushMirror convert from repo_model.PushMirror and remoteAddress to api.TopicResponse +func ToPushMirror(pm *repo_model.PushMirror) (*api.PushMirror, error) { + repo := pm.GetRepository() + remoteAddress, err := getRemoteAddress(repo, pm.RemoteName) + if err != nil { + return nil, err + } + return &api.PushMirror{ + RepoName: repo.Name, + RemoteName: pm.RemoteName, + RemoteAddress: remoteAddress, + CreatedUnix: pm.CreatedUnix.FormatLong(), + LastUpdateUnix: pm.LastUpdateUnix.FormatLong(), + LastError: pm.LastError, + Interval: pm.Interval.String(), + }, nil +} + +func getRemoteAddress(repo *repo_model.Repository, remoteName string) (string, error) { + url, err := git.GetRemoteURL(git.DefaultContext, repo.RepoPath(), remoteName) + if err != nil { + return "", err + } + // remove confidential information + url.User = nil + return url.String(), nil +} diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index 6832207c0fc75..af3376e8d7127 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -10,6 +10,7 @@ import ( "bytes" "fmt" gohtml "html" + "io" "path/filepath" "strings" "sync" @@ -26,7 +27,7 @@ import ( ) // don't index files larger than this many bytes for performance purposes -const sizeLimit = 1000000 +const sizeLimit = 1024 * 1024 var ( // For custom user mapping @@ -46,7 +47,6 @@ func NewContext() { highlightMapping[keys[i].Name()] = keys[i].Value() } } - // The size 512 is simply a conservative rule of thumb c, err := lru.New2Q(512) if err != nil { @@ -60,7 +60,7 @@ func NewContext() { func Code(fileName, language, code string) string { NewContext() - // diff view newline will be passed as empty, change to literal \n so it can be copied + // diff view newline will be passed as empty, change to literal '\n' so it can be copied // preserve literal newline in blame view if code == "" || code == "\n" { return "\n" @@ -128,36 +128,32 @@ func CodeFromLexer(lexer chroma.Lexer, code string) string { return code } - htmlw.Flush() + _ = htmlw.Flush() // Chroma will add newlines for certain lexers in order to highlight them properly - // Once highlighted, strip them here so they don't cause copy/paste trouble in HTML output + // Once highlighted, strip them here, so they don't cause copy/paste trouble in HTML output return strings.TrimSuffix(htmlbuf.String(), "\n") } -// File returns a slice of chroma syntax highlighted lines of code -func File(numLines int, fileName, language string, code []byte) []string { +// File returns a slice of chroma syntax highlighted HTML lines of code +func File(fileName, language string, code []byte) ([]string, error) { NewContext() if len(code) > sizeLimit { - return plainText(string(code), numLines) + return PlainText(code), nil } + formatter := html.New(html.WithClasses(true), html.WithLineNumbers(false), html.PreventSurroundingPre(true), ) - if formatter == nil { - log.Error("Couldn't create chroma formatter") - return plainText(string(code), numLines) - } - - htmlbuf := bytes.Buffer{} - htmlw := bufio.NewWriter(&htmlbuf) + htmlBuf := bytes.Buffer{} + htmlWriter := bufio.NewWriter(&htmlBuf) var lexer chroma.Lexer // provided language overrides everything - if len(language) > 0 { + if language != "" { lexer = lexers.Get(language) } @@ -168,9 +164,9 @@ func File(numLines int, fileName, language string, code []byte) []string { } if lexer == nil { - language := analyze.GetCodeLanguage(fileName, code) + guessLanguage := analyze.GetCodeLanguage(fileName, code) - lexer = lexers.Get(language) + lexer = lexers.Get(guessLanguage) if lexer == nil { lexer = lexers.Match(fileName) if lexer == nil { @@ -181,54 +177,43 @@ func File(numLines int, fileName, language string, code []byte) []string { iterator, err := lexer.Tokenise(nil, string(code)) if err != nil { - log.Error("Can't tokenize code: %v", err) - return plainText(string(code), numLines) + return nil, fmt.Errorf("can't tokenize code: %w", err) } - err = formatter.Format(htmlw, styles.GitHub, iterator) + err = formatter.Format(htmlWriter, styles.GitHub, iterator) if err != nil { - log.Error("Can't format code: %v", err) - return plainText(string(code), numLines) + return nil, fmt.Errorf("can't format code: %w", err) } - htmlw.Flush() - finalNewLine := false - if len(code) > 0 { - finalNewLine = code[len(code)-1] == '\n' - } + _ = htmlWriter.Flush() - m := make([]string, 0, numLines) - for _, v := range strings.SplitN(htmlbuf.String(), "\n", numLines) { - content := v - // need to keep lines that are only \n so copy/paste works properly in browser - if content == "" { - content = "\n" - } else if content == `` { - content += "\n" - } else if content == `` { - content += "\n" - } - content = strings.TrimSuffix(content, ``) - content = strings.TrimPrefix(content, ``) - m = append(m, content) + // at the moment, Chroma generates stable output `...\n` for each line + htmlStr := htmlBuf.String() + lines := strings.Split(htmlStr, ``) + m := make([]string, 0, len(lines)) + for i := 1; i < len(lines); i++ { + line := lines[i] + line = strings.TrimSuffix(line, "") + m = append(m, line) } - if finalNewLine { - m = append(m, "\n") - } - - return m + return m, nil } -// return unhiglighted map -func plainText(code string, numLines int) []string { - m := make([]string, 0, numLines) - for _, v := range strings.SplitN(code, "\n", numLines) { - content := v - // need to keep lines that are only \n so copy/paste works properly in browser - if content == "" { - content = "\n" +// PlainText returns non-highlighted HTML for code +func PlainText(code []byte) []string { + r := bufio.NewReader(bytes.NewReader(code)) + m := make([]string, 0, bytes.Count(code, []byte{'\n'})+1) + for { + content, err := r.ReadString('\n') + if err != nil && err != io.EOF { + log.Error("failed to read string from buffer: %v", err) + break + } + if content == "" && err == io.EOF { + break } - m = append(m, gohtml.EscapeString(content)) + s := gohtml.EscapeString(content) + m = append(m, s) } return m } diff --git a/modules/highlight/highlight_test.go b/modules/highlight/highlight_test.go index e5dfedd2b3c8e..8f83f4a2f6128 100644 --- a/modules/highlight/highlight_test.go +++ b/modules/highlight/highlight_test.go @@ -8,97 +8,146 @@ import ( "strings" "testing" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" - "github.com/stretchr/testify/assert" - "gopkg.in/ini.v1" ) +func lines(s string) []string { + return strings.Split(strings.ReplaceAll(strings.TrimSpace(s), `\n`, "\n"), "\n") +} + func TestFile(t *testing.T) { - setting.Cfg = ini.Empty() tests := []struct { - name string - numLines int - fileName string - code string - want string + name string + code string + want []string }{ { - name: ".drone.yml", - numLines: 12, - fileName: ".drone.yml", - code: util.Dedent(` - kind: pipeline - name: default + name: "empty.py", + code: "", + want: lines(""), + }, + { + name: "tags.txt", + code: "<>", + want: lines("<>"), + }, + { + name: "tags.py", + code: "<>", + want: lines(`<>`), + }, + { + name: "eol-no.py", + code: "a=1", + want: lines(`a=1`), + }, + { + name: "eol-newline1.py", + code: "a=1\n", + want: lines(`a=1\n`), + }, + { + name: "eol-newline2.py", + code: "a=1\n\n", + want: lines(` +a=1\n +\n + `, + ), + }, + { + name: "empty-line-with-space.py", + code: strings.ReplaceAll(strings.TrimSpace(` +def: + a=1 - steps: - - name: test - image: golang:1.13 - environment: - GOPROXY: https://goproxy.cn - commands: - - go get -u - - go build -v - - go test -v -race -coverprofile=coverage.txt -covermode=atomic - `), - want: util.Dedent(` - kind: pipeline - name: default - - steps: - - name: test - image: golang:1.13 - environment: - GOPROXY: https://goproxy.cn - commands: - - go get -u - - go build -v - - go test -v -race -coverprofile=coverage.txt -covermode=atomic +b='' +{space} +c=2 + `), "{space}", " "), + want: lines(` +def:\n + a=1\n +\n +b=''\n + \n +c=2`, + ), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out, err := File(tt.name, "", []byte(tt.code)) + assert.NoError(t, err) + expected := strings.Join(tt.want, "\n") + actual := strings.Join(out, "\n") + assert.Equal(t, strings.Count(actual, "")) + assert.EqualValues(t, expected, actual) + }) + } +} + +func TestPlainText(t *testing.T) { + tests := []struct { + name string + code string + want []string + }{ + { + name: "empty.py", + code: "", + want: lines(""), + }, + { + name: "tags.py", + code: "<>", + want: lines("<>"), + }, + { + name: "eol-no.py", + code: "a=1", + want: lines(`a=1`), + }, + { + name: "eol-newline1.py", + code: "a=1\n", + want: lines(`a=1\n`), + }, + { + name: "eol-newline2.py", + code: "a=1\n\n", + want: lines(` +a=1\n +\n `), }, { - name: ".drone.yml - trailing space", - numLines: 13, - fileName: ".drone.yml", - code: strings.Replace(util.Dedent(` - kind: pipeline - name: default + name: "empty-line-with-space.py", + code: strings.ReplaceAll(strings.TrimSpace(` +def: + a=1 - steps: - - name: test - image: golang:1.13 - environment: - GOPROXY: https://goproxy.cn - commands: - - go get -u - - go build -v - - go test -v -race -coverprofile=coverage.txt -covermode=atomic - `)+"\n", "name: default", "name: default ", 1), - want: util.Dedent(` - kind: pipeline - name: default - - steps: - - name: test - image: golang:1.13 - environment: - GOPROXY: https://goproxy.cn - commands: - - go get -u - - go build -v - - go test -v -race -coverprofile=coverage.txt -covermode=atomic - - - - `), +b='' +{space} +c=2 + `), "{space}", " "), + want: lines(` +def:\n + a=1\n +\n +b=''\n + \n +c=2`), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := strings.Join(File(tt.numLines, tt.fileName, "", []byte(tt.code)), "\n") - assert.Equal(t, tt.want, got) + out := PlainText([]byte(tt.code)) + expected := strings.Join(tt.want, "\n") + actual := strings.Join(out, "\n") + assert.EqualValues(t, expected, actual) }) } } diff --git a/modules/setting/database.go b/modules/setting/database.go index 8fdd5f2bcb2ff..af4e780d76bd4 100644 --- a/modules/setting/database.go +++ b/modules/setting/database.go @@ -39,6 +39,7 @@ var ( LogSQL bool Charset string Timeout int // seconds + SQLiteJournalMode string UseSQLite3 bool UseMySQL bool UseMSSQL bool @@ -91,6 +92,8 @@ func InitDBConfig() { Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db")) Database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500) + Database.SQLiteJournalMode = sec.Key("SQLITE_JOURNAL_MODE").MustString("") + Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2) if Database.UseMySQL { Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFETIME").MustDuration(3 * time.Second) @@ -136,7 +139,12 @@ func DBConnStr() (string, error) { if err := os.MkdirAll(path.Dir(Database.Path), os.ModePerm); err != nil { return "", fmt.Errorf("Failed to create directories: %v", err) } - connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate", Database.Path, Database.Timeout) + journalMode := "" + if Database.SQLiteJournalMode != "" { + journalMode = "&_journal_mode=" + Database.SQLiteJournalMode + } + connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate%s", + Database.Path, Database.Timeout, journalMode) default: return "", fmt.Errorf("Unknown database type: %s", Database.Type) } diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 733bc6d90e600..d0406dbf90284 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -48,6 +48,7 @@ var ( DefaultBranch string AllowAdoptionOfUnadoptedRepositories bool AllowDeleteOfUnadoptedRepositories bool + DisableDownloadSourceArchives bool // Repository editor settings Editor struct { diff --git a/modules/structs/mirror.go b/modules/structs/mirror.go new file mode 100644 index 0000000000000..8e8a8a2705d2f --- /dev/null +++ b/modules/structs/mirror.go @@ -0,0 +1,25 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package structs + +// CreatePushMirrorOption represents need information to create a push mirror of a repository. +type CreatePushMirrorOption struct { + RemoteAddress string `json:"remote_address"` + RemoteUsername string `json:"remote_username"` + RemotePassword string `json:"remote_password"` + Interval string `json:"interval"` +} + +// PushMirror represents information of a push mirror +// swagger:model +type PushMirror struct { + RepoName string `json:"repo_name"` + RemoteName string `json:"remote_name"` + RemoteAddress string `json:"remote_address"` + CreatedUnix string `json:"created"` + LastUpdateUnix string `json:"last_update"` + LastError string `json:"last_error"` + Interval string `json:"interval"` +} diff --git a/modules/templates/base.go b/modules/templates/base.go index 282019f826c1d..9563650e127b7 100644 --- a/modules/templates/base.go +++ b/modules/templates/base.go @@ -35,10 +35,11 @@ func BaseVars() Vars { "IsLandingPageExplore": setting.LandingPageURL == setting.LandingPageExplore, "IsLandingPageOrganizations": setting.LandingPageURL == setting.LandingPageOrganizations, - "ShowRegistrationButton": setting.Service.ShowRegistrationButton, - "ShowMilestonesDashboardPage": setting.Service.ShowMilestonesDashboardPage, - "ShowFooterBranding": setting.ShowFooterBranding, - "ShowFooterVersion": setting.ShowFooterVersion, + "ShowRegistrationButton": setting.Service.ShowRegistrationButton, + "ShowMilestonesDashboardPage": setting.Service.ShowMilestonesDashboardPage, + "ShowFooterBranding": setting.ShowFooterBranding, + "ShowFooterVersion": setting.ShowFooterVersion, + "DisableDownloadSourceArchives": setting.Repository.DisableDownloadSourceArchives, "EnableSwagger": setting.API.EnableSwagger, "EnableOpenIDSignIn": setting.Service.EnableOpenIDSignIn, diff --git a/options/license/MS-LPL b/options/license/MS-LPL new file mode 100644 index 0000000000000..ea8bffcaae217 --- /dev/null +++ b/options/license/MS-LPL @@ -0,0 +1,24 @@ +Microsoft Limited Public License (Ms-LPL) + +This license governs use of the accompanying software. If you use the software, you accept this license. If you do not accept the license, do not use the software. + +1. Definitions +The terms "reproduce," "reproduction," "derivative works," and "distribution" have the same meaning here as under U.S. copyright law. A "contribution" is the original software, or any additions or changes to the software. A "contributor" is any person that distributes its contribution under this license. "Licensed patents" are a contributor's patent claims that read directly on its contribution. + +2. Grant of Rights + (A) Copyright Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free copyright license to reproduce its contribution, prepare derivative works of its contribution, and distribute its contribution or any derivative works that you create. + + (B) Patent Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free license under its licensed patents to make, have made, use, sell, offer for sale, import, and/or otherwise dispose of its contribution in the software or derivative works of the contribution in the software. + +3. Conditions and Limitations + (A) No Trademark License- This license does not grant you rights to use any contributors' name, logo, or trademarks. + + (B) If you bring a patent claim against any contributor over patents that you claim are infringed by the software, your patent license from such contributor to the software ends automatically. + + (C) If you distribute any portion of the software, you must retain all copyright, patent, trademark, and attribution notices that are present in the software. + + (D) If you distribute any portion of the software in source code form, you may do so only under this license by including a complete copy of this license with your distribution. If you distribute any portion of the software in compiled or object code form, you may only do so under a license that complies with this license. + + (E) The software is licensed "as-is." You bear the risk of using it. The contributors give no express warranties, guarantees, or conditions. You may have additional consumer rights under your local laws which this license cannot change. To the extent permitted under your local laws, the contributors exclude the implied warranties of merchantability, fitness for a particular purpose and non-infringement. + + (F) Platform Limitation- The licenses granted in sections 2(A) & 2(B) extend only to the software or derivative works that you create that run on a Microsoft Windows operating system product. diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 44e0c290a0881..e1478fa2aa99a 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -982,6 +982,15 @@ func Routes() *web.Route { }) }, reqRepoReader(unit.TypeReleases)) m.Post("/mirror-sync", reqToken(), reqRepoWriter(unit.TypeCode), repo.MirrorSync) + m.Post("/push_mirrors-sync", reqAdmin(), repo.PushMirrorSync) + m.Group("/push_mirrors", func() { + m.Combo("").Get(repo.ListPushMirrors). + Post(bind(api.CreatePushMirrorOption{}), repo.AddPushMirror) + m.Combo("/{name}"). + Delete(repo.DeletePushMirrorByRemoteName). + Get(repo.GetPushMirrorByName) + }, reqAdmin()) + m.Get("/editorconfig/{filename}", context.ReferencesGitRepo(), context.RepoRefForAPI, reqRepoReader(unit.TypeCode), repo.GetEditorconfig) m.Group("/pulls", func() { m.Combo("").Get(repo.ListPullRequests). diff --git a/routers/api/v1/repo/mirror.go b/routers/api/v1/repo/mirror.go index 3d29383550c32..91e5e0c031617 100644 --- a/routers/api/v1/repo/mirror.go +++ b/routers/api/v1/repo/mirror.go @@ -6,13 +6,25 @@ package repo import ( "errors" + "fmt" "net/http" + "time" + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/convert" mirror_module "code.gitea.io/gitea/modules/mirror" "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/api/v1/utils" + "code.gitea.io/gitea/services/forms" + "code.gitea.io/gitea/services/migrations" + mirror_service "code.gitea.io/gitea/services/mirror" ) // MirrorSync adds a mirrored repository to the sync queue @@ -63,3 +75,317 @@ func MirrorSync(ctx *context.APIContext) { ctx.Status(http.StatusOK) } + +// PushMirrorSync adds all push mirrored repositories to the sync queue +func PushMirrorSync(ctx *context.APIContext) { + // swagger:operation POST /repos/{owner}/{repo}/push_mirrors-sync repository repoPushMirrorSync + // --- + // summary: Sync all push mirrored repository + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo to sync + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo to sync + // type: string + // required: true + // responses: + // "200": + // "$ref": "#/responses/empty" + // "400": + // "$ref": "#/responses/error" + // "403": + // "$ref": "#/responses/forbidden" + + if !setting.Mirror.Enabled { + ctx.Error(http.StatusBadRequest, "PushMirrorSync", "Mirror feature is disabled") + return + } + // Get All push mirrors of a specific repo + pushMirrors, _, err := repo_model.GetPushMirrorsByRepoID(ctx, ctx.Repo.Repository.ID, db.ListOptions{}) + if err != nil { + ctx.Error(http.StatusNotFound, "PushMirrorSync", err) + return + } + for _, mirror := range pushMirrors { + ok := mirror_service.SyncPushMirror(ctx, mirror.ID) + if !ok { + ctx.Error(http.StatusInternalServerError, "PushMirrorSync", "error occurred when syncing push mirror "+mirror.RemoteName) + return + } + } + + ctx.Status(http.StatusOK) +} + +// ListPushMirrors get list of push mirrors of a repository +func ListPushMirrors(ctx *context.APIContext) { + // swagger:operation GET /repos/{owner}/{repo}/push_mirrors repository repoListPushMirrors + // --- + // summary: Get all push mirrors of the repository + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: page + // in: query + // description: page number of results to return (1-based) + // type: integer + // - name: limit + // in: query + // description: page size of results + // type: integer + // responses: + // "200": + // "$ref": "#/responses/PushMirrorList" + // "400": + // "$ref": "#/responses/error" + // "403": + // "$ref": "#/responses/forbidden" + + if !setting.Mirror.Enabled { + ctx.Error(http.StatusBadRequest, "GetPushMirrorsByRepoID", "Mirror feature is disabled") + return + } + + repo := ctx.Repo.Repository + // Get all push mirrors for the specified repository. + pushMirrors, count, err := repo_model.GetPushMirrorsByRepoID(ctx, repo.ID, utils.GetListOptions(ctx)) + if err != nil { + ctx.Error(http.StatusNotFound, "GetPushMirrorsByRepoID", err) + return + } + + responsePushMirrors := make([]*api.PushMirror, 0, len(pushMirrors)) + for _, mirror := range pushMirrors { + m, err := convert.ToPushMirror(mirror) + if err == nil { + responsePushMirrors = append(responsePushMirrors, m) + } + + } + ctx.SetLinkHeader(len(responsePushMirrors), utils.GetListOptions(ctx).PageSize) + ctx.SetTotalCountHeader(count) + ctx.JSON(http.StatusOK, responsePushMirrors) +} + +// GetPushMirrorByName get push mirror of a repository by name +func GetPushMirrorByName(ctx *context.APIContext) { + // swagger:operation GET /repos/{owner}/{repo}/push_mirrors/{name} repository repoGetPushMirrorByRemoteName + // --- + // summary: Get push mirror of the repository by remoteName + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: name + // in: path + // description: remote name of push mirror + // type: string + // required: true + // responses: + // "200": + // "$ref": "#/responses/PushMirror" + // "400": + // "$ref": "#/responses/error" + // "403": + // "$ref": "#/responses/forbidden" + + if !setting.Mirror.Enabled { + ctx.Error(http.StatusBadRequest, "GetPushMirrorByRemoteName", "Mirror feature is disabled") + return + } + + mirrorName := ctx.Params(":name") + // Get push mirror of a specific repo by remoteName + pushMirror, err := repo_model.GetPushMirror(ctx, repo_model.PushMirrorOptions{RepoID: ctx.Repo.Repository.ID, RemoteName: mirrorName}) + if err != nil { + ctx.Error(http.StatusNotFound, "GetPushMirrors", err) + return + } + m, err := convert.ToPushMirror(pushMirror) + if err != nil { + ctx.ServerError("GetPushMirrorByRemoteName", err) + return + } + ctx.JSON(http.StatusOK, m) +} + +// AddPushMirror adds a push mirror to a repository +func AddPushMirror(ctx *context.APIContext) { + // swagger:operation POST /repos/{owner}/{repo}/push_mirrors repository repoAddPushMirror + // --- + // summary: add a push mirror to the repository + // consumes: + // - application/json + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: body + // in: body + // schema: + // "$ref": "#/definitions/CreatePushMirrorOption" + // responses: + // "201": + // "$ref": "#/responses/PushMirror" + // "403": + // "$ref": "#/responses/forbidden" + // "400": + // "$ref": "#/responses/error" + + if !setting.Mirror.Enabled { + ctx.Error(http.StatusBadRequest, "AddPushMirror", "Mirror feature is disabled") + return + } + + pushMirror := web.GetForm(ctx).(*api.CreatePushMirrorOption) + CreatePushMirror(ctx, pushMirror) +} + +// DeletePushMirrorByRemoteName deletes a push mirror from a repository by remoteName +func DeletePushMirrorByRemoteName(ctx *context.APIContext) { + // swagger:operation DELETE /repos/{owner}/{repo}/push_mirrors/{name} repository repoDeletePushMirror + // --- + // summary: deletes a push mirror from a repository by remoteName + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: name + // in: path + // description: remote name of the pushMirror + // type: string + // required: true + // responses: + // "204": + // "$ref": "#/responses/empty" + // "404": + // "$ref": "#/responses/notFound" + // "400": + // "$ref": "#/responses/error" + + if !setting.Mirror.Enabled { + ctx.Error(http.StatusBadRequest, "DeletePushMirrorByName", "Mirror feature is disabled") + return + } + + remoteName := ctx.Params(":name") + // Delete push mirror on repo by name. + err := repo_model.DeletePushMirrors(ctx, repo_model.PushMirrorOptions{RepoID: ctx.Repo.Repository.ID, RemoteName: remoteName}) + if err != nil { + ctx.Error(http.StatusNotFound, "DeletePushMirrors", err) + return + } + ctx.Status(http.StatusNoContent) +} + +func CreatePushMirror(ctx *context.APIContext, mirrorOption *api.CreatePushMirrorOption) { + repo := ctx.Repo.Repository + + interval, err := time.ParseDuration(mirrorOption.Interval) + if err != nil || (interval != 0 && interval < setting.Mirror.MinInterval) { + ctx.Error(http.StatusBadRequest, "CreatePushMirror", err) + return + } + + address, err := forms.ParseRemoteAddr(mirrorOption.RemoteAddress, mirrorOption.RemoteUsername, mirrorOption.RemotePassword) + if err == nil { + err = migrations.IsMigrateURLAllowed(address, ctx.ContextUser) + } + if err != nil { + HandleRemoteAddressError(ctx, err) + return + } + + remoteSuffix, err := util.CryptoRandomString(10) + if err != nil { + ctx.ServerError("CryptoRandomString", err) + return + } + + pushMirror := &repo_model.PushMirror{ + RepoID: repo.ID, + Repo: repo, + RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), + Interval: interval, + } + + if err = repo_model.InsertPushMirror(ctx, pushMirror); err != nil { + ctx.ServerError("InsertPushMirror", err) + return + } + + // if the registration of the push mirrorOption fails remove it from the database + if err = mirror_service.AddPushMirrorRemote(ctx, pushMirror, address); err != nil { + if err := repo_model.DeletePushMirrors(ctx, repo_model.PushMirrorOptions{ID: pushMirror.ID, RepoID: pushMirror.RepoID}); err != nil { + ctx.ServerError("DeletePushMirrors", err) + } + ctx.ServerError("AddPushMirrorRemote", err) + return + } + m, err := convert.ToPushMirror(pushMirror) + if err != nil { + ctx.ServerError("ToPushMirror", err) + return + } + ctx.JSON(http.StatusOK, m) +} + +func HandleRemoteAddressError(ctx *context.APIContext, err error) { + if models.IsErrInvalidCloneAddr(err) { + addrErr := err.(*models.ErrInvalidCloneAddr) + switch { + case addrErr.IsProtocolInvalid: + ctx.Error(http.StatusBadRequest, "CreatePushMirror", "Invalid mirror protocol") + case addrErr.IsURLError: + ctx.Error(http.StatusBadRequest, "CreatePushMirror", "Invalid Url ") + case addrErr.IsPermissionDenied: + ctx.Error(http.StatusUnauthorized, "CreatePushMirror", "Permission denied") + default: + ctx.Error(http.StatusBadRequest, "CreatePushMirror", "Unknown error") + } + return + } +} diff --git a/routers/api/v1/swagger/options.go b/routers/api/v1/swagger/options.go index df3d011246973..e8cfc0706f5ca 100644 --- a/routers/api/v1/swagger/options.go +++ b/routers/api/v1/swagger/options.go @@ -169,4 +169,7 @@ type swaggerParameterBodies struct { // in:body CreateWikiPageOptions api.CreateWikiPageOptions + + // in:body + CreatePushMirrorOption api.CreatePushMirrorOption } diff --git a/routers/api/v1/swagger/repo.go b/routers/api/v1/swagger/repo.go index ab802db7812fe..3522e242764ee 100644 --- a/routers/api/v1/swagger/repo.go +++ b/routers/api/v1/swagger/repo.go @@ -345,6 +345,20 @@ type swaggerWikiCommitList struct { Body api.WikiCommitList `json:"body"` } +// PushMirror +// swagger:response PushMirror +type swaggerPushMirror struct { + // in:body + Body api.PushMirror `json:"body"` +} + +// PushMirrorList +// swagger:response PushMirrorList +type swaggerPushMirrorList struct { + // in:body + Body []api.PushMirror `json:"body"` +} + // RepoCollaboratorPermission // swagger:response RepoCollaboratorPermission type swaggerRepoCollaboratorPermission struct { diff --git a/routers/web/auth/webauthn.go b/routers/web/auth/webauthn.go index 4778c9a9a3690..917cbdd57bf67 100644 --- a/routers/web/auth/webauthn.go +++ b/routers/web/auth/webauthn.go @@ -5,7 +5,6 @@ package auth import ( - "encoding/base32" "errors" "net/http" @@ -129,7 +128,7 @@ func WebAuthnLoginAssertionPost(ctx *context.Context) { } // Success! Get the credential and update the sign count with the new value we received. - dbCred, err := auth.GetWebAuthnCredentialByCredID(user.ID, base32.HexEncoding.EncodeToString(cred.ID)) + dbCred, err := auth.GetWebAuthnCredentialByCredID(user.ID, cred.ID) if err != nil { ctx.ServerError("GetWebAuthnCredentialByCredID", err) return diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 7f6b0feafbd0a..a59824cecdb42 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -90,7 +90,7 @@ func SettingsCtxData(ctx *context.Context) { } ctx.Data["StatsIndexerStatus"] = status } - pushMirrors, err := repo_model.GetPushMirrorsByRepoID(ctx.Repo.Repository.ID) + pushMirrors, _, err := repo_model.GetPushMirrorsByRepoID(ctx, ctx.Repo.Repository.ID, db.ListOptions{}) if err != nil { ctx.ServerError("GetPushMirrorsByRepoID", err) return @@ -284,7 +284,7 @@ func SettingsPost(ctx *context.Context) { return } - m, err := selectPushMirrorByForm(form, repo) + m, err := selectPushMirrorByForm(ctx, form, repo) if err != nil { ctx.NotFound("", nil) return @@ -305,7 +305,7 @@ func SettingsPost(ctx *context.Context) { // as an error on the UI for this action ctx.Data["Err_RepoName"] = nil - m, err := selectPushMirrorByForm(form, repo) + m, err := selectPushMirrorByForm(ctx, form, repo) if err != nil { ctx.NotFound("", nil) return @@ -316,7 +316,7 @@ func SettingsPost(ctx *context.Context) { return } - if err = repo_model.DeletePushMirrorByID(m.ID); err != nil { + if err = repo_model.DeletePushMirrors(ctx, repo_model.PushMirrorOptions{ID: m.ID, RepoID: m.RepoID}); err != nil { ctx.ServerError("DeletePushMirrorByID", err) return } @@ -364,14 +364,14 @@ func SettingsPost(ctx *context.Context) { SyncOnCommit: form.PushMirrorSyncOnCommit, Interval: interval, } - if err := repo_model.InsertPushMirror(m); err != nil { + if err := repo_model.InsertPushMirror(ctx, m); err != nil { ctx.ServerError("InsertPushMirror", err) return } if err := mirror_service.AddPushMirrorRemote(ctx, m, address); err != nil { - if err := repo_model.DeletePushMirrorByID(m.ID); err != nil { - log.Error("DeletePushMirrorByID %v", err) + if err := repo_model.DeletePushMirrors(ctx, repo_model.PushMirrorOptions{ID: m.ID, RepoID: m.RepoID}); err != nil { + log.Error("DeletePushMirrors %v", err) } ctx.ServerError("AddPushMirrorRemote", err) return @@ -1222,13 +1222,13 @@ func SettingsDeleteAvatar(ctx *context.Context) { ctx.Redirect(ctx.Repo.RepoLink + "/settings") } -func selectPushMirrorByForm(form *forms.RepoSettingForm, repo *repo_model.Repository) (*repo_model.PushMirror, error) { +func selectPushMirrorByForm(ctx *context.Context, form *forms.RepoSettingForm, repo *repo_model.Repository) (*repo_model.PushMirror, error) { id, err := strconv.ParseInt(form.PushMirrorID, 10, 64) if err != nil { return nil, err } - pushMirrors, err := repo_model.GetPushMirrorsByRepoID(repo.ID) + pushMirrors, _, err := repo_model.GetPushMirrorsByRepoID(ctx, repo.ID, db.ListOptions{}) if err != nil { return nil, err } diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 1a39b21c6ff80..c5657aa6756d2 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -15,7 +15,6 @@ import ( "net/http" "net/url" "path" - "strconv" "strings" "time" @@ -57,15 +56,6 @@ type namedBlob struct { blob *git.Blob } -func linesBytesCount(s []byte) int { - nl := []byte{'\n'} - n := bytes.Count(s, nl) - if len(s) > 0 && !bytes.HasSuffix(s, nl) { - n++ - } - return n -} - // FIXME: There has to be a more efficient way of doing this func getReadmeFileFromPath(commit *git.Commit, treePath string) (*namedBlob, error) { tree, err := commit.SubTree(treePath) @@ -555,8 +545,14 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ) } else { buf, _ := io.ReadAll(rd) - lineNums := linesBytesCount(buf) - ctx.Data["NumLines"] = strconv.Itoa(lineNums) + + // empty: 0 lines; "a": one line; "a\n": two lines; "a\nb": two lines; + // the NumLines is only used for the display on the UI: "xxx lines" + if len(buf) == 0 { + ctx.Data["NumLines"] = 0 + } else { + ctx.Data["NumLines"] = bytes.Count(buf, []byte{'\n'}) + 1 + } ctx.Data["NumLinesSet"] = true language := "" @@ -584,7 +580,11 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st language = "" } } - fileContent := highlight.File(lineNums, blob.Name(), language, buf) + fileContent, err := highlight.File(blob.Name(), language, buf) + if err != nil { + log.Error("highlight.File failed, fallback to plain text: %v", err) + fileContent = highlight.PlainText(buf) + } status, _ := charset.EscapeControlReader(bytes.NewReader(buf), io.Discard) ctx.Data["EscapeStatus"] = status statuses := make([]charset.EscapeStatus, len(fileContent)) diff --git a/routers/web/user/package.go b/routers/web/user/package.go index aa379152b30b1..59aaf07ff2a81 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -8,6 +8,7 @@ import ( "net/http" "code.gitea.io/gitea/models/db" + org_model "code.gitea.io/gitea/models/organization" packages_model "code.gitea.io/gitea/models/packages" container_model "code.gitea.io/gitea/models/packages/container" "code.gitea.io/gitea/models/perm" @@ -93,6 +94,21 @@ func ListPackages(ctx *context.Context) { ctx.Data["Total"] = total ctx.Data["RepositoryAccessMap"] = repositoryAccessMap + // TODO: context/org -> HandleOrgAssignment() can not be used + if ctx.ContextUser.IsOrganization() { + org := org_model.OrgFromUser(ctx.ContextUser) + ctx.Data["Org"] = org + ctx.Data["OrgLink"] = ctx.ContextUser.OrganisationLink() + + if ctx.Doer != nil { + ctx.Data["IsOrganizationMember"], _ = org_model.IsOrganizationMember(ctx, org.ID, ctx.Doer.ID) + ctx.Data["IsOrganizationOwner"], _ = org_model.IsOrganizationOwner(ctx, org.ID, ctx.Doer.ID) + } else { + ctx.Data["IsOrganizationMember"] = false + ctx.Data["IsOrganizationOwner"] = false + } + } + pager := context.NewPagination(int(total), setting.UI.PackagesPagingNum, page, 5) pager.AddParam(ctx, "q", "Query") pager.AddParam(ctx, "type", "PackageType") diff --git a/routers/web/web.go b/routers/web/web.go index b4e8666c44fd2..a9f43fb2c4feb 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -290,6 +290,13 @@ func RegisterRoutes(m *web.Route) { } } + dlSourceEnabled := func(ctx *context.Context) { + if setting.Repository.DisableDownloadSourceArchives { + ctx.Error(http.StatusNotFound) + return + } + } + // FIXME: not all routes need go through same middleware. // Especially some AJAX requests, we can reduce middleware number to improve performance. // Routers. @@ -1106,7 +1113,7 @@ func RegisterRoutes(m *web.Route) { m.Group("/archive", func() { m.Get("/*", repo.Download) m.Post("/*", repo.InitiateDownload) - }, repo.MustBeNotEmpty, reqRepoCodeReader) + }, repo.MustBeNotEmpty, dlSourceEnabled, reqRepoCodeReader) m.Group("/branches", func() { m.Get("", repo.Branches) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 8321829ad26ba..3b4a8e5f8a6d7 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -106,7 +106,7 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error { pushMirrorsRequested := 0 if pushLimit != 0 { - if err := repo_model.PushMirrorsIterate(pushLimit, func(idx int, bean interface{}) error { + if err := repo_model.PushMirrorsIterate(ctx, pushLimit, func(idx int, bean interface{}) error { if err := handler(idx, bean); err != nil { return err } diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 2927bed72b279..0c8960d78bf20 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -94,7 +94,7 @@ func SyncPushMirror(ctx context.Context, mirrorID int64) bool { log.Error("PANIC whilst syncPushMirror[%d] Panic: %v\nStacktrace: %s", mirrorID, err, log.Stack(2)) }() - m, err := repo_model.GetPushMirrorByID(mirrorID) + m, err := repo_model.GetPushMirror(ctx, repo_model.PushMirrorOptions{ID: mirrorID}) if err != nil { log.Error("GetPushMirrorByID [%d]: %v", mirrorID, err) return false @@ -116,7 +116,7 @@ func SyncPushMirror(ctx context.Context, mirrorID int64) bool { m.LastUpdateUnix = timeutil.TimeStampNow() - if err := repo_model.UpdatePushMirror(m); err != nil { + if err := repo_model.UpdatePushMirror(ctx, m); err != nil { log.Error("UpdatePushMirror [%d]: %v", m.ID, err) return false diff --git a/templates/mail/release.tmpl b/templates/mail/release.tmpl index a95647105ca50..931c0d1b541e5 100644 --- a/templates/mail/release.tmpl +++ b/templates/mail/release.tmpl @@ -31,12 +31,14 @@
{{.locale.Tr "mail.release.downloads"}}
    + {{if not .DisableDownloadSourceArchives}}
  • {{.locale.Tr "mail.release.download.zip"}}
  • {{.locale.Tr "mail.release.download.targz"}}
    • + {{end}} {{if .Release.Attachments}} {{range .Release.Attachments}}
  • diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl index 82b88a00ba799..4a85692a83151 100644 --- a/templates/repo/branch/list.tmpl +++ b/templates/repo/branch/list.tmpl @@ -26,13 +26,15 @@ {{svg "octicon-git-branch"}} {{end}} -
    - {{svg "octicon-download"}} -
    - {{svg "octicon-file-zip"}} ZIP - {{svg "octicon-file-zip"}} TAR.GZ + {{if not $.DisableDownloadSourceArchives}} +
    + {{svg "octicon-download"}} +
    + {{svg "octicon-file-zip"}} ZIP + {{svg "octicon-file-zip"}} TAR.GZ +
    -
    + {{end}} diff --git a/templates/repo/clone_buttons.tmpl b/templates/repo/clone_buttons.tmpl index fb54b27c820aa..67a509835f4f6 100644 --- a/templates/repo/clone_buttons.tmpl +++ b/templates/repo/clone_buttons.tmpl @@ -9,16 +9,7 @@ SSH {{end}} - - - + diff --git a/templates/repo/clone_script.tmpl b/templates/repo/clone_script.tmpl new file mode 100644 index 0000000000000..48b7ad3299ca2 --- /dev/null +++ b/templates/repo/clone_script.tmpl @@ -0,0 +1,23 @@ + diff --git a/templates/repo/empty.tmpl b/templates/repo/empty.tmpl index 730284da20276..99f662cd0bb93 100644 --- a/templates/repo/empty.tmpl +++ b/templates/repo/empty.tmpl @@ -37,7 +37,7 @@ git init {{if ne .Repository.DefaultBranch "master"}}git checkout -b {{.Repository.DefaultBranch}}{{end}} git add README.md git commit -m "first commit" -git remote add origin +git remote add origin git push -u origin {{.Repository.DefaultBranch}}
    @@ -46,24 +46,11 @@ git push -u origin {{.Repository.DefaultBranch}}

    {{.locale.Tr "repo.push_exist_repo"}}

    - 
    git remote add origin 
+									
    git remote add origin 
 git push -u origin {{.Repository.DefaultBranch}}
    - - + {{template "repo/clone_script" .}} {{end}} {{else}}
    diff --git a/templates/repo/home.tmpl b/templates/repo/home.tmpl index 5e1af447a49a3..0734076eff490 100644 --- a/templates/repo/home.tmpl +++ b/templates/repo/home.tmpl @@ -124,6 +124,8 @@ {{if eq $n 0}}
    {{template "repo/clone_buttons" .}} + {{template "repo/clone_script" .}} + {{if not .DisableDownloadSourceArchives}} + {{end}}
    {{end}}
    diff --git a/templates/repo/issue/view_content/pull.tmpl b/templates/repo/issue/view_content/pull.tmpl index be2c8f0c0ef4c..59e0962d1795d 100644 --- a/templates/repo/issue/view_content/pull.tmpl +++ b/templates/repo/issue/view_content/pull.tmpl @@ -357,6 +357,7 @@ 'emptyCommit': {{.Issue.PullRequest.IsEmpty}}, 'pullHeadCommitID': {{.PullHeadCommitID}}, 'isPullBranchDeletable': {{.IsPullBranchDeletable}}, + 'defaultMergeStyle': {{.MergeStyle}}, 'defaultDeleteBranchAfterMerge': {{$prUnit.PullRequestsConfig.DefaultDeleteBranchAfterMerge}}, 'mergeMessageFieldPlaceHolder': {{$.locale.Tr "repo.editor.commit_message_desc"}}, diff --git a/templates/repo/release/list.tmpl b/templates/repo/release/list.tmpl index 58869d470c09c..4eb099129af19 100644 --- a/templates/repo/release/list.tmpl +++ b/templates/repo/release/list.tmpl @@ -37,8 +37,10 @@
    {{if $.Permission.CanRead $.UnitTypeCode}} {{svg "octicon-git-commit" 16 "mr-2"}}{{ShortSha .Sha1}} - {{svg "octicon-file-zip" 16 "mr-2"}}ZIP - {{svg "octicon-file-zip" 16 "mr-2"}}TAR.GZ + {{if not $.DisableDownloadSourceArchives}} + {{svg "octicon-file-zip" 16 "mr-2"}}ZIP + {{svg "octicon-file-zip" 16 "mr-2"}}TAR.GZ + {{end}} {{if (and $.CanCreateRelease $release.IsTag)}} {{svg "octicon-tag" 16 "mr-2"}}{{$.locale.Tr "repo.release.new_release"}} {{end}} @@ -104,8 +106,10 @@
    {{if $.Permission.CanRead $.UnitTypeCode}} {{svg "octicon-git-commit" 16 "mr-2"}}{{ShortSha .Sha1}} - {{svg "octicon-file-zip"}} ZIP - {{svg "octicon-file-zip"}} TAR.GZ + {{if not $.DisableDownloadSourceArchives}} + {{svg "octicon-file-zip"}} ZIP + {{svg "octicon-file-zip"}} TAR.GZ + {{end}} {{end}}
    {{else}} @@ -146,7 +150,7 @@ {{$.locale.Tr "repo.release.downloads"}}
      - {{if and (not .IsDraft) ($.Permission.CanRead $.UnitTypeCode)}} + {{if and (not $.DisableDownloadSourceArchives) (not .IsDraft) ($.Permission.CanRead $.UnitTypeCode)}}
    • {{svg "octicon-file-zip" 16 "mr-2"}}{{$.locale.Tr "repo.release.source_code"}} (ZIP)
      • diff --git a/templates/repo/wiki/revision.tmpl b/templates/repo/wiki/revision.tmpl index 577eb6e6c0dda..71b69220c6006 100644 --- a/templates/repo/wiki/revision.tmpl +++ b/templates/repo/wiki/revision.tmpl @@ -7,6 +7,7 @@
      {{template "repo/clone_buttons" .}} + {{template "repo/clone_script" .}}
      diff --git a/templates/repo/wiki/view.tmpl b/templates/repo/wiki/view.tmpl index 91a15f19e2e35..21affddc623e2 100644 --- a/templates/repo/wiki/view.tmpl +++ b/templates/repo/wiki/view.tmpl @@ -31,6 +31,7 @@
      {{template "repo/clone_buttons" .}} + {{template "repo/clone_script" .}}
      diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index c0dd50a7b9e18..daec38044f9f0 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -8774,6 +8774,233 @@ } } }, + "/repos/{owner}/{repo}/push_mirrors": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "repository" + ], + "summary": "Get all push mirrors of the repository", + "operationId": "repoListPushMirrors", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "page number of results to return (1-based)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "page size of results", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "$ref": "#/responses/PushMirrorList" + }, + "400": { + "$ref": "#/responses/error" + }, + "403": { + "$ref": "#/responses/forbidden" + } + } + }, + "post": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "repository" + ], + "summary": "add a push mirror to the repository", + "operationId": "repoAddPushMirror", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "schema": { + "$ref": "#/definitions/CreatePushMirrorOption" + } + } + ], + "responses": { + "201": { + "$ref": "#/responses/PushMirror" + }, + "400": { + "$ref": "#/responses/error" + }, + "403": { + "$ref": "#/responses/forbidden" + } + } + } + }, + "/repos/{owner}/{repo}/push_mirrors-sync": { + "post": { + "produces": [ + "application/json" + ], + "tags": [ + "repository" + ], + "summary": "Sync all push mirrored repository", + "operationId": "repoPushMirrorSync", + "parameters": [ + { + "type": "string", + "description": "owner of the repo to sync", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo to sync", + "name": "repo", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "$ref": "#/responses/empty" + }, + "400": { + "$ref": "#/responses/error" + }, + "403": { + "$ref": "#/responses/forbidden" + } + } + } + }, + "/repos/{owner}/{repo}/push_mirrors/{name}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "repository" + ], + "summary": "Get push mirror of the repository by remoteName", + "operationId": "repoGetPushMirrorByRemoteName", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "remote name of push mirror", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "$ref": "#/responses/PushMirror" + }, + "400": { + "$ref": "#/responses/error" + }, + "403": { + "$ref": "#/responses/forbidden" + } + } + }, + "delete": { + "produces": [ + "application/json" + ], + "tags": [ + "repository" + ], + "summary": "deletes a push mirror from a repository by remoteName", + "operationId": "repoDeletePushMirror", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "remote name of the pushMirror", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "$ref": "#/responses/empty" + }, + "400": { + "$ref": "#/responses/error" + }, + "404": { + "$ref": "#/responses/notFound" + } + } + } + }, "/repos/{owner}/{repo}/raw/{filepath}": { "get": { "produces": [ @@ -14441,6 +14668,29 @@ }, "x-go-package": "code.gitea.io/gitea/modules/structs" }, + "CreatePushMirrorOption": { + "type": "object", + "title": "CreatePushMirrorOption represents need information to create a push mirror of a repository.", + "properties": { + "interval": { + "type": "string", + "x-go-name": "Interval" + }, + "remote_address": { + "type": "string", + "x-go-name": "RemoteAddress" + }, + "remote_password": { + "type": "string", + "x-go-name": "RemotePassword" + }, + "remote_username": { + "type": "string", + "x-go-name": "RemoteUsername" + } + }, + "x-go-package": "code.gitea.io/gitea/modules/structs" + }, "CreateReleaseOption": { "description": "CreateReleaseOption options when creating a release", "type": "object", @@ -17516,6 +17766,41 @@ }, "x-go-package": "code.gitea.io/gitea/modules/structs" }, + "PushMirror": { + "description": "PushMirror represents information of a push mirror", + "type": "object", + "properties": { + "created": { + "type": "string", + "x-go-name": "CreatedUnix" + }, + "interval": { + "type": "string", + "x-go-name": "Interval" + }, + "last_error": { + "type": "string", + "x-go-name": "LastError" + }, + "last_update": { + "type": "string", + "x-go-name": "LastUpdateUnix" + }, + "remote_address": { + "type": "string", + "x-go-name": "RemoteAddress" + }, + "remote_name": { + "type": "string", + "x-go-name": "RemoteName" + }, + "repo_name": { + "type": "string", + "x-go-name": "RepoName" + } + }, + "x-go-package": "code.gitea.io/gitea/modules/structs" + }, "Reaction": { "description": "Reaction contain one reaction", "type": "object", @@ -19293,6 +19578,21 @@ } } }, + "PushMirror": { + "description": "PushMirror", + "schema": { + "$ref": "#/definitions/PushMirror" + } + }, + "PushMirrorList": { + "description": "PushMirrorList", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/PushMirror" + } + } + }, "Reaction": { "description": "Reaction", "schema": { @@ -19572,7 +19872,7 @@ "parameterBodies": { "description": "parameterBodies", "schema": { - "$ref": "#/definitions/CreateWikiPageOptions" + "$ref": "#/definitions/CreatePushMirrorOption" } }, "redirect": { diff --git a/templates/user/overview/header.tmpl b/templates/user/overview/header.tmpl index 8f9ad1d6ccfba..edd4375e08c39 100644 --- a/templates/user/overview/header.tmpl +++ b/templates/user/overview/header.tmpl @@ -1,14 +1,22 @@
      -
      -
      -
      -
      - {{avatar .ContextUser 32}} - {{.ContextUser.Name}} + + {{with .ContextUser}} +
      +
      +
      +
      + {{avatar . 100}} + {{.DisplayName}} + + {{if .Visibility.IsLimited}}
      {{$.locale.Tr "org.settings.visibility.limited_shortname"}}
      {{end}} + {{if .Visibility.IsPrivate}}
      {{$.locale.Tr "org.settings.visibility.private_shortname"}}
      {{end}} +       +
      -
      + {{end}} +
      @@ -19,6 +27,32 @@ {{svg "octicon-package"}} {{.locale.Tr "packages.title"}} {{end}} + + {{if .ContextUser.IsOrganization}} + {{if .IsOrganizationMember}} + + {{svg "octicon-organization"}} {{$.locale.Tr "org.people"}} + {{if .NumMembers}} +
      {{.NumMembers}}
      + {{end}} +       + + {{svg "octicon-people"}} {{$.locale.Tr "org.teams"}} + {{if .NumTeams}} +
      {{.NumTeams}}
      + {{end}} +       + {{end}} + + {{if .IsOrganizationOwner}} +
      + + {{svg "octicon-tools"}} {{.locale.Tr "repo.settings"}} + +
      + {{end}} + {{end}} +
      diff --git a/web_src/js/components/PullRequestMergeForm.vue b/web_src/js/components/PullRequestMergeForm.vue index 08b1f9cb86e90..64ba98728e6a2 100644 --- a/web_src/js/components/PullRequestMergeForm.vue +++ b/web_src/js/components/PullRequestMergeForm.vue @@ -145,7 +145,10 @@ export default { created() { this.mergeStyleAllowedCount = this.mergeForm.mergeStyles.reduce((v, msd) => v + (msd.allowed ? 1 : 0), 0); - this.switchMergeStyle(this.mergeForm.mergeStyles.find((e) => e.allowed)?.name, !this.mergeForm.canMergeNow); + + let mergeStyle = this.mergeForm.mergeStyles.find((e) => e.allowed && e.name === this.mergeForm.defaultMergeStyle)?.name; + if (!mergeStyle) mergeStyle = this.mergeForm.mergeStyles.find((e) => e.allowed)?.name; + this.switchMergeStyle(mergeStyle, !this.mergeForm.canMergeNow); }, mounted() { diff --git a/web_src/js/features/repo-common.js b/web_src/js/features/repo-common.js index b0ecc5a651275..1cf67401be371 100644 --- a/web_src/js/features/repo-common.js +++ b/web_src/js/features/repo-common.js @@ -44,8 +44,6 @@ export function initRepoArchiveLinks() { } export function initRepoCloneLink() { - const defaultGitProtocol = 'https'; // ssh or https - const $repoCloneSsh = $('#repo-clone-ssh'); const $repoCloneHttps = $('#repo-clone-https'); const $inputLink = $('#repo-clone-url'); @@ -54,41 +52,19 @@ export function initRepoCloneLink() { return; } - const updateUi = () => { - let isSSH = (localStorage.getItem('repo-clone-protocol') || defaultGitProtocol) === 'ssh'; - // there must be at least one clone button (by context/repo.go). if no ssh, then there must be https. - if (isSSH && $repoCloneSsh.length === 0) { - isSSH = false; - } else if (!isSSH && $repoCloneHttps.length === 0) { - isSSH = true; - } - const cloneLink = (isSSH ? $repoCloneSsh : $repoCloneHttps).attr('data-link'); - $inputLink.val(cloneLink); - if (isSSH) { - $repoCloneSsh.addClass('primary'); - $repoCloneHttps.removeClass('primary'); - } else { - $repoCloneSsh.removeClass('primary'); - $repoCloneHttps.addClass('primary'); - } - // the empty repo guide - $('.quickstart .empty-repo-guide .clone-url').text(cloneLink); - }; - updateUi(); - + // restore animation after first init setTimeout(() => { - // restore animation after first init $repoCloneSsh.removeClass('no-transition'); $repoCloneHttps.removeClass('no-transition'); }, 100); $repoCloneSsh.on('click', () => { localStorage.setItem('repo-clone-protocol', 'ssh'); - updateUi(); + window.updateCloneStates(); }); $repoCloneHttps.on('click', () => { localStorage.setItem('repo-clone-protocol', 'https'); - updateUi(); + window.updateCloneStates(); }); $inputLink.on('click', () => {