From c304241121fc42d072b42cd4c088dda87fe2f6e8 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 21 Feb 2023 23:28:54 +0000 Subject: [PATCH 1/2] Use minio/sha256-simd for accelerated SHA256 minio/sha256-simd provides additional acceleration for SHA256 using AVX512, SHA Extensions for x86 and ARM64 for ARM. It provides a drop-in replacement for crypto/sha256 and if the extensions are not available it falls back to standard crypto/sha256. Signed-off-by: Andrew Thornton --- go.mod | 2 +- models/auth/oauth2.go | 2 +- models/auth/twofactor.go | 2 +- models/migrations/base/hash.go | 2 +- models/migrations/v1_14/v166.go | 2 +- modules/auth/password/hash/pbkdf2.go | 2 +- modules/avatar/hash.go | 3 ++- modules/avatar/identicon/identicon.go | 3 ++- modules/base/tool.go | 2 +- modules/context/context.go | 2 +- modules/git/last_commit_cache.go | 3 ++- modules/lfs/content_store.go | 3 ++- modules/lfs/pointer.go | 3 ++- modules/packages/multi_hasher.go | 3 ++- modules/secret/secret.go | 3 ++- modules/util/keypair_test.go | 2 +- routers/api/packages/chef/auth.go | 3 ++- routers/api/packages/maven/maven.go | 3 ++- services/auth/source/oauth2/jwtsigningkey.go | 2 +- services/lfs/server.go | 2 +- services/mailer/token/token.go | 3 ++- services/webhook/deliver.go | 2 +- tests/integration/api_packages_chef_test.go | 2 +- tests/integration/api_packages_container_test.go | 2 +- tests/integration/api_packages_test.go | 2 +- 25 files changed, 35 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 343a70da25a02..f003d444b403d 100644 --- a/go.mod +++ b/go.mod @@ -76,6 +76,7 @@ require ( github.com/mholt/archiver/v3 v3.5.1 github.com/microcosm-cc/bluemonday v1.0.21 github.com/minio/minio-go/v7 v7.0.46 + github.com/minio/sha256-simd v1.0.0 github.com/msteinert/pam v1.1.0 github.com/nektos/act v0.0.0 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 @@ -220,7 +221,6 @@ require ( github.com/mholt/acmez v1.0.4 // indirect github.com/miekg/dns v1.1.50 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect diff --git a/models/auth/oauth2.go b/models/auth/oauth2.go index 09d4bfc4ea35e..bda0668c45f55 100644 --- a/models/auth/oauth2.go +++ b/models/auth/oauth2.go @@ -5,7 +5,6 @@ package auth import ( "context" - "crypto/sha256" "encoding/base32" "encoding/base64" "fmt" @@ -18,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/util" uuid "github.com/google/uuid" + "github.com/minio/sha256-simd" "golang.org/x/crypto/bcrypt" "xorm.io/builder" "xorm.io/xorm" diff --git a/models/auth/twofactor.go b/models/auth/twofactor.go index 5b3a9d011a099..751a281f7e8b9 100644 --- a/models/auth/twofactor.go +++ b/models/auth/twofactor.go @@ -5,7 +5,6 @@ package auth import ( "crypto/md5" - "crypto/sha256" "crypto/subtle" "encoding/base32" "encoding/base64" @@ -18,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" + "github.com/minio/sha256-simd" "github.com/pquerna/otp/totp" "golang.org/x/crypto/pbkdf2" ) diff --git a/models/migrations/base/hash.go b/models/migrations/base/hash.go index 00fd1efd4a3ba..0debec272b3f3 100644 --- a/models/migrations/base/hash.go +++ b/models/migrations/base/hash.go @@ -4,9 +4,9 @@ package base import ( - "crypto/sha256" "encoding/hex" + "github.com/minio/sha256-simd" "golang.org/x/crypto/pbkdf2" ) diff --git a/models/migrations/v1_14/v166.go b/models/migrations/v1_14/v166.go index f797930d6d5bc..de7626076a95d 100644 --- a/models/migrations/v1_14/v166.go +++ b/models/migrations/v1_14/v166.go @@ -4,9 +4,9 @@ package v1_14 //nolint import ( - "crypto/sha256" "encoding/hex" + "github.com/minio/sha256-simd" "golang.org/x/crypto/argon2" "golang.org/x/crypto/bcrypt" "golang.org/x/crypto/pbkdf2" diff --git a/modules/auth/password/hash/pbkdf2.go b/modules/auth/password/hash/pbkdf2.go index 27382fedb8cb0..9ff6d162fcc39 100644 --- a/modules/auth/password/hash/pbkdf2.go +++ b/modules/auth/password/hash/pbkdf2.go @@ -4,12 +4,12 @@ package hash import ( - "crypto/sha256" "encoding/hex" "strings" "code.gitea.io/gitea/modules/log" + "github.com/minio/sha256-simd" "golang.org/x/crypto/pbkdf2" ) diff --git a/modules/avatar/hash.go b/modules/avatar/hash.go index 50db9c1943a30..4fc28a7739f0b 100644 --- a/modules/avatar/hash.go +++ b/modules/avatar/hash.go @@ -4,9 +4,10 @@ package avatar import ( - "crypto/sha256" "encoding/hex" "strconv" + + "github.com/minio/sha256-simd" ) // HashAvatar will generate a unique string, which ensures that when there's a diff --git a/modules/avatar/identicon/identicon.go b/modules/avatar/identicon/identicon.go index 63926d5f19086..9b7a2faf05b50 100644 --- a/modules/avatar/identicon/identicon.go +++ b/modules/avatar/identicon/identicon.go @@ -7,10 +7,11 @@ package identicon import ( - "crypto/sha256" "fmt" "image" "image/color" + + "github.com/minio/sha256-simd" ) const minImageSize = 16 diff --git a/modules/base/tool.go b/modules/base/tool.go index 994e58ac3cd3e..94f19576b48d7 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -6,7 +6,6 @@ package base import ( "crypto/md5" "crypto/sha1" - "crypto/sha256" "encoding/base64" "encoding/hex" "errors" @@ -26,6 +25,7 @@ import ( "code.gitea.io/gitea/modules/util" "github.com/dustin/go-humanize" + "github.com/minio/sha256-simd" ) // EncodeMD5 encodes string to md5 hex value. diff --git a/modules/context/context.go b/modules/context/context.go index a2088217ff387..0c8d7411ed5db 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -6,7 +6,6 @@ package context import ( "context" - "crypto/sha256" "encoding/hex" "errors" "fmt" @@ -40,6 +39,7 @@ import ( "gitea.com/go-chi/cache" "gitea.com/go-chi/session" chi "github.com/go-chi/chi/v5" + "github.com/minio/sha256-simd" "github.com/unrolled/render" "golang.org/x/crypto/pbkdf2" ) diff --git a/modules/git/last_commit_cache.go b/modules/git/last_commit_cache.go index ec8f1cce62fcc..984561b2c6729 100644 --- a/modules/git/last_commit_cache.go +++ b/modules/git/last_commit_cache.go @@ -4,11 +4,12 @@ package git import ( - "crypto/sha256" "fmt" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + + "github.com/minio/sha256-simd" ) // Cache represents a caching interface diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index 94277a6b8eb3b..a4ae21bfd65c6 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -4,7 +4,6 @@ package lfs import ( - "crypto/sha256" "encoding/hex" "errors" "hash" @@ -13,6 +12,8 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" + + "github.com/minio/sha256-simd" ) var ( diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index b5e13d56a3b36..f7f225bf1c1e7 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -4,7 +4,6 @@ package lfs import ( - "crypto/sha256" "encoding/hex" "errors" "fmt" @@ -15,6 +14,8 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + + "github.com/minio/sha256-simd" ) const ( diff --git a/modules/packages/multi_hasher.go b/modules/packages/multi_hasher.go index 83a4b5b7afd64..49fd1942880fa 100644 --- a/modules/packages/multi_hasher.go +++ b/modules/packages/multi_hasher.go @@ -6,12 +6,13 @@ package packages import ( "crypto/md5" "crypto/sha1" - "crypto/sha256" "crypto/sha512" "encoding" "errors" "hash" "io" + + "github.com/minio/sha256-simd" ) const ( diff --git a/modules/secret/secret.go b/modules/secret/secret.go index b84d1cfea877e..628ae505a5c0b 100644 --- a/modules/secret/secret.go +++ b/modules/secret/secret.go @@ -7,11 +7,12 @@ import ( "crypto/aes" "crypto/cipher" "crypto/rand" - "crypto/sha256" "encoding/base64" "encoding/hex" "errors" "io" + + "github.com/minio/sha256-simd" ) // AesEncrypt encrypts text and given key with AES. diff --git a/modules/util/keypair_test.go b/modules/util/keypair_test.go index c6f68c845a47a..c9925f7988103 100644 --- a/modules/util/keypair_test.go +++ b/modules/util/keypair_test.go @@ -7,12 +7,12 @@ import ( "crypto" "crypto/rand" "crypto/rsa" - "crypto/sha256" "crypto/x509" "encoding/pem" "regexp" "testing" + "github.com/minio/sha256-simd" "github.com/stretchr/testify/assert" ) diff --git a/routers/api/packages/chef/auth.go b/routers/api/packages/chef/auth.go index 69f7b763ab66c..1ea453f1f4d13 100644 --- a/routers/api/packages/chef/auth.go +++ b/routers/api/packages/chef/auth.go @@ -7,7 +7,6 @@ import ( "crypto" "crypto/rsa" "crypto/sha1" - "crypto/sha256" "crypto/x509" "encoding/base64" "encoding/pem" @@ -25,6 +24,8 @@ import ( chef_module "code.gitea.io/gitea/modules/packages/chef" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/auth" + + "github.com/minio/sha256-simd" ) const ( diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go index d0c9983cbf1f2..a3a23ecfa8dc9 100644 --- a/routers/api/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -6,7 +6,6 @@ package maven import ( "crypto/md5" "crypto/sha1" - "crypto/sha256" "crypto/sha512" "encoding/hex" "encoding/xml" @@ -27,6 +26,8 @@ import ( maven_module "code.gitea.io/gitea/modules/packages/maven" "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" + + "github.com/minio/sha256-simd" ) const ( diff --git a/services/auth/source/oauth2/jwtsigningkey.go b/services/auth/source/oauth2/jwtsigningkey.go index 93c15743798c2..94feddbf6b6a5 100644 --- a/services/auth/source/oauth2/jwtsigningkey.go +++ b/services/auth/source/oauth2/jwtsigningkey.go @@ -9,7 +9,6 @@ import ( "crypto/elliptic" "crypto/rand" "crypto/rsa" - "crypto/sha256" "crypto/x509" "encoding/base64" "encoding/pem" @@ -25,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/util" "github.com/golang-jwt/jwt/v4" + "github.com/minio/sha256-simd" ini "gopkg.in/ini.v1" ) diff --git a/services/lfs/server.go b/services/lfs/server.go index 320c8e728116d..217d45124e391 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -5,7 +5,6 @@ package lfs import ( stdCtx "context" - "crypto/sha256" "encoding/base64" "encoding/hex" "errors" @@ -32,6 +31,7 @@ import ( "code.gitea.io/gitea/modules/storage" "github.com/golang-jwt/jwt/v4" + "github.com/minio/sha256-simd" ) // requestContext contain variables from the HTTP request. diff --git a/services/mailer/token/token.go b/services/mailer/token/token.go index 8a5a762d6b5fd..aa7b56718863b 100644 --- a/services/mailer/token/token.go +++ b/services/mailer/token/token.go @@ -6,13 +6,14 @@ package token import ( "context" crypto_hmac "crypto/hmac" - "crypto/sha256" "encoding/base32" "fmt" "time" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/util" + + "github.com/minio/sha256-simd" ) // A token is a verifiable container describing an action. diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index effbe45e56910..e389b1f9fe670 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -7,7 +7,6 @@ import ( "context" "crypto/hmac" "crypto/sha1" - "crypto/sha256" "crypto/tls" "encoding/hex" "fmt" @@ -29,6 +28,7 @@ import ( webhook_module "code.gitea.io/gitea/modules/webhook" "github.com/gobwas/glob" + "github.com/minio/sha256-simd" ) // Deliver deliver hook task diff --git a/tests/integration/api_packages_chef_test.go b/tests/integration/api_packages_chef_test.go index 14baddca94280..0ee43e1741860 100644 --- a/tests/integration/api_packages_chef_test.go +++ b/tests/integration/api_packages_chef_test.go @@ -11,7 +11,6 @@ import ( "crypto/rand" "crypto/rsa" "crypto/sha1" - "crypto/sha256" "crypto/x509" "encoding/base64" "encoding/pem" @@ -34,6 +33,7 @@ import ( chef_router "code.gitea.io/gitea/routers/api/packages/chef" "code.gitea.io/gitea/tests" + "github.com/minio/sha256-simd" "github.com/stretchr/testify/assert" ) diff --git a/tests/integration/api_packages_container_test.go b/tests/integration/api_packages_container_test.go index 3d9319f37001c..bbab820ecbbc1 100644 --- a/tests/integration/api_packages_container_test.go +++ b/tests/integration/api_packages_container_test.go @@ -5,7 +5,6 @@ package integration import ( "bytes" - "crypto/sha256" "encoding/base64" "fmt" "net/http" @@ -24,6 +23,7 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/tests" + "github.com/minio/sha256-simd" oci "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" ) diff --git a/tests/integration/api_packages_test.go b/tests/integration/api_packages_test.go index 39852e212c387..4a16cec015b8b 100644 --- a/tests/integration/api_packages_test.go +++ b/tests/integration/api_packages_test.go @@ -5,7 +5,6 @@ package integration import ( "bytes" - "crypto/sha256" "fmt" "net/http" "strings" @@ -24,6 +23,7 @@ import ( packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup" "code.gitea.io/gitea/tests" + "github.com/minio/sha256-simd" "github.com/stretchr/testify/assert" ) From 4cb27fe32bb4a727fec73aa5c681ea00a81dd4f9 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 22 Feb 2023 08:35:31 +0000 Subject: [PATCH 2/2] handle lack of binaryunmarshal support Signed-off-by: Andrew Thornton --- modules/packages/multi_hasher.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/packages/multi_hasher.go b/modules/packages/multi_hasher.go index 49fd1942880fa..83a4b5b7afd64 100644 --- a/modules/packages/multi_hasher.go +++ b/modules/packages/multi_hasher.go @@ -6,13 +6,12 @@ package packages import ( "crypto/md5" "crypto/sha1" + "crypto/sha256" "crypto/sha512" "encoding" "errors" "hash" "io" - - "github.com/minio/sha256-simd" ) const (