diff --git a/go.mod b/go.mod index 708c791c28..66e63dd7f7 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 github.com/CortexFoundation/inference v1.0.2-0.20230307032835-9197d586a4e8 github.com/CortexFoundation/statik v0.0.0-20210315012922-8bb8a7b5dc66 - github.com/CortexFoundation/torrentfs v1.0.56-0.20231209192839-e28c596da6b3 + github.com/CortexFoundation/torrentfs v1.0.56-0.20231212120813-41a5945e7136 github.com/VictoriaMetrics/fastcache v1.12.2 github.com/arsham/figurine v1.3.0 github.com/aws/aws-sdk-go-v2 v1.24.0 @@ -17,7 +17,7 @@ require ( github.com/cespare/cp v1.1.1 github.com/charmbracelet/bubbletea v0.24.2 github.com/cloudflare/cloudflare-go v0.57.1 - github.com/cockroachdb/pebble v0.0.0-20231208175357-58bdc725addc + github.com/cockroachdb/pebble v0.0.0-20231211223132-9848bcdb0fe9 github.com/consensys/gnark-crypto v0.12.1 github.com/crate-crypto/go-kzg-4844 v0.7.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc @@ -76,7 +76,7 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/CortexFoundation/cvm-runtime v0.0.0-20221117094012-b5a251885572 // indirect github.com/CortexFoundation/merkletree v0.0.0-20230724124840-b6e80265a137 // indirect @@ -158,7 +158,7 @@ require ( github.com/google/btree v1.1.2 // indirect github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08 // indirect + github.com/google/pprof v0.0.0-20231212022811-ec68065c825e // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect @@ -178,7 +178,7 @@ require ( github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect - github.com/nutsdb/nutsdb v1.0.2-0.20231209035901-f7c4a82b75c2 // indirect + github.com/nutsdb/nutsdb v1.0.2-0.20231211145816-3468fe949461 // indirect github.com/nxadm/tail v1.4.11 // indirect github.com/oapi-codegen/runtime v1.1.0 // indirect github.com/otiai10/copy v1.14.0 // indirect @@ -221,7 +221,7 @@ require ( github.com/tklauser/numcpus v0.7.0 // indirect github.com/ucwong/filecache v1.0.6-0.20230405163841-810d53ced4bd // indirect github.com/ucwong/go-ttlmap v1.0.2-0.20221020173635-331e7ddde2bb // indirect - github.com/ucwong/golang-kv v1.0.23-0.20231209094038-056d175d4a78 // indirect + github.com/ucwong/golang-kv v1.0.23-0.20231212115239-d45c3e24ca4a // indirect github.com/ucwong/shard v1.0.1-0.20230924231639-2ac2d8ab288c // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect @@ -238,7 +238,7 @@ require ( golang.org/x/net v0.19.0 // indirect golang.org/x/term v0.15.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - modernc.org/libc v1.37.0 // indirect + modernc.org/libc v1.37.1 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.7.2 // indirect modernc.org/sqlite v1.27.0 // indirect diff --git a/go.sum b/go.sum index fa5315df94..86b4fc3cf3 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,8 @@ filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 h1:fb8kj/Dh4CSwgsOzHeZY4Xh68cFVbzXx+ONXGMY//4w= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0/go.mod h1:uReU2sSxZExRPBAg3qKzmAucSi51+SP1OhohieR821Q= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0 h1:Yoicul8bnVdQrhDMTHxdEckRGX01XvwXDHUT9zYZ3k0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= @@ -66,8 +66,8 @@ github.com/CortexFoundation/statik v0.0.0-20210315012922-8bb8a7b5dc66/go.mod h1: github.com/CortexFoundation/torrentfs v1.0.13-0.20200623060705-ce027f43f2f8/go.mod h1:Ma+tGhPPvz4CEZHaqEJQMOEGOfHeQBiAoNd1zyc/w3Q= github.com/CortexFoundation/torrentfs v1.0.14-0.20200703071639-3fcabcabf274/go.mod h1:qnb3YlIJmuetVBtC6Lsejr0Xru+1DNmDCdTqnwy7lhk= github.com/CortexFoundation/torrentfs v1.0.20-0.20200810031954-d36d26f82fcc/go.mod h1:N5BsicP5ynjXIi/Npl/SRzlJ630n1PJV2sRj0Z0t2HA= -github.com/CortexFoundation/torrentfs v1.0.56-0.20231209192839-e28c596da6b3 h1:DZnM/6IKv2dAMmahHcIagNzBdNYhgRQgxMTM4+9L+aA= -github.com/CortexFoundation/torrentfs v1.0.56-0.20231209192839-e28c596da6b3/go.mod h1:duBTPeJdG+vf/T1XWtk9uAmik0hLPAoIFdQ0LO6lEcc= +github.com/CortexFoundation/torrentfs v1.0.56-0.20231212120813-41a5945e7136 h1:jUvkxoBxo5ikPcpsnTylh5Um1M3EW0zwPIZetbnstrw= +github.com/CortexFoundation/torrentfs v1.0.56-0.20231212120813-41a5945e7136/go.mod h1:OoK5o2uSQbVgHxuJ2Lu8B2exPAnX3wLqSV78VCzQc3k= github.com/CortexFoundation/wormhole v0.0.2-0.20231018202213-693acd0cc941 h1:2lhbBGb0Q/v2Uk2P1qGWK//jmrZUGROZzME8psO9QNA= github.com/CortexFoundation/wormhole v0.0.2-0.20231018202213-693acd0cc941/go.mod h1:ipzmPabDgzYKUbXkGVe2gTkBEp+MsDx6pXGiuYzmP6s= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -355,8 +355,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble v0.0.0-20231208175357-58bdc725addc h1:+u38Jdm52Efr8nHmctIkIoFPfUzRIqliJWGjHaR1K/0= -github.com/cockroachdb/pebble v0.0.0-20231208175357-58bdc725addc/go.mod h1:BHuaMa/lK7fUe75BlsteiiTu8ptIG+qSAuDtGMArP18= +github.com/cockroachdb/pebble v0.0.0-20231211223132-9848bcdb0fe9 h1:f3eH58jJkWd70J3E6BNQPNjMEorA9l72dLBJCoX/UEc= +github.com/cockroachdb/pebble v0.0.0-20231211223132-9848bcdb0fe9/go.mod h1:BHuaMa/lK7fUe75BlsteiiTu8ptIG+qSAuDtGMArP18= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= @@ -617,8 +617,8 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= -github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08 h1:PxlBVtIFHR/mtWk2i0gTEdCz+jBnqiuHNSki0epDbVs= -github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20231212022811-ec68065c825e h1:bwOy7hAFd0C91URzMIEBfr6BAz29yk7Qj0cy6S7DJlU= +github.com/google/pprof v0.0.0-20231212022811-ec68065c825e/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -895,8 +895,8 @@ github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= -github.com/nutsdb/nutsdb v1.0.2-0.20231209035901-f7c4a82b75c2 h1:dQkZuf4gSZYpseqBViypj5SnqqmWehx5TdQxqdNRO9Y= -github.com/nutsdb/nutsdb v1.0.2-0.20231209035901-f7c4a82b75c2/go.mod h1:jIbbpBXajzTMZ0o33Yn5zoYIo3v0Dz4WstkVce+sYuQ= +github.com/nutsdb/nutsdb v1.0.2-0.20231211145816-3468fe949461 h1:wb5O82PuENmF1H0+Mv3IUYY66YS5z0cPgCSX/PNbSqE= +github.com/nutsdb/nutsdb v1.0.2-0.20231211145816-3468fe949461/go.mod h1:jIbbpBXajzTMZ0o33Yn5zoYIo3v0Dz4WstkVce+sYuQ= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= @@ -1250,8 +1250,8 @@ github.com/ucwong/filecache v1.0.6-0.20230405163841-810d53ced4bd h1:gBtlvLAsgLk+ github.com/ucwong/filecache v1.0.6-0.20230405163841-810d53ced4bd/go.mod h1:ddwX+NCjMZPdpzcGh1fcEbNTUTCtKgt2hC2rqvmLKgA= github.com/ucwong/go-ttlmap v1.0.2-0.20221020173635-331e7ddde2bb h1:dVZH3AH9f7zB3VBmsjn25B7lfcAyMP4QxdFYTrfj7tg= github.com/ucwong/go-ttlmap v1.0.2-0.20221020173635-331e7ddde2bb/go.mod h1:3yswsBsVuwsOjDvFfC5Na9XSEf4HC7mj3W3g6jvSY/s= -github.com/ucwong/golang-kv v1.0.23-0.20231209094038-056d175d4a78 h1:REUDgiVCtP/oWYjaKtYJAT+/0IG+JtWRvoEbrDf+tis= -github.com/ucwong/golang-kv v1.0.23-0.20231209094038-056d175d4a78/go.mod h1:vvoIDmHn9RX8EN1ODqtTJlhltmme3xs7Gssvl4yeM7E= +github.com/ucwong/golang-kv v1.0.23-0.20231212115239-d45c3e24ca4a h1:VWHe0JTcYPMCkfepGUR3fLfcPlGZnRP6f+efRmy2BY0= +github.com/ucwong/golang-kv v1.0.23-0.20231212115239-d45c3e24ca4a/go.mod h1:MfQo4fuucdLW2W9/YT0Jq5dVWlLTN7I878yPka7oL7E= github.com/ucwong/golang-set v1.8.1-0.20200419153428-d7b0b1ac2d43/go.mod h1:xu0FaiQFGbBcFZj2o7udZ5rbA8jRTsv47hkPoG5qQNM= github.com/ucwong/goleveldb v1.0.3-0.20200508074755-578cba616f37/go.mod h1:dgJUTtDxq/ne6/JzZhHzF24OL/uqILz9IWk8HmT4V2g= github.com/ucwong/goleveldb v1.0.3-0.20200618184106-f1c6bc3a428b/go.mod h1:7Sq6w7AfEZuB/a6mrlvHCSXCSkqojCMMrM3Ei12QAT0= @@ -1794,8 +1794,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -modernc.org/libc v1.37.0 h1:WerjebcsP6A7Jy+f2lCnHAkiSTLf7IaSftBYUtoswak= -modernc.org/libc v1.37.0/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE= +modernc.org/libc v1.37.1 h1:Wi3qhejztgB3hOYQGMc8NwePETHAWXmlU+GQnBNTrw8= +modernc.org/libc v1.37.1/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= diff --git a/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/CHANGELOG.md b/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/CHANGELOG.md index aa30abf373..5c8411cb57 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/CHANGELOG.md +++ b/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/CHANGELOG.md @@ -1,5 +1,15 @@ # Release History +## 1.9.1 (2023-12-11) + +### Bugs Fixed + +* The `retry-after-ms` and `x-ms-retry-after-ms` headers weren't being checked during retries. + +### Other Changes + +* Update dependencies. + ## 1.9.0 (2023-11-06) ### Breaking Changes diff --git a/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared/constants.go b/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared/constants.go index 272f06155e..bb93daee68 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared/constants.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared/constants.go @@ -22,11 +22,13 @@ const ( HeaderLocation = "Location" HeaderOperationLocation = "Operation-Location" HeaderRetryAfter = "Retry-After" + HeaderRetryAfterMS = "Retry-After-Ms" HeaderUserAgent = "User-Agent" HeaderWWWAuthenticate = "WWW-Authenticate" HeaderXMSClientRequestID = "x-ms-client-request-id" HeaderXMSRequestID = "x-ms-request-id" HeaderXMSErrorCode = "x-ms-error-code" + HeaderXMSRetryAfterMS = "x-ms-retry-after-ms" ) const BearerTokenPrefix = "Bearer " @@ -38,5 +40,5 @@ const ( Module = "azcore" // Version is the semantic version (see http://semver.org) of this module. - Version = "v1.9.0" + Version = "v1.9.1" ) diff --git a/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared/shared.go b/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared/shared.go index 16bc105f48..d3da2c5fdf 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared/shared.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared/shared.go @@ -44,22 +44,64 @@ func Delay(ctx context.Context, delay time.Duration) error { } } -// RetryAfter returns non-zero if the response contains a Retry-After header value. +// RetryAfter returns non-zero if the response contains one of the headers with a "retry after" value. +// Headers are checked in the following order: retry-after-ms, x-ms-retry-after-ms, retry-after func RetryAfter(resp *http.Response) time.Duration { if resp == nil { return 0 } - ra := resp.Header.Get(HeaderRetryAfter) - if ra == "" { - return 0 + + type retryData struct { + header string + units time.Duration + + // custom is used when the regular algorithm failed and is optional. + // the returned duration is used verbatim (units is not applied). + custom func(string) time.Duration } - // retry-after values are expressed in either number of - // seconds or an HTTP-date indicating when to try again - if retryAfter, _ := strconv.Atoi(ra); retryAfter > 0 { - return time.Duration(retryAfter) * time.Second - } else if t, err := time.Parse(time.RFC1123, ra); err == nil { - return time.Until(t) + + nop := func(string) time.Duration { return 0 } + + // the headers are listed in order of preference + retries := []retryData{ + { + header: HeaderRetryAfterMS, + units: time.Millisecond, + custom: nop, + }, + { + header: HeaderXMSRetryAfterMS, + units: time.Millisecond, + custom: nop, + }, + { + header: HeaderRetryAfter, + units: time.Second, + + // retry-after values are expressed in either number of + // seconds or an HTTP-date indicating when to try again + custom: func(ra string) time.Duration { + t, err := time.Parse(time.RFC1123, ra) + if err != nil { + return 0 + } + return time.Until(t) + }, + }, } + + for _, retry := range retries { + v := resp.Header.Get(retry.header) + if v == "" { + continue + } + if retryAfter, _ := strconv.Atoi(v); retryAfter > 0 { + return time.Duration(retryAfter) * retry.units + } else if d := retry.custom(v); d > 0 { + return d + } + } + return 0 } diff --git a/vendor/github.com/CortexFoundation/torrentfs/backend/handler.go b/vendor/github.com/CortexFoundation/torrentfs/backend/handler.go index 080415db52..d894d58111 100644 --- a/vendor/github.com/CortexFoundation/torrentfs/backend/handler.go +++ b/vendor/github.com/CortexFoundation/torrentfs/backend/handler.go @@ -1005,7 +1005,7 @@ func (tm *TorrentManager) pendingLoop() { ctx, cancel := context.WithTimeout(context.Background(), timeout*time.Minute) defer cancel() select { - case <-t.GotInfo(): + case <-t.Torrent.GotInfo(): if b, err := bencode.Marshal(t.Torrent.Info()); err == nil { log.Debug("Record full nas in history", "ih", t.InfoHash(), "info", len(b)) if tm.kvdb != nil && tm.kvdb.Get([]byte(SEED_PRE+t.InfoHash())) == nil { diff --git a/vendor/github.com/cockroachdb/pebble/compaction.go b/vendor/github.com/cockroachdb/pebble/compaction.go index 784d5a2805..647efd69fd 100644 --- a/vendor/github.com/cockroachdb/pebble/compaction.go +++ b/vendor/github.com/cockroachdb/pebble/compaction.go @@ -1671,6 +1671,19 @@ type readCompaction struct { fileNum base.FileNum } +type downloadSpan struct { + start []byte + end []byte + // doneChans contains a list of channels passed into compactions as done + // channels. Each channel has a buffer size of 1 and is only passed into + // one compaction. This slice can grow over the lifetime of a downloadSpan. + doneChans []chan error + // compactionsStarted is the number of compactions started for this + // downloadSpan. Must be equal to len(doneChans)-1, i.e. there's one spare + // doneChan created each time a compaction starts up, for the next compaction. + compactionsStarted int +} + func (d *DB) addInProgressCompaction(c *compaction) { d.mu.compact.inProgress[c] = struct{}{} var isBase, isIntraL0 bool @@ -2305,6 +2318,66 @@ func pickElisionOnly(picker compactionPicker, env compactionEnv) *pickedCompacti return picker.pickElisionOnlyCompaction(env) } +// maybeScheduleDownloadCompaction schedules a download compaction. +// +// Requires d.mu to be held. +func (d *DB) maybeScheduleDownloadCompaction(env compactionEnv, maxConcurrentCompactions int) { + for len(d.mu.compact.downloads) > 0 && d.mu.compact.compactingCount < maxConcurrentCompactions { + v := d.mu.versions.currentVersion() + download := d.mu.compact.downloads[0] + env.inProgressCompactions = d.getInProgressCompactionInfoLocked(nil) + var externalFile *fileMetadata + var err error + var level int + for i := range v.Levels { + overlaps := v.Overlaps(i, d.cmp, download.start, download.end, true /* exclusiveEnd */) + iter := overlaps.Iter() + provider := d.objProvider + for f := iter.First(); f != nil; f = iter.Next() { + var objMeta objstorage.ObjectMetadata + objMeta, err = provider.Lookup(fileTypeTable, f.FileBacking.DiskFileNum) + if err != nil { + break + } + if objMeta.IsExternal() { + if f.IsCompacting() { + continue + } + externalFile = f + level = i + break + } + } + if externalFile != nil || err != nil { + break + } + } + if err != nil { + d.mu.compact.downloads = d.mu.compact.downloads[1:] + download.doneChans[download.compactionsStarted] <- err + continue + } + if externalFile == nil { + // The entirety of this span is downloaded, or is being downloaded right + // now. No need to schedule additional downloads for this span. + d.mu.compact.downloads = d.mu.compact.downloads[1:] + continue + } + pc := pickDownloadCompaction(v, d.opts, env, d.mu.versions.picker.getBaseLevel(), download, level, externalFile) + if pc != nil { + doneCh := download.doneChans[download.compactionsStarted] + download.compactionsStarted++ + // Create another doneChan for the next compaction. + download.doneChans = append(download.doneChans, make(chan error, 1)) + + c := newCompaction(pc, d.opts, d.timeNow(), d.ObjProvider()) + d.mu.compact.compactingCount++ + d.addInProgressCompaction(c) + go d.compact(c, doneCh) + } + } +} + // maybeScheduleCompactionPicker schedules a compaction if necessary, // calling `pickFunc` to pick automatic compactions. // @@ -2399,6 +2472,8 @@ func (d *DB) maybeScheduleCompactionPicker( d.addInProgressCompaction(c) go d.compact(c, nil) } + + d.maybeScheduleDownloadCompaction(env, maxConcurrentCompactions) } // deleteCompactionHintType indicates whether the deleteCompactionHint was diff --git a/vendor/github.com/cockroachdb/pebble/compaction_iter.go b/vendor/github.com/cockroachdb/pebble/compaction_iter.go index 0fb9e45f26..55ee67dc82 100644 --- a/vendor/github.com/cockroachdb/pebble/compaction_iter.go +++ b/vendor/github.com/cockroachdb/pebble/compaction_iter.go @@ -423,6 +423,10 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { return &i.key, i.value } + // TODO(sumeer): we could avoid calling Covers if i.iterStripeChange == + // sameStripeSameKey since that check has already been done in + // nextInStripeHelper. However, we also need to handle the case of + // CoversInvisibly below. if cover := i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum); cover == keyspan.CoversVisibly { // A pending range deletion deletes this key. Skip it. i.saveKey() @@ -602,6 +606,8 @@ func snapshotIndex(seq uint64, snapshots []uint64) (int, uint64) { // may set i.err, in which case i.iterKey will be nil. func (i *compactionIter) skipInStripe() { i.skip = true + // TODO(sumeer): we can avoid the overhead of calling i.rangeDelFrag.Covers, + // in this case of nextInStripe, since we are skipping all of them anyway. for i.nextInStripe() == sameStripeSkippable { if i.err != nil { panic(i.err) @@ -643,6 +649,9 @@ const ( // nextInStripe advances the iterator and returns one of the above const ints // indicating how its state changed. // +// All sameStripeSkippable keys that are covered by a RANGEDEL will be skipped +// and not returned. +// // Calls to nextInStripe must be preceded by a call to saveKey to retain a // temporary reference to the original key, so that forward iteration can // proceed with a reference to the original key. Care should be taken to avoid @@ -659,66 +668,72 @@ func (i *compactionIter) nextInStripe() stripeChangeType { // nextInStripeHelper is an internal helper for nextInStripe; callers should use // nextInStripe and not call nextInStripeHelper. func (i *compactionIter) nextInStripeHelper() stripeChangeType { - if !i.iterNext() { - return newStripeNewKey - } - key := i.iterKey + origSnapshotIdx := i.curSnapshotIdx + for { + if !i.iterNext() { + return newStripeNewKey + } + key := i.iterKey - if !i.equal(i.key.UserKey, key.UserKey) { - i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots) - return newStripeNewKey - } + if !i.equal(i.key.UserKey, key.UserKey) { + i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots) + return newStripeNewKey + } - // If i.key and key have the same user key, then - // 1. i.key must not have had a zero sequence number (or it would've be the last - // key with its user key). - // 2. i.key must have a strictly larger sequence number - // There's an exception in that either key may be a range delete. Range - // deletes may share a sequence number with a point key if the keys were - // ingested together. Range keys may also share the sequence number if they - // were ingested, but range keys are interleaved into the compaction - // iterator's input iterator at the maximal sequence number so their - // original sequence number will not be observed here. - if prevSeqNum := base.SeqNumFromTrailer(i.keyTrailer); (prevSeqNum == 0 || prevSeqNum <= key.SeqNum()) && - i.key.Kind() != InternalKeyKindRangeDelete && key.Kind() != InternalKeyKindRangeDelete { - prevKey := i.key - prevKey.Trailer = i.keyTrailer - panic(errors.AssertionFailedf("pebble: invariant violation: %s and %s out of order", prevKey, key)) - } + // If i.key and key have the same user key, then + // 1. i.key must not have had a zero sequence number (or it would've be the last + // key with its user key). + // 2. i.key must have a strictly larger sequence number + // There's an exception in that either key may be a range delete. Range + // deletes may share a sequence number with a point key if the keys were + // ingested together. Range keys may also share the sequence number if they + // were ingested, but range keys are interleaved into the compaction + // iterator's input iterator at the maximal sequence number so their + // original sequence number will not be observed here. + if prevSeqNum := base.SeqNumFromTrailer(i.keyTrailer); (prevSeqNum == 0 || prevSeqNum <= key.SeqNum()) && + i.key.Kind() != InternalKeyKindRangeDelete && key.Kind() != InternalKeyKindRangeDelete { + prevKey := i.key + prevKey.Trailer = i.keyTrailer + panic(errors.AssertionFailedf("pebble: invariant violation: %s and %s out of order", prevKey, key)) + } - origSnapshotIdx := i.curSnapshotIdx - i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots) - switch key.Kind() { - case InternalKeyKindRangeDelete: - // Range tombstones need to be exposed by the compactionIter to the upper level - // `compaction` object, so return them regardless of whether they are in the same - // snapshot stripe. - if i.curSnapshotIdx == origSnapshotIdx { - return sameStripeNonSkippable + i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots) + switch key.Kind() { + case InternalKeyKindRangeDelete: + // Range tombstones need to be exposed by the compactionIter to the upper level + // `compaction` object, so return them regardless of whether they are in the same + // snapshot stripe. + if i.curSnapshotIdx == origSnapshotIdx { + return sameStripeNonSkippable + } + return newStripeSameKey + case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: + // Range keys are interleaved at the max sequence number for a given user + // key, so we should not see any more range keys in this stripe. + panic("unreachable") + case InternalKeyKindInvalid: + if i.curSnapshotIdx == origSnapshotIdx { + return sameStripeNonSkippable + } + return newStripeSameKey + case InternalKeyKindDelete, InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSingleDelete, + InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized: + // Fall through + default: + i.iterKey = nil + i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind())) + i.valid = false + return newStripeNewKey } - return newStripeSameKey - case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: - // Range keys are interleaved at the max sequence number for a given user - // key, so we should not see any more range keys in this stripe. - panic("unreachable") - case InternalKeyKindInvalid: if i.curSnapshotIdx == origSnapshotIdx { - return sameStripeNonSkippable + // Same snapshot. + if i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.CoversVisibly { + continue + } + return sameStripeSkippable } return newStripeSameKey - case InternalKeyKindDelete, InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSingleDelete, - InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized: - // Fall through - default: - i.iterKey = nil - i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind())) - i.valid = false - return newStripeNewKey - } - if i.curSnapshotIdx == origSnapshotIdx { - return sameStripeSkippable } - return newStripeSameKey } func (i *compactionIter) setNext() { @@ -744,6 +759,10 @@ func (i *compactionIter) setNext() { // Else, we continue to loop through entries in the stripe looking for a // DEL. Note that we may stop *before* encountering a DEL, if one exists. + // + // NB: nextInStripe will skip sameStripeSkippable keys that are visibly + // covered by a RANGEDEL. This can include DELs -- this is fine since such + // DELs don't need to be combined with SET to make SETWITHDEL. for { switch i.nextInStripe() { case newStripeNewKey, newStripeSameKey: @@ -826,6 +845,13 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType { if i.err != nil { panic(i.err) } + // NB: MERGE#10+RANGEDEL#9 stays a MERGE, since nextInStripe skips + // sameStripeSkippable keys that are visibly covered by a RANGEDEL. There + // may be MERGE#7 that is invisibly covered and will be preserved, but + // there is no risk that MERGE#10 and MERGE#7 will get merged in the + // future as the RANGEDEL still exists and will be used in user-facing + // reads that see MERGE#10, and will also eventually cause MERGE#7 to be + // deleted in a compaction. key := i.iterKey switch key.Kind() { case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized: @@ -851,16 +877,6 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType { return sameStripeSkippable case InternalKeyKindSet, InternalKeyKindSetWithDelete: - if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly { - // We change the kind of the result key to a Set so that it shadows - // keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't - // strictly necessary, but provides consistency with the behavior of - // MERGE+DEL. - i.key.SetKind(InternalKeyKindSet) - i.skip = true - return sameStripeSkippable - } - // We've hit a Set or SetWithDel value. Merge with the existing // value and return. We change the kind of the resulting key to a // Set so that it shadows keys in lower levels. That is: @@ -875,16 +891,6 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType { return sameStripeSkippable case InternalKeyKindMerge: - if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly { - // We change the kind of the result key to a Set so that it shadows - // keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't - // strictly necessary, but provides consistency with the behavior of - // MERGE+DEL. - i.key.SetKind(InternalKeyKindSet) - i.skip = true - return sameStripeSkippable - } - // We've hit another Merge value. Merge with the existing value and // continue looping. i.err = valueMerger.MergeOlder(i.iterValue) diff --git a/vendor/github.com/cockroachdb/pebble/compaction_picker.go b/vendor/github.com/cockroachdb/pebble/compaction_picker.go index 6567391f34..f77eabcab8 100644 --- a/vendor/github.com/cockroachdb/pebble/compaction_picker.go +++ b/vendor/github.com/cockroachdb/pebble/compaction_picker.go @@ -1906,6 +1906,38 @@ func pickManualCompaction( return pc, false } +func pickDownloadCompaction( + vers *version, + opts *Options, + env compactionEnv, + baseLevel int, + download *downloadSpan, + level int, + file *fileMetadata, +) (pc *pickedCompaction) { + // Check if the file is compacting already. In that case we don't need to do + // anything as it'll be downloaded in the process. + if file.CompactionState == manifest.CompactionStateCompacting { + return nil + } + pc = newPickedCompaction(opts, vers, level, level, baseLevel) + pc.kind = compactionKindRewrite + pc.startLevel.files = manifest.NewLevelSliceKeySorted(opts.Comparer.Compare, []*fileMetadata{file}) + if !pc.setupInputs(opts, env.diskAvailBytes, pc.startLevel) { + // setupInputs returned false indicating there's a conflicting + // concurrent compaction. + return nil + } + if pc.outputLevel.level != level { + panic("pebble: download compaction picked unexpected output level") + } + // Fail-safe to protect against compacting the same sstable concurrently. + if inputRangeAlreadyCompacting(env, pc) { + return nil + } + return pc +} + func (p *compactionPickerByScore) pickReadTriggeredCompaction( env compactionEnv, ) (pc *pickedCompaction) { diff --git a/vendor/github.com/cockroachdb/pebble/db.go b/vendor/github.com/cockroachdb/pebble/db.go index ea23aaaea9..4f93c4bbc6 100644 --- a/vendor/github.com/cockroachdb/pebble/db.go +++ b/vendor/github.com/cockroachdb/pebble/db.go @@ -427,6 +427,9 @@ type DB struct { // The list of manual compactions. The next manual compaction to perform // is at the start of the list. New entries are added to the end. manual []*manualCompaction + // downloads is the list of suggested download tasks. The next download to + // perform is at the start of the list. New entries are added to the end. + downloads []*downloadSpan // inProgress is the set of in-progress flushes and compactions. // It's used in the calculation of some metrics and to initialize L0 // sublevels' state. Some of the compactions contained within this @@ -1894,6 +1897,108 @@ type DownloadSpan struct { EndKey []byte } +func (d *DB) downloadSpan(ctx context.Context, span DownloadSpan) error { + dSpan := &downloadSpan{ + start: span.StartKey, + end: span.EndKey, + // Protected by d.mu. + doneChans: make([]chan error, 1), + } + dSpan.doneChans[0] = make(chan error, 1) + doneChan := dSpan.doneChans[0] + compactionIdx := 0 + + func() { + d.mu.Lock() + defer d.mu.Unlock() + + d.mu.compact.downloads = append(d.mu.compact.downloads, dSpan) + d.maybeScheduleCompaction() + }() + + // Requires d.mu to be held. + noExternalFilesInSpan := func() (noExternalFiles bool) { + vers := d.mu.versions.currentVersion() + + for i := 0; i < len(vers.Levels); i++ { + if vers.Levels[i].Empty() { + continue + } + overlap := vers.Overlaps(i, d.cmp, span.StartKey, span.EndKey, true /* exclusiveEnd */) + foundExternalFile := false + overlap.Each(func(metadata *manifest.FileMetadata) { + objMeta, err := d.objProvider.Lookup(fileTypeTable, metadata.FileBacking.DiskFileNum) + if err != nil { + return + } + if objMeta.IsExternal() { + foundExternalFile = true + } + }) + if foundExternalFile { + return false + } + } + return true + } + + // Requires d.mu to be held. + removeUsFromList := func() { + // Check where we are in d.mu.compact.downloads. Remove us from the + // list. + for i := range d.mu.compact.downloads { + if d.mu.compact.downloads[i] != dSpan { + continue + } + copy(d.mu.compact.downloads[i:], d.mu.compact.downloads[i+1:]) + d.mu.compact.downloads = d.mu.compact.downloads[:len(d.mu.compact.downloads)-1] + break + } + } + + for { + select { + case <-ctx.Done(): + d.mu.Lock() + defer d.mu.Unlock() + removeUsFromList() + return ctx.Err() + case err := <-doneChan: + if err != nil { + d.mu.Lock() + defer d.mu.Unlock() + removeUsFromList() + return err + } + compactionIdx++ + // Grab the next doneCh to wait on. + func() { + d.mu.Lock() + defer d.mu.Unlock() + doneChan = dSpan.doneChans[compactionIdx] + }() + default: + doneSpan := func() bool { + d.mu.Lock() + defer d.mu.Unlock() + // It's possible to have downloaded all files without writing to any + // doneChans. This is expected if there are a significant amount + // of overlapping writes that schedule regular, non-download compactions. + if noExternalFilesInSpan() { + removeUsFromList() + return true + } + d.maybeScheduleCompaction() + d.mu.compact.cond.Wait() + return false + }() + if doneSpan { + return nil + } + } + } +} + // Download ensures that the LSM does not use any external sstables for the // given key ranges. It does so by performing appropriate compactions so that // all external data becomes available locally. @@ -1908,7 +2013,23 @@ type DownloadSpan struct { // TODO(radu): consider passing a priority/impact knob to express how important // the download is (versus live traffic performance, LSM health). func (d *DB) Download(ctx context.Context, spans []DownloadSpan) error { - return errors.Errorf("not implemented") + ctx, cancel := context.WithCancel(ctx) + defer cancel() + if err := d.closed.Load(); err != nil { + panic(err) + } + if d.opts.ReadOnly { + return ErrReadOnly + } + for i := range spans { + if err := ctx.Err(); err != nil { + return err + } + if err := d.downloadSpan(ctx, spans[i]); err != nil { + return err + } + } + return nil } // Flush the memtable to stable storage. diff --git a/vendor/github.com/nutsdb/nutsdb/db.go b/vendor/github.com/nutsdb/nutsdb/db.go index 73566e3e65..362249a37a 100644 --- a/vendor/github.com/nutsdb/nutsdb/db.go +++ b/vendor/github.com/nutsdb/nutsdb/db.go @@ -442,7 +442,7 @@ func (db *DB) parseDataFiles(dataFileIds []int) (err error) { // its because it already deleted in the feature WAL log. // so we can just ignore here. bucketId := entry.Meta.BucketId - if b, err := db.bm.GetBucketById(bucketId); b == nil && err == nil { + if _, err := db.bm.GetBucketById(bucketId); errors.Is(err, ErrBucketNotExist) { continue } @@ -702,7 +702,8 @@ func (db *DB) buildSortedSetIdx(record *Record, entry *Entry) error { _, _, err = ss.ZPopMin(string(key)) } - if err != nil { + // We don't need to panic if sorted set is not found. + if err != nil && !errors.Is(err, ErrSortedSetNotFound) { return fmt.Errorf("when build sortedSetIdx err: %s", err) } diff --git a/vendor/github.com/nutsdb/nutsdb/tx.go b/vendor/github.com/nutsdb/nutsdb/tx.go index 4f2bd20974..4f8ea057da 100644 --- a/vendor/github.com/nutsdb/nutsdb/tx.go +++ b/vendor/github.com/nutsdb/nutsdb/tx.go @@ -18,10 +18,11 @@ import ( "bytes" "errors" "fmt" - "github.com/bwmarrin/snowflake" - "github.com/xujiajun/utils/strconv2" "strings" "sync/atomic" + + "github.com/bwmarrin/snowflake" + "github.com/xujiajun/utils/strconv2" ) const ( @@ -279,7 +280,7 @@ func (tx *Tx) Commit() (err error) { } tx.db.RecordCount += curWriteCount - if err := tx.DeleteBucketInIndex(); err != nil { + if err := tx.buildBucketInIndex(); err != nil { return err } @@ -288,8 +289,8 @@ func (tx *Tx) Commit() (err error) { func (tx *Tx) getNewAddRecordCount() (int64, error) { var res int64 - changeCountInEntries := tx.GetChangeCountInEntriesChanges() - changeCountInBucket := tx.GetChangeCountInBucketChanges() + changeCountInEntries := tx.getChangeCountInEntriesChanges() + changeCountInBucket := tx.getChangeCountInBucketChanges() res += changeCountInEntries res += changeCountInBucket return res, nil @@ -716,10 +717,24 @@ func (tx *Tx) SubmitBucket() error { return tx.db.bm.SubmitPendingBucketChange(bucketReqs) } -func (tx *Tx) DeleteBucketInIndex() error { +// buildBucketInIndex build indexes on creation and deletion of buckets +func (tx *Tx) buildBucketInIndex() error { for _, mapper := range tx.pendingBucketList { for _, bucket := range mapper { - if bucket.Meta.Op == BucketDeleteOperation { + if bucket.Meta.Op == BucketInsertOperation { + switch bucket.Ds { + case DataStructureBTree: + tx.db.Index.bTree.getWithDefault(bucket.Id) + case DataStructureList: + tx.db.Index.list.getWithDefault(bucket.Id) + case DataStructureSet: + tx.db.Index.set.getWithDefault(bucket.Id) + case DataStructureSortedSet: + tx.db.Index.sortedSet.getWithDefault(bucket.Id, tx.db) + default: + return ErrDataStructureNotSupported + } + } else if bucket.Meta.Op == BucketDeleteOperation { switch bucket.Ds { case DataStructureBTree: tx.db.Index.bTree.delete(bucket.Id) @@ -738,7 +753,7 @@ func (tx *Tx) DeleteBucketInIndex() error { return nil } -func (tx *Tx) GetChangeCountInEntriesChanges() int64 { +func (tx *Tx) getChangeCountInEntriesChanges() int64 { var res int64 var err error writeLen := len(tx.pendingWrites) @@ -753,7 +768,7 @@ func (tx *Tx) GetChangeCountInEntriesChanges() int64 { return res } -func (tx *Tx) GetChangeCountInBucketChanges() int64 { +func (tx *Tx) getChangeCountInBucketChanges() int64 { var res int64 for _, bucketsInDs := range tx.pendingBucketList { for _, bucket := range bucketsInDs { @@ -793,7 +808,7 @@ func (tx *Tx) GetChangeCountInBucketChanges() int64 { return res } -func (tx *Tx) GetBucketStatus(ds Ds, name BucketName) BucketStatus { +func (tx *Tx) getBucketStatus(ds Ds, name BucketName) BucketStatus { if len(tx.pendingBucketList) > 0 { if bucketInDs, exist := tx.pendingBucketList[ds]; exist { if bucket, exist := bucketInDs[name]; exist { diff --git a/vendor/github.com/nutsdb/nutsdb/tx_btree.go b/vendor/github.com/nutsdb/nutsdb/tx_btree.go index 391e2d88a4..ed265474cf 100644 --- a/vendor/github.com/nutsdb/nutsdb/tx_btree.go +++ b/vendor/github.com/nutsdb/nutsdb/tx_btree.go @@ -35,6 +35,56 @@ func (tx *Tx) Put(bucket string, key, value []byte, ttl uint32) error { return tx.put(bucket, key, value, ttl, DataSetFlag, uint64(time.Now().UnixMilli()), DataStructureBTree) } +// PutIfNotExists set the value for a key in the bucket only if the key doesn't exist already. +func (tx *Tx) PutIfNotExists(bucket string, key, value []byte, ttl uint32) error { + if err := tx.checkTxIsClosed(); err != nil { + return err + } + + b, err := tx.db.bm.GetBucket(DataStructureBTree, bucket) + if err != nil { + return err + } + bucketId := b.Id + + idx, bucketExists := tx.db.Index.bTree.exist(bucketId) + if !bucketExists { + return ErrNotFoundBucket + } + record, recordExists := idx.Find(key) + + if recordExists && !record.IsExpired() { + return nil + } + + return tx.put(bucket, key, value, ttl, DataSetFlag, uint64(time.Now().UnixMilli()), DataStructureBTree) +} + +// PutIfExits set the value for a key in the bucket only if the key already exits. +func (tx *Tx) PutIfExists(bucket string, key, value []byte, ttl uint32) error { + if err := tx.checkTxIsClosed(); err != nil { + return err + } + + b, err := tx.db.bm.GetBucket(DataStructureBTree, bucket) + if err != nil { + return err + } + bucketId := b.Id + + idx, bucketExists := tx.db.Index.bTree.exist(bucketId) + if !bucketExists { + return ErrNotFoundBucket + } + + record, recordExists := idx.Find(key) + if recordExists && !record.IsExpired() { + return tx.put(bucket, key, value, ttl, DataSetFlag, uint64(time.Now().UnixMilli()), DataStructureBTree) + } + + return nil +} + // Get retrieves the value for a key in the bucket. // The returned value is only valid for the life of the transaction. func (tx *Tx) Get(bucket string, key []byte) (value []byte, err error) { diff --git a/vendor/github.com/nutsdb/nutsdb/tx_list.go b/vendor/github.com/nutsdb/nutsdb/tx_list.go index a9d829ced9..2c3202c6c8 100644 --- a/vendor/github.com/nutsdb/nutsdb/tx_list.go +++ b/vendor/github.com/nutsdb/nutsdb/tx_list.go @@ -52,9 +52,13 @@ func (tx *Tx) RPeek(bucket string, key []byte) ([]byte, error) { if err != nil { return nil, err } - bucketId := b.Id - l := tx.db.Index.list.getWithDefault(bucketId) - if l == nil { + var ( + bucketId = b.Id + l *List + exist bool + ) + + if l, exist = tx.db.Index.list.exist(bucketId); !exist { return nil, ErrBucket } @@ -183,9 +187,13 @@ func (tx *Tx) LPeek(bucket string, key []byte) (item []byte, err error) { if err != nil { return nil, err } - bucketId := b.Id - l := tx.db.Index.list.getWithDefault(bucketId) - if l == nil { + var ( + bucketId = b.Id + l *List + exist bool + ) + + if l, exist = tx.db.Index.list.exist(bucketId); !exist { return nil, ErrBucket } if tx.CheckExpire(bucket, key) { @@ -214,9 +222,14 @@ func (tx *Tx) LSize(bucket string, key []byte) (int, error) { if err != nil { return 0, err } - bucketId := b.Id - l := tx.db.Index.list.getWithDefault(bucketId) - if l == nil { + + var ( + bucketId = b.Id + l *List + exist bool + ) + + if l, exist = tx.db.Index.list.exist(bucketId); !exist { return 0, ErrBucket } if tx.CheckExpire(bucket, key) { @@ -239,9 +252,13 @@ func (tx *Tx) LRange(bucket string, key []byte, start, end int) ([][]byte, error if err != nil { return nil, err } - bucketId := b.Id - l := tx.db.Index.list.getWithDefault(bucketId) - if l == nil { + var ( + bucketId = b.Id + l *List + exist bool + ) + + if l, exist = tx.db.Index.list.exist(bucketId); !exist { return nil, ErrBucket } if tx.CheckExpire(bucket, key) { @@ -317,8 +334,17 @@ func (tx *Tx) LTrim(bucket string, key []byte, start, end int) error { if err != nil { return err } - bucketId := b.Id - l := tx.db.Index.list.getWithDefault(bucketId) + + var ( + bucketId = b.Id + l *List + exist bool + ) + + if l, exist = tx.db.Index.list.exist(bucketId); !exist { + return ErrBucket + } + if tx.CheckExpire(bucket, key) { return ErrListNotFound } @@ -384,11 +410,15 @@ func (tx *Tx) LKeys(bucket, pattern string, f func(key string) bool) error { if err != nil { return err } - bucketId := b.Id - l := tx.db.Index.list.getWithDefault(bucketId) - if l == nil { + var ( + bucketId = b.Id + l *List + exist bool + ) + if l, exist = tx.db.Index.list.exist(bucketId); !exist { return ErrBucket } + for key := range l.Items { if tx.CheckExpire(bucket, []byte(key)) { continue @@ -408,8 +438,17 @@ func (tx *Tx) ExpireList(bucket string, key []byte, ttl uint32) error { if err != nil { return err } - bucketId := b.Id - l := tx.db.Index.list.getWithDefault(bucketId) + + var ( + bucketId = b.Id + l *List + exist bool + ) + + if l, exist = tx.db.Index.list.exist(bucketId); !exist { + return ErrBucket + } + l.TTL[string(key)] = ttl l.TimeStamp[string(key)] = uint64(time.Now().Unix()) ttls := strconv2.Int64ToStr(int64(ttl)) @@ -425,8 +464,17 @@ func (tx *Tx) CheckExpire(bucket string, key []byte) bool { if err != nil { return false } - bucketId := b.Id - l := tx.db.Index.list.getWithDefault(bucketId) + + var ( + bucketId = b.Id + l *List + exist bool + ) + + if l, exist = tx.db.Index.list.exist(bucketId); !exist { + return false + } + if l.IsExpire(string(key)) { _ = tx.push(bucket, key, DataDeleteFlag) return true @@ -442,9 +490,13 @@ func (tx *Tx) GetListTTL(bucket string, key []byte) (uint32, error) { if err != nil { return 0, err } - bucketId := b.Id - l := tx.db.Index.list.getWithDefault(bucketId) - if l == nil { + + var ( + bucketId = b.Id + l *List + exist bool + ) + if l, exist = tx.db.Index.list.exist(bucketId); !exist { return 0, ErrBucket } return l.GetListTTL(string(key)) diff --git a/vendor/github.com/nutsdb/nutsdb/tx_zset.go b/vendor/github.com/nutsdb/nutsdb/tx_zset.go index 6284e22dc9..277f1108a5 100644 --- a/vendor/github.com/nutsdb/nutsdb/tx_zset.go +++ b/vendor/github.com/nutsdb/nutsdb/tx_zset.go @@ -59,9 +59,18 @@ func (tx *Tx) ZMembers(bucket string, key []byte) (map[*SortedSetMember]struct{} if err != nil { return nil, err } - bucketId := b.Id - members, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZMembers(string(key)) + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return nil, ErrBucket + } + + members, err := sortedSet.ZMembers(string(key)) if err != nil { return nil, err } @@ -91,13 +100,18 @@ func (tx *Tx) ZCard(bucket string, key []byte) (int, error) { if err != nil { return 0, err } - bucketId := b.Id - card, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZCard(string(key)) - if err != nil { - return 0, err + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return 0, ErrBucket } - return card, nil + + return sortedSet.ZCard(string(key)) } // ZCount Returns the number of elements in the sorted set specified by key in a bucket with a score between min and max and opts. @@ -114,14 +128,18 @@ func (tx *Tx) ZCount(bucket string, key []byte, start, end float64, opts *GetByS if err != nil { return 0, err } - bucketId := b.Id - count, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZCount(string(key), SCORE(start), SCORE(end), opts) - if err != nil { - return 0, err + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return 0, ErrBucket } - return count, nil + return sortedSet.ZCount(string(key), SCORE(start), SCORE(end), opts) } // ZPopMax Removes and returns the member with the highest score in the sorted set specified by key in a bucket. @@ -134,9 +152,18 @@ func (tx *Tx) ZPopMax(bucket string, key []byte) (*SortedSetMember, error) { if err != nil { return nil, err } - bucketId := b.Id - record, score, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZPeekMax(string(key)) + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return nil, ErrBucket + } + + record, score, err := sortedSet.ZPeekMax(string(key)) if err != nil { return nil, err } @@ -159,8 +186,18 @@ func (tx *Tx) ZPopMin(bucket string, key []byte) (*SortedSetMember, error) { if err != nil { return nil, err } - bucketId := b.Id - record, score, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZPeekMin(string(key)) + + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return nil, ErrBucket + } + + record, score, err := sortedSet.ZPeekMin(string(key)) if err != nil { return nil, err } @@ -183,9 +220,18 @@ func (tx *Tx) ZPeekMax(bucket string, key []byte) (*SortedSetMember, error) { if err != nil { return nil, err } - bucketId := b.Id - record, score, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZPeekMax(string(key)) + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return nil, ErrBucket + } + + record, score, err := sortedSet.ZPeekMax(string(key)) if err != nil { return nil, err } @@ -208,9 +254,18 @@ func (tx *Tx) ZPeekMin(bucket string, key []byte) (*SortedSetMember, error) { if err != nil { return nil, err } - bucketId := b.Id - record, score, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZPeekMin(string(key)) + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return nil, ErrBucket + } + + record, score, err := sortedSet.ZPeekMin(string(key)) if err != nil { return nil, err } @@ -234,8 +289,18 @@ func (tx *Tx) ZRangeByScore(bucket string, key []byte, start, end float64, opts if err != nil { return nil, err } - bucketId := b.Id - records, scores, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZRangeByScore(string(key), SCORE(start), SCORE(end), opts) + + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return nil, ErrBucket + } + + records, scores, err := sortedSet.ZRangeByScore(string(key), SCORE(start), SCORE(end), opts) if err != nil { return nil, err } @@ -263,9 +328,18 @@ func (tx *Tx) ZRangeByRank(bucket string, key []byte, start, end int) ([]*Sorted if err != nil { return nil, err } - bucketId := b.Id - records, scores, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZRangeByRank(string(key), start, end) + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return nil, ErrBucket + } + + records, scores, err := sortedSet.ZRangeByRank(string(key), start, end) if err != nil { return nil, err } @@ -292,8 +366,18 @@ func (tx *Tx) ZRem(bucket string, key []byte, value []byte) error { if err != nil { return err } - bucketId := b.Id - exist, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZExist(string(key), value) + + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return ErrBucket + } + + exist, err = sortedSet.ZExist(string(key), value) if err != nil { return err } @@ -327,8 +411,18 @@ func (tx *Tx) ZRank(bucket string, key, value []byte) (int, error) { if err != nil { return 0, err } - bucketId := b.Id - return tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZRank(string(key), value) + + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return 0, ErrBucket + } + + return sortedSet.ZRank(string(key), value) } // ZRevRank Returns the rank of member in the sorted set specified by key in a bucket, with the scores ordered from high to low. @@ -341,8 +435,18 @@ func (tx *Tx) ZRevRank(bucket string, key, value []byte) (int, error) { if err != nil { return 0, err } - bucketId := b.Id - return tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZRevRank(string(key), value) + + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return 0, ErrBucket + } + + return sortedSet.ZRevRank(string(key), value) } // ZScore Returns the score of members in a sorted set specified by key in a bucket. @@ -355,12 +459,18 @@ func (tx *Tx) ZScore(bucket string, key, value []byte) (float64, error) { if err != nil { return 0.0, err } - bucketId := b.Id - if score, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).ZScore(string(key), value); err != nil { - return 0, err - } else { - return score, nil + + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return 0, ErrBucket } + + return sortedSet.ZScore(string(key), value) } // ZKeys find all keys matching a given pattern in a bucket @@ -373,8 +483,18 @@ func (tx *Tx) ZKeys(bucket, pattern string, f func(key string) bool) error { if err != nil { return err } - bucketId := b.Id - for key := range tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).M { + + var ( + bucketId = b.Id + sortedSet *SortedSet + exist bool + ) + + if sortedSet, exist = tx.db.Index.sortedSet.exist(bucketId); !exist { + return ErrBucket + } + + for key := range sortedSet.M { if end, err := MatchForRange(pattern, key, f); end || err != nil { return err } diff --git a/vendor/modernc.org/libc/libc_darwin.go b/vendor/modernc.org/libc/libc_darwin.go index 11a9698f69..5a38f141b2 100644 --- a/vendor/modernc.org/libc/libc_darwin.go +++ b/vendor/modernc.org/libc/libc_darwin.go @@ -1951,12 +1951,12 @@ func Xsetattrlist(t *TLS, path, attrList, attrBuf uintptr, attrBufSize types.Siz } // int copyfile(const char *from, const char *to, copyfile_state_t state, copyfile_flags_t flags); -func Xcopyfile(...interface{}) int32 { +func Xcopyfile(t *TLS, _ ...interface{}) int32 { panic(todo("")) } // int truncate(const char *path, off_t length); -func Xtruncate(...interface{}) int32 { +func Xtruncate(t *TLS, _ ...interface{}) int32 { panic(todo("")) } @@ -2449,3 +2449,28 @@ func Xmalloc_size(t *TLS, ptr uintptr) types.Size_t { } panic(todo("")) } + +// int open(const char *pathname, int flags, ...); +func Xopen64(t *TLS, pathname uintptr, flags int32, args uintptr) int32 { + if __ccgo_strace { + trc("t=%v pathname=%v flags=%v args=%v, (%v:)", t, pathname, flags, args, origin(2)) + } + var mode types.Mode_t + if args != 0 { + mode = (types.Mode_t)(VaUint32(&args)) + } + fdcwd := fcntl.AT_FDCWD + n, _, err := unix.Syscall6(unix.SYS_OPENAT, uintptr(fdcwd), pathname, uintptr(flags), uintptr(mode), 0, 0) + if err != 0 { + // if dmesgs { + // dmesg("%v: %q %#x: %v", origin(1), GoString(pathname), flags, err) + // } + t.setErrno(err) + return -1 + } + + // if dmesgs { + // dmesg("%v: %q flags %#x mode %#o: fd %v", origin(1), GoString(pathname), flags, mode, n) + // } + return int32(n) +} diff --git a/vendor/modernc.org/libc/libc_darwin_amd64.go b/vendor/modernc.org/libc/libc_darwin_amd64.go index 5f3929e690..74a51ade5a 100644 --- a/vendor/modernc.org/libc/libc_darwin_amd64.go +++ b/vendor/modernc.org/libc/libc_darwin_amd64.go @@ -11,6 +11,7 @@ import ( "golang.org/x/sys/unix" "modernc.org/libc/fcntl" "modernc.org/libc/signal" + "modernc.org/libc/stdio" "modernc.org/libc/sys/types" "modernc.org/libc/utime" ) @@ -521,3 +522,10 @@ func Xfopen64(t *TLS, pathname, mode uintptr) uintptr { panic("OOM") } + +func Xrewinddir(tls *TLS, f uintptr) { + if __ccgo_strace { + trc("tls=%v f=%v, (%v:)", tls, f, origin(2)) + } + Xfseek(tls, f, 0, stdio.SEEK_SET) +} diff --git a/vendor/modernc.org/libc/libc_darwin_arm64.go b/vendor/modernc.org/libc/libc_darwin_arm64.go index b156eae1d7..ecb7c911de 100644 --- a/vendor/modernc.org/libc/libc_darwin_arm64.go +++ b/vendor/modernc.org/libc/libc_darwin_arm64.go @@ -12,6 +12,7 @@ import ( "golang.org/x/sys/unix" "modernc.org/libc/fcntl" "modernc.org/libc/signal" + "modernc.org/libc/stdio" "modernc.org/libc/sys/types" "modernc.org/libc/utime" ) @@ -521,3 +522,10 @@ func Xfopen64(t *TLS, pathname, mode uintptr) uintptr { panic("OOM") } + +func Xrewinddir(tls *TLS, f uintptr) { + if __ccgo_strace { + trc("tls=%v f=%v, (%v:)", tls, f, origin(2)) + } + Xfseek(tls, f, 0, stdio.SEEK_SET) +} diff --git a/vendor/modernc.org/libc/libc_openbsd.go b/vendor/modernc.org/libc/libc_openbsd.go index 50c528a9e8..17e46f89e6 100644 --- a/vendor/modernc.org/libc/libc_openbsd.go +++ b/vendor/modernc.org/libc/libc_openbsd.go @@ -1989,6 +1989,13 @@ func Xbswap16(t *TLS, x uint16) uint16 { return X__builtin_bswap16(t, x) } +func X__swap16md(t *TLS, x uint16) uint16 { + if __ccgo_strace { + trc("t=%v x=%v, (%v:)", t, x, origin(2)) + } + return X__builtin_bswap16(t, x) +} + // uint32_t __builtin_bswap32 (uint32_t x) func Xbswap32(t *TLS, x uint32) uint32 { if __ccgo_strace { diff --git a/vendor/modernc.org/libc/libc_openbsd_amd64.go b/vendor/modernc.org/libc/libc_openbsd_amd64.go index 20017d9a04..c45c44c9dc 100644 --- a/vendor/modernc.org/libc/libc_openbsd_amd64.go +++ b/vendor/modernc.org/libc/libc_openbsd_amd64.go @@ -11,6 +11,7 @@ import ( "golang.org/x/sys/unix" "modernc.org/libc/fcntl" + "modernc.org/libc/stdio" "modernc.org/libc/fts" "modernc.org/libc/sys/types" "modernc.org/libc/time" @@ -709,3 +710,12 @@ func Xopendir(t *TLS, name uintptr) uintptr { (*darwinDir)(unsafe.Pointer(p)).eof = false return p } + +func Xrewinddir(tls *TLS, f uintptr) { + if __ccgo_strace { + trc("tls=%v f=%v, (%v:)", tls, f, origin(2)) + } + Xfseek(tls, f, 0, stdio.SEEK_SET) +} + + diff --git a/vendor/modules.txt b/vendor/modules.txt index c5a7660af4..5208dc1e97 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 +# github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 ## explicit; go 1.18 github.com/Azure/azure-sdk-for-go/sdk/azcore github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud @@ -47,7 +47,7 @@ github.com/CortexFoundation/robot/backend # github.com/CortexFoundation/statik v0.0.0-20210315012922-8bb8a7b5dc66 ## explicit; go 1.16 github.com/CortexFoundation/statik -# github.com/CortexFoundation/torrentfs v1.0.56-0.20231209192839-e28c596da6b3 +# github.com/CortexFoundation/torrentfs v1.0.56-0.20231212120813-41a5945e7136 ## explicit; go 1.21 github.com/CortexFoundation/torrentfs github.com/CortexFoundation/torrentfs/backend @@ -358,7 +358,7 @@ github.com/cockroachdb/errors/withstack # github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b ## explicit; go 1.16 github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20231208175357-58bdc725addc +# github.com/cockroachdb/pebble v0.0.0-20231211223132-9848bcdb0fe9 ## explicit; go 1.21 github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom @@ -599,7 +599,7 @@ github.com/google/go-querystring/query ## explicit; go 1.12 github.com/google/gofuzz github.com/google/gofuzz/bytesource -# github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08 +# github.com/google/pprof v0.0.0-20231212022811-ec68065c825e ## explicit; go 1.19 github.com/google/pprof/profile # github.com/google/uuid v1.4.0 @@ -766,7 +766,7 @@ github.com/naoina/go-stringutil ## explicit github.com/naoina/toml github.com/naoina/toml/ast -# github.com/nutsdb/nutsdb v1.0.2-0.20231209035901-f7c4a82b75c2 +# github.com/nutsdb/nutsdb v1.0.2-0.20231211145816-3468fe949461 ## explicit; go 1.18 github.com/nutsdb/nutsdb # github.com/nxadm/tail v1.4.11 @@ -990,7 +990,7 @@ github.com/ucwong/filecache # github.com/ucwong/go-ttlmap v1.0.2-0.20221020173635-331e7ddde2bb ## explicit; go 1.19 github.com/ucwong/go-ttlmap -# github.com/ucwong/golang-kv v1.0.23-0.20231209094038-056d175d4a78 +# github.com/ucwong/golang-kv v1.0.23-0.20231212115239-d45c3e24ca4a ## explicit; go 1.21 github.com/ucwong/golang-kv github.com/ucwong/golang-kv/badger @@ -1230,7 +1230,7 @@ gopkg.in/urfave/cli.v1 # gopkg.in/yaml.v3 v3.0.1 ## explicit gopkg.in/yaml.v3 -# modernc.org/libc v1.37.0 +# modernc.org/libc v1.37.1 ## explicit; go 1.20 modernc.org/libc modernc.org/libc/errno