From f948c030358539bcfdff0007670844633dde10fd Mon Sep 17 00:00:00 2001 From: Jacob Baungard Hansen Date: Fri, 21 Jul 2023 09:29:35 +0200 Subject: [PATCH 001/130] Bump minio-go to v7.0.61 Bumps minio-go to latest version. This should fix: https://github.com/thanos-io/thanos/issues/6541 Signed-off-by: Jacob Baungard Hansen --- CHANGELOG.md | 1 + go.mod | 24 ++++++++++++------------ go.sum | 49 ++++++++++++++++++++++++------------------------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a43c278..564faa91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,5 +32,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. - [#39](https://github.com/thanos-io/objstore/pull/39) COS: Upgrade cos sdk version to `v0.7.40`. - [#35](https://github.com/thanos-io/objstore/pull/35) Azure: Update Azure SDK and fix breaking changes. +- [#65](https://github.com/thanos-io/objstore/pull/65) *: Upgrade minio-go version to `v7.0.61`. ### Removed diff --git a/go.mod b/go.mod index 7ae1f6d7..5bf1fd24 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/fatih/structtag v1.2.0 github.com/go-kit/log v0.2.1 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible - github.com/minio/minio-go/v7 v7.0.45 + github.com/minio/minio-go/v7 v7.0.61 github.com/ncw/swift v1.0.53 github.com/opentracing/opentracing-go v1.2.0 github.com/oracle/oci-go-sdk/v65 v65.13.0 @@ -53,7 +53,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect @@ -64,12 +64,12 @@ require ( github.com/googleapis/gax-go/v2 v2.3.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/klauspost/cpuid/v2 v2.1.0 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -78,14 +78,14 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect - github.com/rs/xid v1.4.0 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/rs/xid v1.5.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/stretchr/objx v0.2.0 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect google.golang.org/appengine v1.6.7 // indirect @@ -93,7 +93,7 @@ require ( google.golang.org/grpc v1.46.2 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/ini.v1 v1.66.6 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect ) require ( @@ -102,5 +102,5 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.3.0 // indirect + golang.org/x/crypto v0.11.0 // indirect ) diff --git a/go.sum b/go.sum index 5b85eb43..21e06236 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4 h1:rydBwnBoywKQMjWF0z8SriYtQ+uUcaFsxuijMjJr5PI= github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4/go.mod h1:kQa0V74HNYMfuJH6jiPiwNdpWXl4xd/K4tzlrcvYDQI= github.com/efficientgo/e2e v0.13.1-0.20220922081603-45de9fc588a8 h1:UFLc39BcUXahSNCLUrKjNGZABMUZaS4M74EZvTRnq3k= @@ -282,12 +282,11 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= -github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -305,10 +304,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.45 h1:g4IeM9M9pW/Lo8AGGNOjBZYlvmtlE1N5TQEYWXRWzIs= -github.com/minio/minio-go/v7 v7.0.45/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI= +github.com/minio/minio-go/v7 v7.0.61/go.mod h1:BTu8FcrEw+HidY0zd/0eny43QnVNkXRPXrLXFuQBHXg= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -363,15 +362,15 @@ github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -416,8 +415,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -496,8 +495,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -596,10 +595,10 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -611,8 +610,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -850,8 +849,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= -gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From d0c43443ecda0d8cffbd4d215569a0dba9ba39ec Mon Sep 17 00:00:00 2001 From: Christian Simon Date: Thu, 27 Jul 2023 12:56:35 +0100 Subject: [PATCH 002/130] Use a histogram to record transferred bytes for get get_range (#69) This will allow to understand the distribution of object store fetches. The metric has been purposely named so it can in the future also cover uploads. Signed-off-by: Christian Simon --- objstore.go | 30 +++++++++++++++++++++++++--- objstore_test.go | 51 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/objstore.go b/objstore.go index d7d04888..2a0fe4dc 100644 --- a/objstore.go +++ b/objstore.go @@ -424,6 +424,13 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu ConstLabels: prometheus.Labels{"bucket": name}, }, []string{"operation"}), + opsTransferredBytes: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Name: "objstore_bucket_operation_transferred_bytes", + Help: "Number of bytes transferred from/to bucket per operation.", + ConstLabels: prometheus.Labels{"bucket": name}, + Buckets: prometheus.ExponentialBuckets(2<<14, 2, 16), // 32KiB, 64KiB, ... 1GiB + }, []string{"operation"}), + opsDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ Name: "objstore_bucket_operation_duration_seconds", Help: "Duration of successful operations against the bucket", @@ -450,6 +457,14 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) } + // fetched bytes only relevant for get and getrange + for _, op := range []string{ + OpGet, + OpGetRange, + // TODO: Add uploads + } { + bkt.opsTransferredBytes.WithLabelValues(op) + } bkt.lastSuccessfulUploadTime.WithLabelValues(b.Name()) return bkt } @@ -461,8 +476,8 @@ type metricBucket struct { opsFailures *prometheus.CounterVec isOpFailureExpected IsOpFailureExpectedFunc - opsFetchedBytes *prometheus.CounterVec - + opsFetchedBytes *prometheus.CounterVec + opsTransferredBytes *prometheus.HistogramVec opsDuration *prometheus.HistogramVec lastSuccessfulUploadTime *prometheus.GaugeVec } @@ -473,6 +488,7 @@ func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { ops: b.ops, opsFailures: b.opsFailures, opsFetchedBytes: b.opsFetchedBytes, + opsTransferredBytes: b.opsTransferredBytes, isOpFailureExpected: fn, opsDuration: b.opsDuration, lastSuccessfulUploadTime: b.lastSuccessfulUploadTime, @@ -530,6 +546,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err b.opsFailures, b.isOpFailureExpected, b.opsFetchedBytes, + b.opsTransferredBytes, ), nil } @@ -551,6 +568,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in b.opsFailures, b.isOpFailureExpected, b.opsFetchedBytes, + b.opsTransferredBytes, ), nil } @@ -627,13 +645,15 @@ type timingReadCloser struct { start time.Time op string + readBytes int64 duration *prometheus.HistogramVec failed *prometheus.CounterVec isFailureExpected IsOpFailureExpectedFunc fetchedBytes *prometheus.CounterVec + transferredBytes *prometheus.HistogramVec } -func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec) *timingReadCloser { +func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -648,6 +668,8 @@ func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramV failed: failed, isFailureExpected: isFailureExpected, fetchedBytes: fetchedBytes, + transferredBytes: transferredBytes, + readBytes: 0, } } @@ -662,6 +684,7 @@ func (rc *timingReadCloser) Close() error { } if !rc.alreadyGotErr && err == nil { rc.duration.WithLabelValues(rc.op).Observe(time.Since(rc.start).Seconds()) + rc.transferredBytes.WithLabelValues(rc.op).Observe(float64(rc.readBytes)) rc.alreadyGotErr = true } return err @@ -670,6 +693,7 @@ func (rc *timingReadCloser) Close() error { func (rc *timingReadCloser) Read(b []byte) (n int, err error) { n, err = rc.ReadCloser.Read(b) rc.fetchedBytes.WithLabelValues(rc.op).Add(float64(n)) + rc.readBytes += int64(n) // Report metric just once. if !rc.alreadyGotErr && err != nil && err != io.EOF { if !rc.isFailureExpected(err) { diff --git a/objstore_test.go b/objstore_test.go index 92b781a8..ab5c5e5b 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -79,7 +79,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { testutil.Ok(t, m.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1")))) testutil.Ok(t, m.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2")))) - testutil.Ok(t, m.Upload(context.Background(), "dir/obj3", bytes.NewReader([]byte("3")))) + testutil.Ok(t, m.Upload(context.Background(), "dir/obj3", bytes.NewReader(bytes.Repeat([]byte("3"), 1024*1024)))) testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` # HELP objstore_bucket_operations_total Total number of all attempted operations against a bucket. @@ -110,17 +110,60 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { `), `objstore_bucket_operations_total`)) testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` - # HELP objstore_bucket_operation_fetched_bytes_total Total number of bytes fetched from bucket, per operation. + # HELP objstore_bucket_operation_fetched_bytes_total Total number of bytes fetched from bucket, per operation. # TYPE objstore_bucket_operation_fetched_bytes_total counter objstore_bucket_operation_fetched_bytes_total{bucket="",operation="attributes"} 0 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="delete"} 0 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="exists"} 0 - objstore_bucket_operation_fetched_bytes_total{bucket="",operation="get"} 3 + objstore_bucket_operation_fetched_bytes_total{bucket="",operation="get"} 1.048578e+06 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="get_range"} 0 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="iter"} 0 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="upload"} 0 `), `objstore_bucket_operation_fetched_bytes_total`)) + testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` + # HELP objstore_bucket_operation_transferred_bytes Number of bytes transferred from/to bucket per operation. + # TYPE objstore_bucket_operation_transferred_bytes histogram + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="32768"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="65536"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="131072"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="262144"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="524288"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="1.048576e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="2.097152e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="4.194304e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="8.388608e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="1.6777216e+07"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="3.3554432e+07"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="6.7108864e+07"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="1.34217728e+08"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="2.68435456e+08"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="5.36870912e+08"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="1.073741824e+09"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get",le="+Inf"} 3 + objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="get"} 1.048578e+06 + objstore_bucket_operation_transferred_bytes_count{bucket="",operation="get"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="32768"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="65536"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="131072"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="262144"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="524288"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="1.048576e+06"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="2.097152e+06"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="4.194304e+06"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="8.388608e+06"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="1.6777216e+07"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="3.3554432e+07"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="6.7108864e+07"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="1.34217728e+08"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="2.68435456e+08"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="5.36870912e+08"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="1.073741824e+09"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="+Inf"} 0 + objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="get_range"} 0 + objstore_bucket_operation_transferred_bytes_count{bucket="",operation="get_range"} 0 + `), `objstore_bucket_operation_transferred_bytes`)) + testutil.Ok(t, UploadDir(context.Background(), log.NewNopLogger(), m, tempDir, "/dir-copy", WithUploadConcurrency(10))) testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` @@ -143,7 +186,7 @@ func TestTimingTracingReader(t *testing.T) { tr := NopCloserWithSize(r) tr = newTimingReadCloser(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { return false - }, m.opsFetchedBytes) + }, m.opsFetchedBytes, m.opsTransferredBytes) size, err := TryToGetSize(tr) From d01642831b1f125125d60cc8c14b9bd8dd40f9ab Mon Sep 17 00:00:00 2001 From: Fred Tibbitts III <44442080+fredtibbitts@users.noreply.github.com> Date: Fri, 4 Aug 2023 04:40:00 -0400 Subject: [PATCH 003/130] OCI: OKE Workload Identity support (#64) * Add support for OKE workload identity to the OCI provider Signed-off-by: Fred Tibbitts * Update README based on review Signed-off-by: Fred Tibbitts * Add changelog entry Signed-off-by: Fred Tibbitts * go mod tidy Signed-off-by: Fred Tibbitts --------- Signed-off-by: Fred Tibbitts --- CHANGELOG.md | 1 + README.md | 16 +++++++++++++++- go.mod | 2 +- go.sum | 5 +++-- providers/oci/oci.go | 26 ++++++++++++++++++++------ 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 564faa91..b1385032 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. +- [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/README.md b/README.md index c7388894..11b773a4 100644 --- a/README.md +++ b/README.md @@ -578,7 +578,7 @@ prefix: "" ### Oracle Cloud Infrastructure Object Storage -To configure Oracle Cloud Infrastructure (OCI) Object Storage as Thanos Object Store, you need to provide appropriate authentication credentials to your OCI tenancy. The OCI object storage client implementation for Thanos supports either the default keypair or instance principal authentication. +To configure Oracle Cloud Infrastructure (OCI) Object Storage as a Thanos Object Store, you need to provide appropriate authentication credentials to your OCI tenancy. The OCI object storage client implementation for Thanos supports default keypair, instance principal, and OKE workload identity authentication. #### API Signing Key @@ -642,6 +642,20 @@ config: You can also include any of the optional configuration just like the example in `Default Provider`. +#### OKE Workload Identity Provider + +For Example: + +```yaml +type: OCI +config: + provider: "oke-workload-identity" + bucket: "" + region: "" +``` + +The `bucket` and `region` fields are required. The `region` field identifies the bucket region. + ##### HuaweiCloud OBS To use HuaweiCloud OBS as an object store, you should apply for a HuaweiCloud Account to create an object storage bucket at first. More details: [HuaweiCloud OBS](https://support.huaweicloud.com/obs/index.html) diff --git a/go.mod b/go.mod index 5bf1fd24..8dc98880 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/minio/minio-go/v7 v7.0.61 github.com/ncw/swift v1.0.53 github.com/opentracing/opentracing-go v1.2.0 - github.com/oracle/oci-go-sdk/v65 v65.13.0 + github.com/oracle/oci-go-sdk/v65 v65.41.1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.12.2 github.com/prometheus/common v0.36.0 diff --git a/go.sum b/go.sum index 21e06236..21367dc7 100644 --- a/go.sum +++ b/go.sum @@ -326,8 +326,8 @@ github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/oracle/oci-go-sdk/v65 v65.13.0 h1:0+9ea5goYfhI3/MPfbIQU6yzHYWE6sCk6VuUepxk5Nk= -github.com/oracle/oci-go-sdk/v65 v65.13.0/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE= +github.com/oracle/oci-go-sdk/v65 v65.41.1 h1:+lbosOyNiib3TGJDvLq1HwEAuFqkOjPJDIkyxM15WdQ= +github.com/oracle/oci-go-sdk/v65 v65.41.1/go.mod h1:MXMLMzHnnd9wlpgadPkdlkZ9YrwQmCOmbX5kjVEJodw= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -597,6 +597,7 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 2be3210a..b6c9680a 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "os" "strings" "testing" "time" @@ -30,9 +31,10 @@ const DirDelim = "/" type Provider string const ( - DefaultConfigProvider = Provider("default") - InstancePrincipalConfigProvider = Provider("instance-principal") - RawConfigProvider = Provider("raw") + defaultConfigProvider = Provider("default") + instancePrincipalConfigProvider = Provider("instance-principal") + rawConfigProvider = Provider("raw") + okeWorkloadIdentityConfigProvider = Provider("oke-workload-identity") ) var DefaultConfig = Config{ @@ -295,19 +297,31 @@ func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { provider := Provider(strings.ToLower(config.Provider)) level.Info(logger).Log("msg", "creating OCI client", "provider", provider) switch provider { - case DefaultConfigProvider: + case defaultConfigProvider: configurationProvider = common.DefaultConfigProvider() - case InstancePrincipalConfigProvider: + case instancePrincipalConfigProvider: configurationProvider, err = auth.InstancePrincipalConfigurationProvider() if err != nil { return nil, errors.Wrapf(err, "unable to create OCI instance principal config provider") } - case RawConfigProvider: + case rawConfigProvider: if err := config.validateConfig(); err != nil { return nil, errors.Wrapf(err, "invalid oci configurations") } configurationProvider = common.NewRawConfigurationProvider(config.Tenancy, config.User, config.Region, config.Fingerprint, config.PrivateKey, &config.Passphrase) + case okeWorkloadIdentityConfigProvider: + if err := os.Setenv(auth.ResourcePrincipalVersionEnvVar, auth.ResourcePrincipalVersion2_2); err != nil { + return nil, errors.Wrapf(err, "unable to set environment variable: %s", auth.ResourcePrincipalVersionEnvVar) + } + if err := os.Setenv(auth.ResourcePrincipalRegionEnvVar, config.Region); err != nil { + return nil, errors.Wrapf(err, "unable to set environment variable: %s", auth.ResourcePrincipalRegionEnvVar) + } + + configurationProvider, err = auth.OkeWorkloadIdentityConfigurationProvider() + if err != nil { + return nil, errors.Wrapf(err, "unable to create OKE workload identity config provider") + } default: return nil, errors.Wrapf(err, fmt.Sprintf("unsupported OCI provider: %s", provider)) } From a3fe9c5394329eaddaaf7a79758b377449f82451 Mon Sep 17 00:00:00 2001 From: Christian Simon Date: Fri, 4 Aug 2023 09:44:36 +0100 Subject: [PATCH 004/130] Fix overlapping metric registrations (#67) This fixes a problem when multiple metrics bucket are regsitered with different names. Also adds a test so this won't be happening in the future. Signed-off-by: Christian Simon --- objstore.go | 14 +++++++------- objstore_test.go | 8 ++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/objstore.go b/objstore.go index 2a0fe4dc..61e3dfb4 100644 --- a/objstore.go +++ b/objstore.go @@ -438,10 +438,11 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, }, []string{"operation"}), - lastSuccessfulUploadTime: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ - Name: "objstore_bucket_last_successful_upload_time", - Help: "Second timestamp of the last successful upload to the bucket.", - }, []string{"bucket"}), + lastSuccessfulUploadTime: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "objstore_bucket_last_successful_upload_time", + Help: "Second timestamp of the last successful upload to the bucket.", + ConstLabels: prometheus.Labels{"bucket": name}, + }), } for _, op := range []string{ OpIter, @@ -465,7 +466,6 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu } { bkt.opsTransferredBytes.WithLabelValues(op) } - bkt.lastSuccessfulUploadTime.WithLabelValues(b.Name()) return bkt } @@ -479,7 +479,7 @@ type metricBucket struct { opsFetchedBytes *prometheus.CounterVec opsTransferredBytes *prometheus.HistogramVec opsDuration *prometheus.HistogramVec - lastSuccessfulUploadTime *prometheus.GaugeVec + lastSuccessfulUploadTime prometheus.Gauge } func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { @@ -599,7 +599,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err } return err } - b.lastSuccessfulUploadTime.WithLabelValues(b.bkt.Name()).SetToCurrentTime() + b.lastSuccessfulUploadTime.SetToCurrentTime() b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return nil } diff --git a/objstore_test.go b/objstore_test.go index ab5c5e5b..4b105281 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -72,6 +72,14 @@ func TestMetricBucket_Close(t *testing.T) { testutil.Assert(t, promtest.ToFloat64(bkt.lastSuccessfulUploadTime) > lastUpload) } +// TestMetricBucket_MultipleClients tests that the metrics from two different buckets clients are not conflicting with each other. +func TestMetricBucket_Multiple_Clients(t *testing.T) { + reg := prometheus.NewPedanticRegistry() + + WrapWithMetrics(NewInMemBucket(), reg, "abc") + WrapWithMetrics(NewInMemBucket(), reg, "def") +} + func TestDownloadUploadDirConcurrency(t *testing.T) { r := prometheus.NewRegistry() m := WrapWithMetrics(NewInMemBucket(), r, "") From c042a6a16c582d2399451a06fcbc8afdc43b8faf Mon Sep 17 00:00:00 2001 From: Matej Gera <38492574+matej-g@users.noreply.github.com> Date: Fri, 4 Aug 2023 10:48:40 +0200 Subject: [PATCH 005/130] GCS: Bump dependency version (#70) * Bump GCS dependency versions Signed-off-by: Matej Gera * Add CHANGELOG Signed-off-by: Matej Gera --------- Signed-off-by: Matej Gera --- CHANGELOG.md | 1 + go.mod | 32 +++-- go.sum | 279 ++++++-------------------------------- providers/gcs/gcs_test.go | 3 +- 4 files changed, 61 insertions(+), 254 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1385032..84b2cccd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,5 +34,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#39](https://github.com/thanos-io/objstore/pull/39) COS: Upgrade cos sdk version to `v0.7.40`. - [#35](https://github.com/thanos-io/objstore/pull/35) Azure: Update Azure SDK and fix breaking changes. - [#65](https://github.com/thanos-io/objstore/pull/65) *: Upgrade minio-go version to `v7.0.61`. +- [#70](https://github.com/thanos-io/objstore/pull/70) GCS: Update cloud.google.com/go/storage version to `v1.27.0`. ### Removed diff --git a/go.mod b/go.mod index 8dc98880..dee14caa 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/thanos-io/objstore go 1.18 require ( - cloud.google.com/go/storage v1.10.0 + cloud.google.com/go/storage v1.27.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go-v2 v1.16.0 github.com/aws/aws-sdk-go-v2/config v1.15.1 @@ -24,17 +24,18 @@ require ( go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 - google.golang.org/api v0.80.0 + golang.org/x/oauth2 v0.4.0 + golang.org/x/sync v0.1.0 + google.golang.org/api v0.103.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.100.2 // indirect - cloud.google.com/go/compute v1.6.1 // indirect - cloud.google.com/go/iam v0.3.0 // indirect + cloud.google.com/go v0.107.0 // indirect + cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v0.8.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect @@ -50,7 +51,7 @@ require ( github.com/aws/smithy-go v1.11.1 // indirect github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -59,9 +60,11 @@ require ( github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/gax-go/v2 v2.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect @@ -81,17 +84,16 @@ require ( github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect - github.com/stretchr/objx v0.2.0 // indirect - go.opencensus.io v0.23.0 // indirect + go.opencensus.io v0.24.0 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect - golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220524023933-508584e28198 // indirect - google.golang.org/grpc v1.46.2 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/grpc v1.53.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 21367dc7..860aca72 100644 --- a/go.sum +++ b/go.sum @@ -13,37 +13,23 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= +cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -52,8 +38,9 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= @@ -67,7 +54,6 @@ github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= @@ -79,7 +65,6 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible h1:9gWa46nstkJ9miBReJcN8Gq34cBFbzSpQZVVT9N09TM= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go-v2 v1.16.0 h1:cBAYjiiexRAg9v2z9vb6IdxAa7ef4KCtjW7w7e3GxGo= github.com/aws/aws-sdk-go-v2 v1.16.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI= github.com/aws/aws-sdk-go-v2/config v1.15.1 h1:hTIZFepYESYyowQUBo47lu69WSxsYqGUILY9Nu8+7pY= @@ -111,10 +96,10 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -122,14 +107,6 @@ github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -144,16 +121,9 @@ github.com/efficientgo/e2e v0.13.1-0.20220922081603-45de9fc588a8/go.mod h1:Hi+sz github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -187,8 +157,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -204,10 +172,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -221,10 +187,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= @@ -232,9 +197,7 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -242,32 +205,22 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0 h1:nRJtk3y8Fm770D42QV6T90ZnvFZyk7agSo3Q+Z9p3WI= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= @@ -360,7 +313,6 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -373,17 +325,19 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= @@ -392,21 +346,17 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7y github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -439,8 +389,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -449,9 +397,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -480,21 +425,10 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -502,21 +436,10 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -525,11 +448,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -561,40 +482,16 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -607,8 +504,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= @@ -658,23 +553,12 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -691,28 +575,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.80.0 h1:IQWaGVCYnsm4MO3hh+WtSXMzMzuyFx/fuR8qkN3A0Qo= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -744,57 +608,14 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220524023933-508584e28198 h1:a1g7i05I2vUwq5eYrmxBJy6rPbw/yo7WzzwPJmcC0P4= -google.golang.org/genproto v0.0.0-20220524023933-508584e28198/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -807,26 +628,9 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2 h1:u+MLGgVf7vRdjEYZ8wDFhAVNmhkbJ5hmrA1LMWK1CAQ= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -839,9 +643,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -854,7 +657,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -863,6 +665,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 912bcc2b..283de6b6 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -41,5 +41,6 @@ func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) { // We expect an error when reading back. _, err = io.ReadAll(reader) - testutil.Equals(t, io.ErrUnexpectedEOF, err) + testutil.NotOk(t, err) + testutil.Equals(t, "storage: partial request not satisfied", err.Error()) } From f10e813caaa7332aba4d1ffa66669326048c8b02 Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Tue, 15 Aug 2023 22:33:14 -0700 Subject: [PATCH 006/130] Implementing IsAccessDeniedErr method on the objstore Signed-off-by: Alan Protasio --- CHANGELOG.md | 2 +- README.md | 3 ++- inmem.go | 4 ++-- objstore.go | 8 ++++---- prefixed_bucket.go | 6 +++--- providers/azure/azure.go | 9 ++++++--- providers/bos/bos.go | 4 ++-- providers/cos/cos.go | 4 ++-- providers/filesystem/filesystem.go | 4 ++-- providers/gcs/gcs.go | 9 +++++++-- providers/obs/obs.go | 4 ++-- providers/oci/oci.go | 8 ++++++-- providers/oss/oss.go | 10 ++++++++-- providers/s3/s3.go | 10 +++------- providers/swift/swift.go | 6 +++--- testing.go | 4 ++-- tracing/opentelemetry/opentelemetry.go | 4 ++-- tracing/opentracing/opentracing.go | 4 ++-- 18 files changed, 59 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84b2cccd..23e92b8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,5 +35,5 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#35](https://github.com/thanos-io/objstore/pull/35) Azure: Update Azure SDK and fix breaking changes. - [#65](https://github.com/thanos-io/objstore/pull/65) *: Upgrade minio-go version to `v7.0.61`. - [#70](https://github.com/thanos-io/objstore/pull/70) GCS: Update cloud.google.com/go/storage version to `v1.27.0`. - +- [#71](https://github.com/thanos-io/objstore/pull/71) Replace method `IsCustomerManagedKeyError` for a more generic `IsAccessDeniedErr` on the bucket interface. ### Removed diff --git a/README.md b/README.md index 11b773a4..c9dbe34d 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,8 @@ type BucketReader interface { // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. IsObjNotFoundErr(err error) bool - // IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. + // IsAccessDeniedErr returns true if access to object is denied. + IsAccessDeniedErr(err error) bool ``` Those interfaces represent the object storage operations your code can use from `objstore` clients. diff --git a/inmem.go b/inmem.go index aee4aec6..3f6f35e9 100644 --- a/inmem.go +++ b/inmem.go @@ -207,8 +207,8 @@ func (b *InMemBucket) IsObjNotFoundErr(err error) bool { return errors.Is(err, errNotFound) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *InMemBucket) IsCustomerManagedKeyError(_ error) bool { +// IsAccessDeniedErr returns true if access to object is denied. +func (b *InMemBucket) IsAccessDeniedErr(err error) bool { return false } diff --git a/objstore.go b/objstore.go index 61e3dfb4..b9b56bf4 100644 --- a/objstore.go +++ b/objstore.go @@ -85,8 +85,8 @@ type BucketReader interface { // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. IsObjNotFoundErr(err error) bool - // IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. - IsCustomerManagedKeyError(err error) bool + // IsAccessDeniedErr returns true if acces to object is denied. + IsAccessDeniedErr(err error) bool // Attributes returns information about the specified object. Attributes(ctx context.Context, name string) (ObjectAttributes, error) @@ -624,8 +624,8 @@ func (b *metricBucket) IsObjNotFoundErr(err error) bool { return b.bkt.IsObjNotFoundErr(err) } -func (b *metricBucket) IsCustomerManagedKeyError(err error) bool { - return b.bkt.IsCustomerManagedKeyError(err) +func (b *metricBucket) IsAccessDeniedErr(err error) bool { + return b.bkt.IsAccessDeniedErr(err) } func (b *metricBucket) Close() error { diff --git a/prefixed_bucket.go b/prefixed_bucket.go index 41448011..f2b71434 100644 --- a/prefixed_bucket.go +++ b/prefixed_bucket.go @@ -74,9 +74,9 @@ func (p *PrefixedBucket) IsObjNotFoundErr(err error) bool { return p.bkt.IsObjNotFoundErr(err) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (p *PrefixedBucket) IsCustomerManagedKeyError(err error) bool { - return p.bkt.IsCustomerManagedKeyError(err) +// IsAccessDeniedErr returns true if access to object is denied. +func (p *PrefixedBucket) IsAccessDeniedErr(err error) bool { + return p.bkt.IsAccessDeniedErr(err) } // Attributes returns information about the specified object. diff --git a/providers/azure/azure.go b/providers/azure/azure.go index a5f41ed1..376fb629 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -235,9 +235,12 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return bloberror.HasCode(err, bloberror.BlobNotFound) || bloberror.HasCode(err, bloberror.InvalidURI) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(_ error) bool { - return false +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(err error) bool { + if err == nil { + return false + } + return bloberror.HasCode(err, bloberror.AuthorizationPermissionMismatch) || bloberror.HasCode(err, bloberror.InsufficientAccountPermissions) } func (b *Bucket) getBlobReader(ctx context.Context, name string, httpRange blob.HTTPRange) (io.ReadCloser, error) { diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 2a9b5cae..72e1b1e0 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -287,8 +287,8 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return false } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(_ error) bool { +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(_ error) bool { return false } diff --git a/providers/cos/cos.go b/providers/cos/cos.go index 36ecead5..e518cae2 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -364,8 +364,8 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { } } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(_ error) bool { +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(_ error) bool { return false } diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 8ccd33b1..21c70485 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -258,8 +258,8 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return os.IsNotExist(errors.Cause(err)) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(_ error) bool { +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(_ error) bool { return false } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 8b107c83..5ea45c7e 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -19,6 +19,8 @@ import ( "golang.org/x/oauth2/google" "google.golang.org/api/iterator" "google.golang.org/api/option" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "gopkg.in/yaml.v2" "github.com/thanos-io/objstore" @@ -188,8 +190,11 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return errors.Is(err, storage.ErrObjectNotExist) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(_ error) bool { +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(err error) bool { + if s, ok := status.FromError(err); ok && s.Code() == codes.PermissionDenied { + return true + } return false } diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 1bc58aef..35a04c80 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -327,8 +327,8 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return false } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(_ error) bool { +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(_ error) bool { return false } diff --git a/providers/oci/oci.go b/providers/oci/oci.go index b6c9680a..2db35461 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -227,8 +227,12 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return false } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(_ error) bool { +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(err error) bool { + failure, isServiceError := common.IsServiceError(err) + if isServiceError { + return failure.GetHTTPStatusCode() == http.StatusForbidden + } return false } diff --git a/providers/oss/oss.go b/providers/oss/oss.go index 08fcf1e3..e7e3a648 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -379,7 +379,13 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return false } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(_ error) bool { +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(err error) bool { + switch aliErr := errors.Cause(err).(type) { + case alioss.ServiceError: + if aliErr.StatusCode == http.StatusForbidden { + return true + } + } return false } diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 337bd0d8..f92d3973 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -98,9 +98,6 @@ const ( // Storage class header. amzStorageClass = "X-Amz-Storage-Class" - - // amzKmsKeyAccessDeniedErrorMessage is the error message returned by s3 when the permissions to the KMS key is revoked. - amzKmsKeyAccessDeniedErrorMessage = "The ciphertext refers to a customer master key that does not exist, does not exist in this region, or you are not allowed to access." ) var DefaultConfig = Config{ @@ -541,10 +538,9 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return minio.ToErrorResponse(errors.Cause(err)).Code == "NoSuchKey" } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(err error) bool { - errResponse := minio.ToErrorResponse(errors.Cause(err)) - return errResponse.Code == "AccessDenied" && errResponse.Message == amzKmsKeyAccessDeniedErrorMessage +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(err error) bool { + return minio.ToErrorResponse(errors.Cause(err)).Code == "AccessDenied" } func (b *Bucket) Close() error { return nil } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index c24d03fd..9bfa3cf8 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -290,9 +290,9 @@ func (c *Container) IsObjNotFoundErr(err error) bool { return errors.Is(err, swift.ObjectNotFound) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Container) IsCustomerManagedKeyError(_ error) bool { - return false +// IsAccessDeniedErr returns true if access to object is denied. +func (c *Container) IsAccessDeniedErr(err error) bool { + return errors.Is(err, swift.Forbidden) } // Upload writes the contents of the reader as an object into the container. diff --git a/testing.go b/testing.go index 4e41b278..b8e3744c 100644 --- a/testing.go +++ b/testing.go @@ -309,6 +309,6 @@ func (d *delayingBucket) IsObjNotFoundErr(err error) bool { return d.bkt.IsObjNotFoundErr(err) } -func (d *delayingBucket) IsCustomerManagedKeyError(err error) bool { - return d.bkt.IsCustomerManagedKeyError(err) +func (d *delayingBucket) IsAccessDeniedErr(err error) bool { + return d.bkt.IsAccessDeniedErr(err) } diff --git a/tracing/opentelemetry/opentelemetry.go b/tracing/opentelemetry/opentelemetry.go index 0e7a279a..f65b0f0e 100644 --- a/tracing/opentelemetry/opentelemetry.go +++ b/tracing/opentelemetry/opentelemetry.go @@ -128,8 +128,8 @@ func (t TracingBucket) IsObjNotFoundErr(err error) bool { return t.bkt.IsObjNotFoundErr(err) } -func (t TracingBucket) IsCustomerManagedKeyError(err error) bool { - return t.bkt.IsCustomerManagedKeyError(err) +func (t TracingBucket) IsAccessDeniedErr(err error) bool { + return t.bkt.IsAccessDeniedErr(err) } func (t TracingBucket) WithExpectedErrs(expectedFunc objstore.IsOpFailureExpectedFunc) objstore.Bucket { diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index 8174afb1..8b99e304 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -124,8 +124,8 @@ func (t TracingBucket) IsObjNotFoundErr(err error) bool { return t.bkt.IsObjNotFoundErr(err) } -func (t TracingBucket) IsCustomerManagedKeyError(err error) bool { - return t.bkt.IsCustomerManagedKeyError(err) +func (t TracingBucket) IsAccessDeniedErr(err error) bool { + return t.bkt.IsAccessDeniedErr(err) } func (t TracingBucket) WithExpectedErrs(expectedFunc objstore.IsOpFailureExpectedFunc) objstore.Bucket { From 900ce69cb9cdfe2536d897b511109de7604901a3 Mon Sep 17 00:00:00 2001 From: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> Date: Fri, 25 Aug 2023 15:36:52 +0300 Subject: [PATCH 007/130] Added file path in erros from DownloadFile Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> --- objstore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/objstore.go b/objstore.go index b9b56bf4..84eafce3 100644 --- a/objstore.go +++ b/objstore.go @@ -315,7 +315,7 @@ func DownloadFile(ctx context.Context, logger log.Logger, bkt BucketReader, src, f, err := os.Create(dst) if err != nil { - return errors.Wrap(err, "create file") + return errors.Wrapf(err, "create file %s", dst) } defer func() { if err != nil { @@ -327,7 +327,7 @@ func DownloadFile(ctx context.Context, logger log.Logger, bkt BucketReader, src, defer logerrcapture.Do(logger, f.Close, "close block's output file") if _, err = io.Copy(f, rc); err != nil { - return errors.Wrap(err, "copy object to file") + return errors.Wrapf(err, "copy object to file %s", src) } return nil } From 169cb53125986de20372445f292826bad704fa40 Mon Sep 17 00:00:00 2001 From: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> Date: Fri, 25 Aug 2023 15:42:39 +0300 Subject: [PATCH 008/130] CHANGELOG.md Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23e92b8c..be680688 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. +- [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From 1b257a36f9a3befcea136e99c36472bc38d205de Mon Sep 17 00:00:00 2001 From: Nicklos Holik <2022214+nholik@users.noreply.github.com> Date: Tue, 29 Aug 2023 11:21:04 -0400 Subject: [PATCH 009/130] azure: Support using connection string authentication (#51) Support using connection string authentication for azure storage Signed-off-by: Nick Holik Signed-off-by: Kemal Akkoyun Co-authored-by: Kemal Akkoyun --- CHANGELOG.md | 1 + README.md | 3 +++ providers/azure/azure.go | 27 ++++++++++++++++++--------- providers/azure/azure_test.go | 10 ++++++++++ providers/azure/helpers.go | 10 ++++++++++ 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be680688..32d0f742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile +- [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/README.md b/README.md index c9dbe34d..2a155271 100644 --- a/README.md +++ b/README.md @@ -419,6 +419,7 @@ type: AZURE config: storage_account: "" storage_account_key: "" + storage_connection_string: "" container: "" endpoint: "" user_assigned_id: "" @@ -454,6 +455,8 @@ If `msi_resource` is used, authentication is done via system-assigned managed id If `user_assigned_id` is used, authentication is done via user-assigned managed identity. When using `user_assigned_id` the `msi_resource` defaults to `https://.` +If `storage_connection_string` is set, the values of `storage_account` and `endpoint` values will not be used. Use this method over `storage_account_key` if you need to authenticate via a SAS token. + The generic `max_retries` will be used as value for the `pipeline_config`'s `max_tries` and `reader_config`'s `max_retry_requests`. For more control, `max_retries` could be ignored (0) and one could set specific retry values. ##### OpenStack Swift diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 376fb629..a72f3052 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -44,15 +44,16 @@ var DefaultConfig = Config{ // Config Azure storage configuration. type Config struct { - StorageAccountName string `yaml:"storage_account"` - StorageAccountKey string `yaml:"storage_account_key"` - ContainerName string `yaml:"container"` - Endpoint string `yaml:"endpoint"` - UserAssignedID string `yaml:"user_assigned_id"` - MaxRetries int `yaml:"max_retries"` - ReaderConfig ReaderConfig `yaml:"reader_config"` - PipelineConfig PipelineConfig `yaml:"pipeline_config"` - HTTPConfig exthttp.HTTPConfig `yaml:"http_config"` + StorageAccountName string `yaml:"storage_account"` + StorageAccountKey string `yaml:"storage_account_key"` + StorageConnectionString string `yaml:"storage_connection_string"` + ContainerName string `yaml:"container"` + Endpoint string `yaml:"endpoint"` + UserAssignedID string `yaml:"user_assigned_id"` + MaxRetries int `yaml:"max_retries"` + ReaderConfig ReaderConfig `yaml:"reader_config"` + PipelineConfig PipelineConfig `yaml:"pipeline_config"` + HTTPConfig exthttp.HTTPConfig `yaml:"http_config"` // Deprecated: Is automatically set by the Azure SDK. MSIResource string `yaml:"msi_resource"` @@ -76,6 +77,14 @@ func (conf *Config) validate() error { errMsg = append(errMsg, "user_assigned_id cannot be set when using storage_account_key authentication") } + if conf.UserAssignedID != "" && conf.StorageConnectionString != "" { + errMsg = append(errMsg, "user_assigned_id cannot be set when using storage_connection_string authentication") + } + + if conf.StorageAccountKey != "" && conf.StorageConnectionString != "" { + errMsg = append(errMsg, "storage_account_key and storage_connection_string cannot both be set") + } + if conf.StorageAccountName == "" { errMsg = append(errMsg, "storage_account_name is required but not configured") } diff --git a/providers/azure/azure_test.go b/providers/azure/azure_test.go index 4bc6785e..c49695ab 100644 --- a/providers/azure/azure_test.go +++ b/providers/azure/azure_test.go @@ -132,6 +132,16 @@ container: "MyContainer"`), wantFailParse: false, wantFailValidate: false, }, + { + name: "Valid User Assigned and Connection String set", + config: []byte(`storage_account: "myAccount" +storage_account_key: "" +user_assigned_id: "1234-56578678-655" +storage_connection_string: "myConnectionString" +container: "MyContainer"`), + wantFailParse: false, + wantFailValidate: true, + }, } func TestConfig_validate(t *testing.T) { diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index b76154d6..7b4a5fbe 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -38,6 +38,16 @@ func getContainerClient(conf Config) (*container.Client, error) { Transport: &http.Client{Transport: dt}, }, } + + // Use connection string if set + if conf.StorageConnectionString != "" { + containerClient, err := container.NewClientFromConnectionString(conf.StorageConnectionString, conf.ContainerName, opt) + if err != nil { + return nil, err + } + return containerClient, nil + } + containerURL := fmt.Sprintf("https://%s.%s/%s", conf.StorageAccountName, conf.Endpoint, conf.ContainerName) // Use shared keys if set From bbb31144d73d31dfccc73597f6b3fc40f9da03a5 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 21 Jul 2023 17:49:27 +0100 Subject: [PATCH 010/130] [FEAT] Add uploaded TSDB bytes metric Signed-off-by: rita.canavarro --- objstore.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/objstore.go b/objstore.go index 84eafce3..9d704783 100644 --- a/objstore.go +++ b/objstore.go @@ -443,6 +443,12 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu Help: "Second timestamp of the last successful upload to the bucket.", ConstLabels: prometheus.Labels{"bucket": name}, }), + + opsUploadedBytes: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Name: "objstore_bucket_operation_uploaded_bytes_total", + Help: "Total number of bytes uploaded from TSDB block, per operation.", + ConstLabels: prometheus.Labels{"bucket": name}, + }, []string{"operation"}), } for _, op := range []string{ OpIter, @@ -457,6 +463,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsFailures.WithLabelValues(op) bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) + bkt.opsUploadedBytes.WithLabelValues(op) } // fetched bytes only relevant for get and getrange for _, op := range []string{ @@ -478,6 +485,7 @@ type metricBucket struct { opsFetchedBytes *prometheus.CounterVec opsTransferredBytes *prometheus.HistogramVec + opsUploadedBytes *prometheus.CounterVec opsDuration *prometheus.HistogramVec lastSuccessfulUploadTime prometheus.Gauge } From 8f0a6ab88312ed85a6da36ea33e3d60ab0cccc7b Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 21 Jul 2023 17:54:41 +0100 Subject: [PATCH 011/130] [FIX] Tests Signed-off-by: rita.canavarro --- objstore.go | 7 ++++++- objstore_test.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/objstore.go b/objstore.go index 9d704783..67fd0d16 100644 --- a/objstore.go +++ b/objstore.go @@ -497,6 +497,7 @@ func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { opsFailures: b.opsFailures, opsFetchedBytes: b.opsFetchedBytes, opsTransferredBytes: b.opsTransferredBytes, + opsUploadedBytes: b.opsUploadedBytes, isOpFailureExpected: fn, opsDuration: b.opsDuration, lastSuccessfulUploadTime: b.lastSuccessfulUploadTime, @@ -555,6 +556,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, + b.opsUploadedBytes, ), nil } @@ -577,6 +579,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, + b.opsUploadedBytes, ), nil } @@ -659,9 +662,10 @@ type timingReadCloser struct { isFailureExpected IsOpFailureExpectedFunc fetchedBytes *prometheus.CounterVec transferredBytes *prometheus.HistogramVec + uploadedBytes *prometheus.CounterVec } -func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { +func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, uploadedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -678,6 +682,7 @@ func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramV fetchedBytes: fetchedBytes, transferredBytes: transferredBytes, readBytes: 0, + uploadedBytes: uploadedBytes, } } diff --git a/objstore_test.go b/objstore_test.go index 4b105281..cc5ef021 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -194,7 +194,7 @@ func TestTimingTracingReader(t *testing.T) { tr := NopCloserWithSize(r) tr = newTimingReadCloser(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { return false - }, m.opsFetchedBytes, m.opsTransferredBytes) + }, m.opsFetchedBytes, m.opsTransferredBytes, m.opsUploadedBytes) size, err := TryToGetSize(tr) From 0b3153e1964d91cea5b2b321f7a948bf88b38869 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 21 Jul 2023 17:54:53 +0100 Subject: [PATCH 012/130] [FEAT] Changelog Signed-off-by: rita.canavarro --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32d0f742..f4fd871d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. +- [#66](https://github.com/thanos-io/objstore/pull/66) Add for the number of total uploaded blocks. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. From 54d6fb552844dbf23a3253eccc994d4aeb4180e9 Mon Sep 17 00:00:00 2001 From: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> Date: Tue, 25 Jul 2023 09:15:55 +0100 Subject: [PATCH 013/130] Update CHANGELOG.md Co-authored-by: Bartlomiej Plotka Signed-off-by: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4fd871d..7231f68a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. -- [#66](https://github.com/thanos-io/objstore/pull/66) Add for the number of total uploaded blocks. +- [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_written_bytes_total` that counts the number of total bytes per written to the bucket operation. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. From a3d95a12296462317d60801b3fe8042690932c4e Mon Sep 17 00:00:00 2001 From: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> Date: Tue, 25 Jul 2023 09:16:37 +0100 Subject: [PATCH 014/130] Update objstore.go Co-authored-by: Bartlomiej Plotka Signed-off-by: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> --- objstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objstore.go b/objstore.go index 67fd0d16..84bce17d 100644 --- a/objstore.go +++ b/objstore.go @@ -445,7 +445,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu }), opsUploadedBytes: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ - Name: "objstore_bucket_operation_uploaded_bytes_total", + Name: "objstore_bucket_operation_written_bytes_total", Help: "Total number of bytes uploaded from TSDB block, per operation.", ConstLabels: prometheus.Labels{"bucket": name}, }, []string{"operation"}), From 64d79d1dedf45b3cfdf8dfaf5106f9272e708e25 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 25 Jul 2023 14:05:53 +0100 Subject: [PATCH 015/130] [FEAT] Increment metric Signed-off-by: rita.canavarro --- inmem.go | 9 +++---- objstore.go | 31 +++++++++++++------------ objstore_test.go | 22 +++++++++++------- prefixed_bucket.go | 2 +- prefixed_bucket_test.go | 9 ++++--- providers/bos/bos.go | 20 ++++++++-------- providers/cos/cos.go | 18 +++++++------- providers/filesystem/filesystem.go | 15 ++++++------ providers/filesystem/filesystem_test.go | 8 ++++--- providers/gcs/gcs.go | 9 +++---- providers/obs/obs.go | 16 ++++++------- providers/oci/oci.go | 6 ++--- providers/oss/oss.go | 16 ++++++------- providers/s3/s3.go | 8 +++---- providers/s3/s3_e2e_test.go | 3 ++- providers/swift/swift.go | 15 ++++++------ testing.go | 29 +++++++++++++++-------- tracing/opentelemetry/opentelemetry.go | 2 +- tracing/opentracing/opentracing.go | 4 ++-- 19 files changed, 134 insertions(+), 108 deletions(-) diff --git a/inmem.go b/inmem.go index 3f6f35e9..45d13ef3 100644 --- a/inmem.go +++ b/inmem.go @@ -175,19 +175,20 @@ func (b *InMemBucket) Attributes(_ context.Context, name string) (ObjectAttribut } // Upload writes the file specified in src to into the memory. -func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) error { +func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { b.mtx.Lock() defer b.mtx.Unlock() body, err := io.ReadAll(r) if err != nil { - return err + return 0, err } b.objects[name] = body + size := int64(len(body)) b.attrs[name] = ObjectAttributes{ - Size: int64(len(body)), + Size: size, LastModified: time.Now(), } - return nil + return size, nil } // Delete removes all data prefixed with the dir. diff --git a/objstore.go b/objstore.go index 84bce17d..c9ec22d2 100644 --- a/objstore.go +++ b/objstore.go @@ -42,7 +42,7 @@ type Bucket interface { // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. - Upload(ctx context.Context, name string, r io.Reader) error + Upload(ctx context.Context, name string, r io.Reader) (int64, error) // Delete removes the object with the given name. // If object does not exist in the moment of deletion, Delete should throw error. @@ -285,7 +285,7 @@ func UploadFile(ctx context.Context, logger log.Logger, bkt Bucket, src, dst str } defer logerrcapture.Do(logger, r.Close, "close file %s", src) - if err := bkt.Upload(ctx, dst, r); err != nil { + if _, err := bkt.Upload(ctx, dst, r); err != nil { return errors.Wrapf(err, "upload file %s as %s", src, dst) } level.Debug(logger).Log("msg", "uploaded file", "from", src, "dst", dst, "bucket", bkt.Name()) @@ -444,7 +444,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu ConstLabels: prometheus.Labels{"bucket": name}, }), - opsUploadedBytes: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + opsWrittenBytes: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Name: "objstore_bucket_operation_written_bytes_total", Help: "Total number of bytes uploaded from TSDB block, per operation.", ConstLabels: prometheus.Labels{"bucket": name}, @@ -463,7 +463,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsFailures.WithLabelValues(op) bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) - bkt.opsUploadedBytes.WithLabelValues(op) + bkt.opsWrittenBytes.WithLabelValues(op) } // fetched bytes only relevant for get and getrange for _, op := range []string{ @@ -485,7 +485,7 @@ type metricBucket struct { opsFetchedBytes *prometheus.CounterVec opsTransferredBytes *prometheus.HistogramVec - opsUploadedBytes *prometheus.CounterVec + opsWrittenBytes *prometheus.CounterVec opsDuration *prometheus.HistogramVec lastSuccessfulUploadTime prometheus.Gauge } @@ -497,7 +497,7 @@ func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { opsFailures: b.opsFailures, opsFetchedBytes: b.opsFetchedBytes, opsTransferredBytes: b.opsTransferredBytes, - opsUploadedBytes: b.opsUploadedBytes, + opsWrittenBytes: b.opsWrittenBytes, isOpFailureExpected: fn, opsDuration: b.opsDuration, lastSuccessfulUploadTime: b.lastSuccessfulUploadTime, @@ -556,7 +556,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, - b.opsUploadedBytes, + b.opsWrittenBytes, ), nil } @@ -579,7 +579,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, - b.opsUploadedBytes, + b.opsWrittenBytes, ), nil } @@ -599,20 +599,21 @@ func (b *metricBucket) Exists(ctx context.Context, name string) (bool, error) { return ok, nil } -func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { const op = OpUpload b.ops.WithLabelValues(op).Inc() start := time.Now() - if err := b.bkt.Upload(ctx, name, r); err != nil { + writtenBytes, err := b.bkt.Upload(ctx, name, r) + if err != nil { if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } - return err + return 0, err } b.lastSuccessfulUploadTime.SetToCurrentTime() b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) - return nil + return writtenBytes, nil } func (b *metricBucket) Delete(ctx context.Context, name string) error { @@ -662,10 +663,10 @@ type timingReadCloser struct { isFailureExpected IsOpFailureExpectedFunc fetchedBytes *prometheus.CounterVec transferredBytes *prometheus.HistogramVec - uploadedBytes *prometheus.CounterVec + writtenBytes *prometheus.CounterVec } -func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, uploadedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { +func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, writtenBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -682,7 +683,7 @@ func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramV fetchedBytes: fetchedBytes, transferredBytes: transferredBytes, readBytes: 0, - uploadedBytes: uploadedBytes, + writtenBytes: writtenBytes, } } diff --git a/objstore_test.go b/objstore_test.go index cc5ef021..26b86ab4 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -85,9 +85,12 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), r, "") tempDir := t.TempDir() - testutil.Ok(t, m.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1")))) - testutil.Ok(t, m.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2")))) - testutil.Ok(t, m.Upload(context.Background(), "dir/obj3", bytes.NewReader(bytes.Repeat([]byte("3"), 1024*1024)))) + _, err := m.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1"))) + testutil.Ok(t, err) + _, err = m.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2"))) + testutil.Ok(t, err) + _, err = m.Upload(context.Background(), "dir/obj3", bytes.NewReader(bytes.Repeat([]byte("3"), 1024*1024))) + testutil.Ok(t, err) testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` # HELP objstore_bucket_operations_total Total number of all attempted operations against a bucket. @@ -194,7 +197,7 @@ func TestTimingTracingReader(t *testing.T) { tr := NopCloserWithSize(r) tr = newTimingReadCloser(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { return false - }, m.opsFetchedBytes, m.opsTransferredBytes, m.opsUploadedBytes) + }, m.opsFetchedBytes, m.opsTransferredBytes, m.opsWrittenBytes) size, err := TryToGetSize(tr) @@ -221,13 +224,16 @@ func TestDownloadDir_CleanUp(t *testing.T) { } tempDir := t.TempDir() - testutil.Ok(t, b.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1")))) - testutil.Ok(t, b.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2")))) - testutil.Ok(t, b.Upload(context.Background(), "dir/obj3", bytes.NewReader([]byte("3")))) + _, err := b.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1"))) + testutil.Ok(t, err) + _, err = b.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2"))) + testutil.Ok(t, err) + _, err = b.Upload(context.Background(), "dir/obj3", bytes.NewReader([]byte("3"))) + testutil.Ok(t, err) // We exapect the third Get to fail testutil.NotOk(t, DownloadDir(context.Background(), log.NewNopLogger(), b, "dir/", "dir/", tempDir)) - _, err := os.Stat(tempDir) + _, err = os.Stat(tempDir) testutil.Assert(t, os.IsNotExist(err)) } diff --git a/prefixed_bucket.go b/prefixed_bucket.go index f2b71434..cbc5b491 100644 --- a/prefixed_bucket.go +++ b/prefixed_bucket.go @@ -86,7 +86,7 @@ func (p PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttr // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. -func (p *PrefixedBucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (p *PrefixedBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { return p.bkt.Upload(ctx, conditionalPrefix(p.prefix, name), r) } diff --git a/prefixed_bucket_test.go b/prefixed_bucket_test.go index f93c8580..e7f2d9bc 100644 --- a/prefixed_bucket_test.go +++ b/prefixed_bucket_test.go @@ -29,7 +29,8 @@ func TestPrefixedBucket_Acceptance(t *testing.T) { } func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { - testutil.Ok(t, bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/file1.jpg", strings.NewReader("test-data1"))) + _, err := bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/file1.jpg", strings.NewReader("test-data1")) + testutil.Ok(t, err) pBkt := NewPrefixedBucket(bkt, prefix) rc1, err := pBkt.Get(context.Background(), "file1.jpg") @@ -41,7 +42,8 @@ func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { testutil.Ok(t, err) testutil.Equals(t, "test-data1", string(content)) - testutil.Ok(t, pBkt.Upload(context.Background(), "file2.jpg", strings.NewReader("test-data2"))) + _, err = pBkt.Upload(context.Background(), "file2.jpg", strings.NewReader("test-data2")) + testutil.Ok(t, err) rc2, err := bkt.Get(context.Background(), strings.Trim(prefix, "/")+"/file2.jpg") testutil.Ok(t, err) defer func() { testutil.Ok(t, rc2.Close()) }() @@ -69,7 +71,8 @@ func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { testutil.Ok(t, err) testutil.Assert(t, attrs.Size == 10, "expected size to be equal to 10") - testutil.Ok(t, bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/dir/file1.jpg", strings.NewReader("test-data1"))) + _, err = bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/dir/file1.jpg", strings.NewReader("test-data1")) + testutil.Ok(t, err) seen := []string{} testutil.Ok(t, pBkt.Iter(context.Background(), "", func(fn string) error { seen = append(seen, fn) diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 72e1b1e0..f5aefd8a 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -110,29 +110,29 @@ func (b *Bucket) Delete(_ context.Context, name string) error { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { size, err := objstore.TryToGetSize(r) if err != nil { - return errors.Wrapf(err, "getting size of %s", name) + return 0, errors.Wrapf(err, "getting size of %s", name) } partNums, lastSlice := int(math.Floor(float64(size)/partSize)), size%partSize if partNums == 0 { body, err := bce.NewBodyFromSizedReader(r, lastSlice) if err != nil { - return errors.Wrapf(err, "failed to create SizedReader for %s", name) + return 0, errors.Wrapf(err, "failed to create SizedReader for %s", name) } if _, err := b.client.PutObject(b.name, name, body, nil); err != nil { - return errors.Wrapf(err, "failed to upload %s", name) + return 0, errors.Wrapf(err, "failed to upload %s", name) } - return nil + return 0, nil } result, err := b.client.BasicInitiateMultipartUpload(b.name, name) if err != nil { - return errors.Wrapf(err, "failed to initiate MultipartUpload for %s", name) + return 0, errors.Wrapf(err, "failed to initiate MultipartUpload for %s", name) } uploadEveryPart := func(partSize int64, part int, uploadId string) (string, error) { @@ -156,7 +156,7 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { for part := 1; part <= partNums; part++ { etag, err := uploadEveryPart(partSize, part, result.UploadId) if err != nil { - return errors.Wrapf(err, "failed to upload part %d for %s", part, name) + return 0, errors.Wrapf(err, "failed to upload part %d for %s", part, name) } parts = append(parts, api.UploadInfoType{PartNumber: part, ETag: etag}) } @@ -164,15 +164,15 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { if lastSlice != 0 { etag, err := uploadEveryPart(lastSlice, partNums+1, result.UploadId) if err != nil { - return errors.Wrapf(err, "failed to upload the last part for %s", name) + return 0, errors.Wrapf(err, "failed to upload the last part for %s", name) } parts = append(parts, api.UploadInfoType{PartNumber: partNums + 1, ETag: etag}) } if _, err := b.client.CompleteMultipartUploadFromStruct(b.name, name, result.UploadId, &api.CompleteMultipartUploadArgs{Parts: parts}); err != nil { - return errors.Wrapf(err, "failed to set %s upload completed", name) + return 0, errors.Wrapf(err, "failed to set %s upload completed", name) } - return nil + return size, nil } // Iter calls f for each entry in the given directory (not recursive). The argument to f is the full diff --git a/providers/cos/cos.go b/providers/cos/cos.go index e518cae2..1b99f640 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -197,24 +197,24 @@ func (r fixedLengthReader) Size() int64 { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { size, err := objstore.TryToGetSize(r) if err != nil { - return errors.Wrapf(err, "getting size of %s", name) + return 0, errors.Wrapf(err, "getting size of %s", name) } // partSize 128MB. const partSize = 1024 * 1024 * 128 partNums, lastSlice := int(math.Floor(float64(size)/partSize)), size%partSize if partNums == 0 { if _, err := b.client.Object.Put(ctx, name, r, nil); err != nil { - return errors.Wrapf(err, "Put object: %s", name) + return 0, errors.Wrapf(err, "Put object: %s", name) } - return nil + return 0, nil } // 1. init. result, _, err := b.client.Object.InitiateMultipartUpload(ctx, name, nil) if err != nil { - return errors.Wrapf(err, "InitiateMultipartUpload %s", name) + return 0, errors.Wrapf(err, "InitiateMultipartUpload %s", name) } uploadEveryPart := func(partSize int64, part int, uploadID string) (string, error) { r := newFixedLengthReader(r, partSize) @@ -235,7 +235,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { for part := 1; part <= partNums; part++ { etag, err := uploadEveryPart(partSize, part, result.UploadID) if err != nil { - return errors.Wrapf(err, "uploadPart %d, %s", part, name) + return 0, errors.Wrapf(err, "uploadPart %d, %s", part, name) } optcom.Parts = append(optcom.Parts, cos.Object{ PartNumber: part, ETag: etag}, @@ -246,7 +246,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { part := partNums + 1 etag, err := uploadEveryPart(lastSlice, part, result.UploadID) if err != nil { - return errors.Wrapf(err, "uploadPart %d, %s", part, name) + return 0, errors.Wrapf(err, "uploadPart %d, %s", part, name) } optcom.Parts = append(optcom.Parts, cos.Object{ PartNumber: part, ETag: etag}, @@ -254,9 +254,9 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { } // 4. complete. if _, _, err := b.client.Object.CompleteMultipartUpload(ctx, name, result.UploadID, optcom); err != nil { - return errors.Wrapf(err, "CompleteMultipartUpload %s", name) + return 0, errors.Wrapf(err, "CompleteMultipartUpload %s", name) } - return nil + return size, nil } // Delete removes the object with the given name. diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 21c70485..026f2368 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -191,26 +191,27 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload writes the file specified in src to into the memory. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { if ctx.Err() != nil { - return ctx.Err() + return 0, ctx.Err() } file := filepath.Join(b.rootDir, name) if err := os.MkdirAll(filepath.Dir(file), os.ModePerm); err != nil { - return err + return 0, err } f, err := os.Create(file) if err != nil { - return err + return 0, err } defer errcapture.Do(&err, f.Close, "close") - if _, err := io.Copy(f, r); err != nil { - return errors.Wrapf(err, "copy to %s", file) + written, err := io.Copy(f, r) + if err != nil { + return 0, errors.Wrapf(err, "copy to %s", file) } - return nil + return written, nil } func isDirEmpty(name string) (ok bool, err error) { diff --git a/providers/filesystem/filesystem_test.go b/providers/filesystem/filesystem_test.go index c3621fe0..d8a0a91d 100644 --- a/providers/filesystem/filesystem_test.go +++ b/providers/filesystem/filesystem_test.go @@ -23,8 +23,10 @@ func TestDelete_EmptyDirDeletionRaceCondition(t *testing.T) { testutil.Ok(t, err) // Upload 2 objects in a subfolder. - testutil.Ok(t, b.Upload(ctx, "subfolder/first", strings.NewReader("first"))) - testutil.Ok(t, b.Upload(ctx, "subfolder/second", strings.NewReader("second"))) + _, err = b.Upload(ctx, "subfolder/first", strings.NewReader("first")) + testutil.Ok(t, err) + _, err = b.Upload(ctx, "subfolder/second", strings.NewReader("second")) + testutil.Ok(t, err) // Prepare goroutines to concurrently delete the 2 objects (each one deletes a different object) start := make(chan struct{}) @@ -116,7 +118,7 @@ func TestUpload_CancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - err = b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content"))) + _, err = b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content"))) testutil.NotOk(t, err) testutil.Equals(t, context.Canceled, err) } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 5ea45c7e..be17b6d1 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -171,13 +171,14 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload writes the file specified in src to remote GCS location specified as target. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { w := b.bkt.Object(name).NewWriter(ctx) - if _, err := io.Copy(w, r); err != nil { - return err + written, err := io.Copy(w, r) + if err != nil { + return 0, err } - return w.Close() + return written, w.Close() } // Delete removes the object with the given name. diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 35a04c80..1d95f7b2 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -128,26 +128,26 @@ func (b *Bucket) Delete(ctx context.Context, name string) error { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { size, err := objstore.TryToGetSize(r) if err != nil { - return errors.Wrapf(err, "failed to get size apriori to upload %s", name) + return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) } if size <= 0 { - return errors.New("object size must be provided") + return 0, errors.New("object size must be provided") } if size <= MinMultipartUploadSize { err = b.putObjectSingle(name, r) if err != nil { - return err + return 0, err } } else { var initOutput *obs.InitiateMultipartUploadOutput initOutput, err = b.initiateMultipartUpload(name) if err != nil { - return err + return 0, err } uploadId := initOutput.UploadId @@ -165,7 +165,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { }() parts, err := b.multipartUpload(size, name, uploadId, r) if err != nil { - return err + return 0, err } _, err = b.client.CompleteMultipartUpload(&obs.CompleteMultipartUploadInput{ @@ -175,10 +175,10 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { Parts: parts, }) if err != nil { - return errors.Wrap(err, "failed to complete multipart upload") + return 0, errors.Wrap(err, "failed to complete multipart upload") } } - return nil + return size, nil } func (b *Bucket) putObjectSingle(key string, body io.Reader) error { diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 2db35461..f57f5fe4 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -168,7 +168,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, offset, length int64 // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (written int64, err error) { req := transfer.UploadStreamRequest{ UploadRequest: transfer.UploadRequest{ NamespaceName: common.String(b.namespace), @@ -185,9 +185,9 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err erro } uploadManager := transfer.NewUploadManager() - _, err = uploadManager.UploadStream(ctx, req) + response, err := uploadManager.UploadStream(ctx, req) - return err + return response.PutObjectResponse.RawResponse.ContentLength, err } // Exists checks if the given object exists in the bucket. diff --git a/providers/oss/oss.go b/providers/oss/oss.go index e7e3a648..b259f576 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -68,11 +68,11 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. size, err := objstore.TryToGetSize(r) if err != nil { - return errors.Wrapf(err, "failed to get size apriori to upload %s", name) + return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) } chunksnum, lastslice := int(math.Floor(float64(size)/PartSize)), size%PartSize @@ -81,13 +81,13 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { switch chunksnum { case 0: if err := b.bucket.PutObject(name, ncloser); err != nil { - return errors.Wrap(err, "failed to upload oss object") + return 0, errors.Wrap(err, "failed to upload oss object") } default: { init, err := b.bucket.InitiateMultipartUpload(name) if err != nil { - return errors.Wrap(err, "failed to initiate multi-part upload") + return 0, errors.Wrap(err, "failed to initiate multi-part upload") } chunk := 0 uploadEveryPart := func(everypartsize int64, cnk int) (alioss.UploadPart, error) { @@ -105,23 +105,23 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { for ; chunk < chunksnum; chunk++ { part, err := uploadEveryPart(PartSize, chunk+1) if err != nil { - return errors.Wrap(err, "failed to upload every part") + return 0, errors.Wrap(err, "failed to upload every part") } parts = append(parts, part) } if lastslice != 0 { part, err := uploadEveryPart(lastslice, chunksnum+1) if err != nil { - return errors.Wrap(err, "failed to upload the last chunk") + return 0, errors.Wrap(err, "failed to upload the last chunk") } parts = append(parts, part) } if _, err := b.bucket.CompleteMultipartUpload(init, parts); err != nil { - return errors.Wrap(err, "failed to set multi-part upload completive") + return 0, errors.Wrap(err, "failed to set multi-part upload completive") } } } - return nil + return size, nil } // Delete removes the object with the given name. diff --git a/providers/s3/s3.go b/providers/s3/s3.go index f92d3973..3ee29d24 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -475,10 +475,10 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { sse, err := b.getServerSideEncryption(ctx) if err != nil { - return err + return 0, err } // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. @@ -509,10 +509,10 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { NumThreads: 4, }, ); err != nil { - return errors.Wrap(err, "upload s3 object") + return 0, errors.Wrap(err, "upload s3 object") } - return nil + return size, nil } // Attributes returns information about the specified object. diff --git a/providers/s3/s3_e2e_test.go b/providers/s3/s3_e2e_test.go index 4b75a014..2bec75d9 100644 --- a/providers/s3/s3_e2e_test.go +++ b/providers/s3/s3_e2e_test.go @@ -50,6 +50,7 @@ func BenchmarkUpload(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - testutil.Ok(b, bkt.Upload(ctx, "test", strings.NewReader(str))) + _, err = bkt.Upload(ctx, "test", strings.NewReader(str)) + testutil.Ok(b, err) } } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index 9bfa3cf8..f45376ea 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -296,7 +296,7 @@ func (c *Container) IsAccessDeniedErr(err error) bool { } // Upload writes the contents of the reader as an object into the container. -func (c *Container) Upload(_ context.Context, name string, r io.Reader) (err error) { +func (c *Container) Upload(_ context.Context, name string, r io.Reader) (written int64, err error) { size, err := objstore.TryToGetSize(r) if err != nil { level.Warn(c.logger).Log("msg", "could not guess file size, using large object to avoid issues if the file is larger than limit", "name", name, "err", err) @@ -314,23 +314,24 @@ func (c *Container) Upload(_ context.Context, name string, r io.Reader) (err err } if c.useDynamicLargeObjects { if file, err = c.connection.DynamicLargeObjectCreateFile(&opts); err != nil { - return errors.Wrap(err, "create DLO file") + return 0, errors.Wrap(err, "create DLO file") } } else { if file, err = c.connection.StaticLargeObjectCreateFile(&opts); err != nil { - return errors.Wrap(err, "create SLO file") + return 0, errors.Wrap(err, "create SLO file") } } } else { if file, err = c.connection.ObjectCreate(c.name, name, true, "", "", swift.Headers{}); err != nil { - return errors.Wrap(err, "create file") + return 0, errors.Wrap(err, "create file") } } defer errcapture.Do(&err, file.Close, "upload object close") - if _, err := io.Copy(file, r); err != nil { - return errors.Wrap(err, "uploading object") + written, err = io.Copy(file, r) + if err != nil { + return 0, errors.Wrap(err, "uploading object") } - return nil + return written, nil } // Delete removes the object with the given name. diff --git a/testing.go b/testing.go index b8e3744c..b8e9bd35 100644 --- a/testing.go +++ b/testing.go @@ -100,7 +100,8 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Assert(t, bkt.IsObjNotFoundErr(err), "expected not found error but got %s", err) // Upload first object. - testutil.Ok(t, bkt.Upload(ctx, "id1/obj_1.some", strings.NewReader("@test-data@"))) + _, err = bkt.Upload(ctx, "id1/obj_1.some", strings.NewReader("@test-data@")) + testutil.Ok(t, err) // Double check we can immediately read it. rc1, err := bkt.Get(ctx, "id1/obj_1.some") @@ -150,14 +151,21 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Assert(t, ok, "expected exits") // Upload other objects. - testutil.Ok(t, bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@"))) + _, err = bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@")) + testutil.Ok(t, err) // Upload should be idempotent. - testutil.Ok(t, bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@"))) - testutil.Ok(t, bkt.Upload(ctx, "id1/obj_3.some", strings.NewReader("@test-data3@"))) - testutil.Ok(t, bkt.Upload(ctx, "id1/sub/subobj_1.some", strings.NewReader("@test-data4@"))) - testutil.Ok(t, bkt.Upload(ctx, "id1/sub/subobj_2.some", strings.NewReader("@test-data5@"))) - testutil.Ok(t, bkt.Upload(ctx, "id2/obj_4.some", strings.NewReader("@test-data6@"))) - testutil.Ok(t, bkt.Upload(ctx, "obj_5.some", strings.NewReader("@test-data7@"))) + _, err = bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@")) + testutil.Ok(t, err) + _, err = bkt.Upload(ctx, "id1/obj_3.some", strings.NewReader("@test-data3@")) + testutil.Ok(t, err) + _, err = bkt.Upload(ctx, "id1/sub/subobj_1.some", strings.NewReader("@test-data4@")) + testutil.Ok(t, err) + _, err = bkt.Upload(ctx, "id1/sub/subobj_2.some", strings.NewReader("@test-data5@")) + testutil.Ok(t, err) + _, err = bkt.Upload(ctx, "id2/obj_4.some", strings.NewReader("@test-data6@")) + testutil.Ok(t, err) + _, err = bkt.Upload(ctx, "obj_5.some", strings.NewReader("@test-data7@")) + testutil.Ok(t, err) // Can we iter over items from top dir? var seen []string @@ -247,7 +255,8 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { sort.Strings(seen) testutil.Equals(t, expected, seen) - testutil.Ok(t, bkt.Upload(ctx, "obj_6.som", bytes.NewReader(make([]byte, 1024*1024*200)))) + _, err = bkt.Upload(ctx, "obj_6.som", bytes.NewReader(make([]byte, 1024*1024*200))) + testutil.Ok(t, err) testutil.Ok(t, bkt.Delete(ctx, "obj_6.som")) } @@ -285,7 +294,7 @@ func (d *delayingBucket) Exists(ctx context.Context, name string) (bool, error) return d.bkt.Exists(ctx, name) } -func (d *delayingBucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (d *delayingBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { time.Sleep(d.delay) return d.bkt.Upload(ctx, name, r) } diff --git a/tracing/opentelemetry/opentelemetry.go b/tracing/opentelemetry/opentelemetry.go index f65b0f0e..1bedb348 100644 --- a/tracing/opentelemetry/opentelemetry.go +++ b/tracing/opentelemetry/opentelemetry.go @@ -90,7 +90,7 @@ func (t TracingBucket) Attributes(ctx context.Context, name string) (_ objstore. return t.bkt.Attributes(ctx, name) } -func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { +func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { ctx, span := t.tracer.Start(ctx, "bucket_upload") defer span.End() span.SetAttributes(attribute.String("name", name)) diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index 8b99e304..543fbfdd 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -96,10 +96,10 @@ func (t TracingBucket) Attributes(ctx context.Context, name string) (attrs objst return } -func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { +func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { doWithSpan(ctx, "bucket_upload", func(spanCtx context.Context, span opentracing.Span) { span.LogKV("name", name) - err = t.bkt.Upload(spanCtx, name, r) + writtenBytes, err = t.bkt.Upload(spanCtx, name, r) }) return } From 312ee41de61c7e3f2b54eeb7b35b63aee7c96be5 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 25 Jul 2023 14:24:02 +0100 Subject: [PATCH 016/130] [FIX] Azure upload logic Signed-off-by: rita.canavarro --- providers/azure/azure.go | 12 +++++++++--- providers/s3/s3.go | 7 ++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/providers/azure/azure.go b/providers/azure/azure.go index a72f3052..42e0e7f6 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -307,17 +307,23 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { level.Debug(b.logger).Log("msg", "uploading blob", "blob", name) + size, err := objstore.TryToGetSize(r) + if err != nil { + return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) + } + blobClient := b.containerClient.NewBlockBlobClient(name) opts := &blockblob.UploadStreamOptions{ BlockSize: 3 * 1024 * 1024, Concurrency: 4, } + if _, err := blobClient.UploadStream(ctx, r, opts); err != nil { - return errors.Wrapf(err, "cannot upload Azure blob, address: %s", name) + return 0, errors.Wrapf(err, "cannot upload Azure blob, address: %s", name) } - return nil + return size, nil } // Delete removes the object with the given name. diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 3ee29d24..86b1feaf 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -492,7 +492,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e if size < int64(partSize) { partSize = 0 } - if _, err := b.client.PutObject( + resp, err := b.client.PutObject( ctx, b.name, name, @@ -508,11 +508,12 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e // TODO(bwplotka): Consider adjusting this number to GOMAXPROCS or to expose this in config if it becomes bottleneck. NumThreads: 4, }, - ); err != nil { + ) + if err != nil { return 0, errors.Wrap(err, "upload s3 object") } - return size, nil + return resp.Size, nil } // Attributes returns information about the specified object. From ea0fd63a1f69338265b1f82c4ff2c2c2d115ba3d Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 28 Jul 2023 09:55:55 +0100 Subject: [PATCH 017/130] [FIX] Function declaration Signed-off-by: rita.canavarro --- objstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objstore.go b/objstore.go index c9ec22d2..7849ea8a 100644 --- a/objstore.go +++ b/objstore.go @@ -666,7 +666,7 @@ type timingReadCloser struct { writtenBytes *prometheus.CounterVec } -func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, writtenBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { +func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec, writtenBytes *prometheus.CounterVec) *timingReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) From bc60f0b3c497632d4f6c3bd2aa5b926f39fe504e Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 28 Jul 2023 10:34:25 +0100 Subject: [PATCH 018/130] [FEAT] Update README Signed-off-by: rita.canavarro --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2a155271..f0e71bd9 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ type Bucket interface { // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. - Upload(ctx context.Context, name string, r io.Reader) error + Upload(ctx context.Context, name string, r io.Reader) (int64, error) // Delete removes the object with the given name. // If object does not exist in the moment of deletion, Delete should throw error. From 3f65cd73cffdec56bb6f8980d20ce68ef1a0f790 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 31 Jul 2023 16:33:33 +0100 Subject: [PATCH 019/130] [FEAT] Clean up Signed-off-by: rita.canavarro --- README.md | 2 +- go.mod | 3 ++- go.sum | 12 +++++++++++ inmem.go | 6 +++--- objstore.go | 12 +++++------ objstore_test.go | 18 ++++++---------- prefixed_bucket.go | 2 +- prefixed_bucket_test.go | 9 +++----- providers/azure/azure.go | 10 ++++----- providers/bos/bos.go | 20 +++++++++--------- providers/cos/cos.go | 18 ++++++++-------- providers/filesystem/filesystem.go | 14 ++++++------- providers/gcs/gcs.go | 9 ++++---- providers/obs/obs.go | 16 +++++++------- providers/oci/oci.go | 6 +++--- providers/oss/oss.go | 16 +++++++------- providers/s3/s3.go | 10 ++++----- providers/s3/s3_e2e_test.go | 3 +-- providers/swift/swift.go | 14 ++++++------- testing.go | 29 +++++++++----------------- tracing/opentelemetry/opentelemetry.go | 2 +- tracing/opentracing/opentracing.go | 4 ++-- 22 files changed, 114 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index f0e71bd9..2a155271 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ type Bucket interface { // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. - Upload(ctx context.Context, name string, r io.Reader) (int64, error) + Upload(ctx context.Context, name string, r io.Reader) error // Delete removes the object with the given name. // If object does not exist in the moment of deletion, Delete should throw error. diff --git a/go.mod b/go.mod index dee14caa..81d39061 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect @@ -99,7 +100,7 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/kr/text v0.2.0 // indirect diff --git a/go.sum b/go.sum index 860aca72..a6266e2c 100644 --- a/go.sum +++ b/go.sum @@ -44,10 +44,13 @@ cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 h1:Oj853U9kG+RLTCQXpjvOnrv0WaZHxgmZz1TlLywgOPY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= @@ -142,6 +145,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -206,6 +210,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -270,6 +276,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -365,6 +372,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -427,6 +436,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= @@ -493,6 +503,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= @@ -506,6 +517,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/inmem.go b/inmem.go index 45d13ef3..84e27780 100644 --- a/inmem.go +++ b/inmem.go @@ -175,12 +175,12 @@ func (b *InMemBucket) Attributes(_ context.Context, name string) (ObjectAttribut } // Upload writes the file specified in src to into the memory. -func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { +func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) error { b.mtx.Lock() defer b.mtx.Unlock() body, err := io.ReadAll(r) if err != nil { - return 0, err + return err } b.objects[name] = body size := int64(len(body)) @@ -188,7 +188,7 @@ func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) (int64 Size: size, LastModified: time.Now(), } - return size, nil + return nil } // Delete removes all data prefixed with the dir. diff --git a/objstore.go b/objstore.go index 7849ea8a..14ba8c47 100644 --- a/objstore.go +++ b/objstore.go @@ -42,7 +42,7 @@ type Bucket interface { // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. - Upload(ctx context.Context, name string, r io.Reader) (int64, error) + Upload(ctx context.Context, name string, r io.Reader) error // Delete removes the object with the given name. // If object does not exist in the moment of deletion, Delete should throw error. @@ -285,7 +285,7 @@ func UploadFile(ctx context.Context, logger log.Logger, bkt Bucket, src, dst str } defer logerrcapture.Do(logger, r.Close, "close file %s", src) - if _, err := bkt.Upload(ctx, dst, r); err != nil { + if err := bkt.Upload(ctx, dst, r); err != nil { return errors.Wrapf(err, "upload file %s as %s", src, dst) } level.Debug(logger).Log("msg", "uploaded file", "from", src, "dst", dst, "bucket", bkt.Name()) @@ -599,21 +599,21 @@ func (b *metricBucket) Exists(ctx context.Context, name string) (bool, error) { return ok, nil } -func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) error { const op = OpUpload b.ops.WithLabelValues(op).Inc() start := time.Now() - writtenBytes, err := b.bkt.Upload(ctx, name, r) + err := b.bkt.Upload(ctx, name, r) if err != nil { if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } - return 0, err + return err } b.lastSuccessfulUploadTime.SetToCurrentTime() b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) - return writtenBytes, nil + return nil } func (b *metricBucket) Delete(ctx context.Context, name string) error { diff --git a/objstore_test.go b/objstore_test.go index 26b86ab4..370d53fc 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -85,12 +85,9 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), r, "") tempDir := t.TempDir() - _, err := m.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1"))) - testutil.Ok(t, err) - _, err = m.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2"))) - testutil.Ok(t, err) - _, err = m.Upload(context.Background(), "dir/obj3", bytes.NewReader(bytes.Repeat([]byte("3"), 1024*1024))) - testutil.Ok(t, err) + testutil.Ok(t, m.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1")))) + testutil.Ok(t, m.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2")))) + testutil.Ok(t, m.Upload(context.Background(), "dir/obj3", bytes.NewReader(bytes.Repeat([]byte("3"), 1024*1024)))) testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` # HELP objstore_bucket_operations_total Total number of all attempted operations against a bucket. @@ -224,12 +221,9 @@ func TestDownloadDir_CleanUp(t *testing.T) { } tempDir := t.TempDir() - _, err := b.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1"))) - testutil.Ok(t, err) - _, err = b.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2"))) - testutil.Ok(t, err) - _, err = b.Upload(context.Background(), "dir/obj3", bytes.NewReader([]byte("3"))) - testutil.Ok(t, err) + testutil.Ok(t, b.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1")))) + testutil.Ok(t, b.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2")))) + testutil.Ok(t, b.Upload(context.Background(), "dir/obj3", bytes.NewReader([]byte("3")))) // We exapect the third Get to fail testutil.NotOk(t, DownloadDir(context.Background(), log.NewNopLogger(), b, "dir/", "dir/", tempDir)) diff --git a/prefixed_bucket.go b/prefixed_bucket.go index cbc5b491..f2b71434 100644 --- a/prefixed_bucket.go +++ b/prefixed_bucket.go @@ -86,7 +86,7 @@ func (p PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttr // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. -func (p *PrefixedBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (p *PrefixedBucket) Upload(ctx context.Context, name string, r io.Reader) error { return p.bkt.Upload(ctx, conditionalPrefix(p.prefix, name), r) } diff --git a/prefixed_bucket_test.go b/prefixed_bucket_test.go index e7f2d9bc..f93c8580 100644 --- a/prefixed_bucket_test.go +++ b/prefixed_bucket_test.go @@ -29,8 +29,7 @@ func TestPrefixedBucket_Acceptance(t *testing.T) { } func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { - _, err := bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/file1.jpg", strings.NewReader("test-data1")) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/file1.jpg", strings.NewReader("test-data1"))) pBkt := NewPrefixedBucket(bkt, prefix) rc1, err := pBkt.Get(context.Background(), "file1.jpg") @@ -42,8 +41,7 @@ func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { testutil.Ok(t, err) testutil.Equals(t, "test-data1", string(content)) - _, err = pBkt.Upload(context.Background(), "file2.jpg", strings.NewReader("test-data2")) - testutil.Ok(t, err) + testutil.Ok(t, pBkt.Upload(context.Background(), "file2.jpg", strings.NewReader("test-data2"))) rc2, err := bkt.Get(context.Background(), strings.Trim(prefix, "/")+"/file2.jpg") testutil.Ok(t, err) defer func() { testutil.Ok(t, rc2.Close()) }() @@ -71,8 +69,7 @@ func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { testutil.Ok(t, err) testutil.Assert(t, attrs.Size == 10, "expected size to be equal to 10") - _, err = bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/dir/file1.jpg", strings.NewReader("test-data1")) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/dir/file1.jpg", strings.NewReader("test-data1"))) seen := []string{} testutil.Ok(t, pBkt.Iter(context.Background(), "", func(fn string) error { seen = append(seen, fn) diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 42e0e7f6..bbddac64 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -307,11 +307,11 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { level.Debug(b.logger).Log("msg", "uploading blob", "blob", name) - size, err := objstore.TryToGetSize(r) + _, err := objstore.TryToGetSize(r) if err != nil { - return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) + return errors.Wrapf(err, "failed to get size apriori to upload %s", name) } blobClient := b.containerClient.NewBlockBlobClient(name) @@ -321,9 +321,9 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e } if _, err := blobClient.UploadStream(ctx, r, opts); err != nil { - return 0, errors.Wrapf(err, "cannot upload Azure blob, address: %s", name) + return errors.Wrapf(err, "cannot upload Azure blob, address: %s", name) } - return size, nil + return nil } // Delete removes the object with the given name. diff --git a/providers/bos/bos.go b/providers/bos/bos.go index f5aefd8a..72e1b1e0 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -110,29 +110,29 @@ func (b *Bucket) Delete(_ context.Context, name string) error { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { size, err := objstore.TryToGetSize(r) if err != nil { - return 0, errors.Wrapf(err, "getting size of %s", name) + return errors.Wrapf(err, "getting size of %s", name) } partNums, lastSlice := int(math.Floor(float64(size)/partSize)), size%partSize if partNums == 0 { body, err := bce.NewBodyFromSizedReader(r, lastSlice) if err != nil { - return 0, errors.Wrapf(err, "failed to create SizedReader for %s", name) + return errors.Wrapf(err, "failed to create SizedReader for %s", name) } if _, err := b.client.PutObject(b.name, name, body, nil); err != nil { - return 0, errors.Wrapf(err, "failed to upload %s", name) + return errors.Wrapf(err, "failed to upload %s", name) } - return 0, nil + return nil } result, err := b.client.BasicInitiateMultipartUpload(b.name, name) if err != nil { - return 0, errors.Wrapf(err, "failed to initiate MultipartUpload for %s", name) + return errors.Wrapf(err, "failed to initiate MultipartUpload for %s", name) } uploadEveryPart := func(partSize int64, part int, uploadId string) (string, error) { @@ -156,7 +156,7 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, err for part := 1; part <= partNums; part++ { etag, err := uploadEveryPart(partSize, part, result.UploadId) if err != nil { - return 0, errors.Wrapf(err, "failed to upload part %d for %s", part, name) + return errors.Wrapf(err, "failed to upload part %d for %s", part, name) } parts = append(parts, api.UploadInfoType{PartNumber: part, ETag: etag}) } @@ -164,15 +164,15 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, err if lastSlice != 0 { etag, err := uploadEveryPart(lastSlice, partNums+1, result.UploadId) if err != nil { - return 0, errors.Wrapf(err, "failed to upload the last part for %s", name) + return errors.Wrapf(err, "failed to upload the last part for %s", name) } parts = append(parts, api.UploadInfoType{PartNumber: partNums + 1, ETag: etag}) } if _, err := b.client.CompleteMultipartUploadFromStruct(b.name, name, result.UploadId, &api.CompleteMultipartUploadArgs{Parts: parts}); err != nil { - return 0, errors.Wrapf(err, "failed to set %s upload completed", name) + return errors.Wrapf(err, "failed to set %s upload completed", name) } - return size, nil + return nil } // Iter calls f for each entry in the given directory (not recursive). The argument to f is the full diff --git a/providers/cos/cos.go b/providers/cos/cos.go index 1b99f640..e518cae2 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -197,24 +197,24 @@ func (r fixedLengthReader) Size() int64 { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { size, err := objstore.TryToGetSize(r) if err != nil { - return 0, errors.Wrapf(err, "getting size of %s", name) + return errors.Wrapf(err, "getting size of %s", name) } // partSize 128MB. const partSize = 1024 * 1024 * 128 partNums, lastSlice := int(math.Floor(float64(size)/partSize)), size%partSize if partNums == 0 { if _, err := b.client.Object.Put(ctx, name, r, nil); err != nil { - return 0, errors.Wrapf(err, "Put object: %s", name) + return errors.Wrapf(err, "Put object: %s", name) } - return 0, nil + return nil } // 1. init. result, _, err := b.client.Object.InitiateMultipartUpload(ctx, name, nil) if err != nil { - return 0, errors.Wrapf(err, "InitiateMultipartUpload %s", name) + return errors.Wrapf(err, "InitiateMultipartUpload %s", name) } uploadEveryPart := func(partSize int64, part int, uploadID string) (string, error) { r := newFixedLengthReader(r, partSize) @@ -235,7 +235,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e for part := 1; part <= partNums; part++ { etag, err := uploadEveryPart(partSize, part, result.UploadID) if err != nil { - return 0, errors.Wrapf(err, "uploadPart %d, %s", part, name) + return errors.Wrapf(err, "uploadPart %d, %s", part, name) } optcom.Parts = append(optcom.Parts, cos.Object{ PartNumber: part, ETag: etag}, @@ -246,7 +246,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e part := partNums + 1 etag, err := uploadEveryPart(lastSlice, part, result.UploadID) if err != nil { - return 0, errors.Wrapf(err, "uploadPart %d, %s", part, name) + return errors.Wrapf(err, "uploadPart %d, %s", part, name) } optcom.Parts = append(optcom.Parts, cos.Object{ PartNumber: part, ETag: etag}, @@ -254,9 +254,9 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e } // 4. complete. if _, _, err := b.client.Object.CompleteMultipartUpload(ctx, name, result.UploadID, optcom); err != nil { - return 0, errors.Wrapf(err, "CompleteMultipartUpload %s", name) + return errors.Wrapf(err, "CompleteMultipartUpload %s", name) } - return size, nil + return nil } // Delete removes the object with the given name. diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 026f2368..5d5085e9 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -191,27 +191,27 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload writes the file specified in src to into the memory. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { if ctx.Err() != nil { - return 0, ctx.Err() + return ctx.Err() } file := filepath.Join(b.rootDir, name) if err := os.MkdirAll(filepath.Dir(file), os.ModePerm); err != nil { - return 0, err + return err } f, err := os.Create(file) if err != nil { - return 0, err + return err } defer errcapture.Do(&err, f.Close, "close") - written, err := io.Copy(f, r) + _, err = io.Copy(f, r) if err != nil { - return 0, errors.Wrapf(err, "copy to %s", file) + return errors.Wrapf(err, "copy to %s", file) } - return written, nil + return nil } func isDirEmpty(name string) (ok bool, err error) { diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index be17b6d1..df902ed4 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -171,14 +171,13 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload writes the file specified in src to remote GCS location specified as target. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { w := b.bkt.Object(name).NewWriter(ctx) - - written, err := io.Copy(w, r) + _, err := io.Copy(w, r) if err != nil { - return 0, err + return err } - return written, w.Close() + return w.Close() } // Delete removes the object with the given name. diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 1d95f7b2..35a04c80 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -128,26 +128,26 @@ func (b *Bucket) Delete(ctx context.Context, name string) error { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { size, err := objstore.TryToGetSize(r) if err != nil { - return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) + return errors.Wrapf(err, "failed to get size apriori to upload %s", name) } if size <= 0 { - return 0, errors.New("object size must be provided") + return errors.New("object size must be provided") } if size <= MinMultipartUploadSize { err = b.putObjectSingle(name, r) if err != nil { - return 0, err + return err } } else { var initOutput *obs.InitiateMultipartUploadOutput initOutput, err = b.initiateMultipartUpload(name) if err != nil { - return 0, err + return err } uploadId := initOutput.UploadId @@ -165,7 +165,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e }() parts, err := b.multipartUpload(size, name, uploadId, r) if err != nil { - return 0, err + return err } _, err = b.client.CompleteMultipartUpload(&obs.CompleteMultipartUploadInput{ @@ -175,10 +175,10 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e Parts: parts, }) if err != nil { - return 0, errors.Wrap(err, "failed to complete multipart upload") + return errors.Wrap(err, "failed to complete multipart upload") } } - return size, nil + return nil } func (b *Bucket) putObjectSingle(key string, body io.Reader) error { diff --git a/providers/oci/oci.go b/providers/oci/oci.go index f57f5fe4..2db35461 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -168,7 +168,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, offset, length int64 // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (written int64, err error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { req := transfer.UploadStreamRequest{ UploadRequest: transfer.UploadRequest{ NamespaceName: common.String(b.namespace), @@ -185,9 +185,9 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (written } uploadManager := transfer.NewUploadManager() - response, err := uploadManager.UploadStream(ctx, req) + _, err = uploadManager.UploadStream(ctx, req) - return response.PutObjectResponse.RawResponse.ContentLength, err + return err } // Exists checks if the given object exists in the bucket. diff --git a/providers/oss/oss.go b/providers/oss/oss.go index b259f576..e7e3a648 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -68,11 +68,11 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. size, err := objstore.TryToGetSize(r) if err != nil { - return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) + return errors.Wrapf(err, "failed to get size apriori to upload %s", name) } chunksnum, lastslice := int(math.Floor(float64(size)/PartSize)), size%PartSize @@ -81,13 +81,13 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, err switch chunksnum { case 0: if err := b.bucket.PutObject(name, ncloser); err != nil { - return 0, errors.Wrap(err, "failed to upload oss object") + return errors.Wrap(err, "failed to upload oss object") } default: { init, err := b.bucket.InitiateMultipartUpload(name) if err != nil { - return 0, errors.Wrap(err, "failed to initiate multi-part upload") + return errors.Wrap(err, "failed to initiate multi-part upload") } chunk := 0 uploadEveryPart := func(everypartsize int64, cnk int) (alioss.UploadPart, error) { @@ -105,23 +105,23 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, err for ; chunk < chunksnum; chunk++ { part, err := uploadEveryPart(PartSize, chunk+1) if err != nil { - return 0, errors.Wrap(err, "failed to upload every part") + return errors.Wrap(err, "failed to upload every part") } parts = append(parts, part) } if lastslice != 0 { part, err := uploadEveryPart(lastslice, chunksnum+1) if err != nil { - return 0, errors.Wrap(err, "failed to upload the last chunk") + return errors.Wrap(err, "failed to upload the last chunk") } parts = append(parts, part) } if _, err := b.bucket.CompleteMultipartUpload(init, parts); err != nil { - return 0, errors.Wrap(err, "failed to set multi-part upload completive") + return errors.Wrap(err, "failed to set multi-part upload completive") } } } - return size, nil + return nil } // Delete removes the object with the given name. diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 86b1feaf..0e5e24e1 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -475,10 +475,10 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { sse, err := b.getServerSideEncryption(ctx) if err != nil { - return 0, err + return err } // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. @@ -492,7 +492,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e if size < int64(partSize) { partSize = 0 } - resp, err := b.client.PutObject( + _, err = b.client.PutObject( ctx, b.name, name, @@ -510,10 +510,10 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e }, ) if err != nil { - return 0, errors.Wrap(err, "upload s3 object") + return errors.Wrap(err, "upload s3 object") } - return resp.Size, nil + return nil } // Attributes returns information about the specified object. diff --git a/providers/s3/s3_e2e_test.go b/providers/s3/s3_e2e_test.go index 2bec75d9..2bb3a851 100644 --- a/providers/s3/s3_e2e_test.go +++ b/providers/s3/s3_e2e_test.go @@ -50,7 +50,6 @@ func BenchmarkUpload(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, err = bkt.Upload(ctx, "test", strings.NewReader(str)) - testutil.Ok(b, err) + testutil.Ok(bkt.Upload(ctx, "test", strings.NewReader(str)), err) } } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index f45376ea..b5bbd903 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -296,7 +296,7 @@ func (c *Container) IsAccessDeniedErr(err error) bool { } // Upload writes the contents of the reader as an object into the container. -func (c *Container) Upload(_ context.Context, name string, r io.Reader) (written int64, err error) { +func (c *Container) Upload(_ context.Context, name string, r io.Reader) (err error) { size, err := objstore.TryToGetSize(r) if err != nil { level.Warn(c.logger).Log("msg", "could not guess file size, using large object to avoid issues if the file is larger than limit", "name", name, "err", err) @@ -314,24 +314,24 @@ func (c *Container) Upload(_ context.Context, name string, r io.Reader) (written } if c.useDynamicLargeObjects { if file, err = c.connection.DynamicLargeObjectCreateFile(&opts); err != nil { - return 0, errors.Wrap(err, "create DLO file") + return errors.Wrap(err, "create DLO file") } } else { if file, err = c.connection.StaticLargeObjectCreateFile(&opts); err != nil { - return 0, errors.Wrap(err, "create SLO file") + return errors.Wrap(err, "create SLO file") } } } else { if file, err = c.connection.ObjectCreate(c.name, name, true, "", "", swift.Headers{}); err != nil { - return 0, errors.Wrap(err, "create file") + return errors.Wrap(err, "create file") } } defer errcapture.Do(&err, file.Close, "upload object close") - written, err = io.Copy(file, r) + _, err = io.Copy(file, r) if err != nil { - return 0, errors.Wrap(err, "uploading object") + return errors.Wrap(err, "uploading object") } - return written, nil + return nil } // Delete removes the object with the given name. diff --git a/testing.go b/testing.go index b8e9bd35..b8e3744c 100644 --- a/testing.go +++ b/testing.go @@ -100,8 +100,7 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Assert(t, bkt.IsObjNotFoundErr(err), "expected not found error but got %s", err) // Upload first object. - _, err = bkt.Upload(ctx, "id1/obj_1.some", strings.NewReader("@test-data@")) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(ctx, "id1/obj_1.some", strings.NewReader("@test-data@"))) // Double check we can immediately read it. rc1, err := bkt.Get(ctx, "id1/obj_1.some") @@ -151,21 +150,14 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Assert(t, ok, "expected exits") // Upload other objects. - _, err = bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@")) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@"))) // Upload should be idempotent. - _, err = bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@")) - testutil.Ok(t, err) - _, err = bkt.Upload(ctx, "id1/obj_3.some", strings.NewReader("@test-data3@")) - testutil.Ok(t, err) - _, err = bkt.Upload(ctx, "id1/sub/subobj_1.some", strings.NewReader("@test-data4@")) - testutil.Ok(t, err) - _, err = bkt.Upload(ctx, "id1/sub/subobj_2.some", strings.NewReader("@test-data5@")) - testutil.Ok(t, err) - _, err = bkt.Upload(ctx, "id2/obj_4.some", strings.NewReader("@test-data6@")) - testutil.Ok(t, err) - _, err = bkt.Upload(ctx, "obj_5.some", strings.NewReader("@test-data7@")) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@"))) + testutil.Ok(t, bkt.Upload(ctx, "id1/obj_3.some", strings.NewReader("@test-data3@"))) + testutil.Ok(t, bkt.Upload(ctx, "id1/sub/subobj_1.some", strings.NewReader("@test-data4@"))) + testutil.Ok(t, bkt.Upload(ctx, "id1/sub/subobj_2.some", strings.NewReader("@test-data5@"))) + testutil.Ok(t, bkt.Upload(ctx, "id2/obj_4.some", strings.NewReader("@test-data6@"))) + testutil.Ok(t, bkt.Upload(ctx, "obj_5.some", strings.NewReader("@test-data7@"))) // Can we iter over items from top dir? var seen []string @@ -255,8 +247,7 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { sort.Strings(seen) testutil.Equals(t, expected, seen) - _, err = bkt.Upload(ctx, "obj_6.som", bytes.NewReader(make([]byte, 1024*1024*200))) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(ctx, "obj_6.som", bytes.NewReader(make([]byte, 1024*1024*200)))) testutil.Ok(t, bkt.Delete(ctx, "obj_6.som")) } @@ -294,7 +285,7 @@ func (d *delayingBucket) Exists(ctx context.Context, name string) (bool, error) return d.bkt.Exists(ctx, name) } -func (d *delayingBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (d *delayingBucket) Upload(ctx context.Context, name string, r io.Reader) error { time.Sleep(d.delay) return d.bkt.Upload(ctx, name, r) } diff --git a/tracing/opentelemetry/opentelemetry.go b/tracing/opentelemetry/opentelemetry.go index 1bedb348..f65b0f0e 100644 --- a/tracing/opentelemetry/opentelemetry.go +++ b/tracing/opentelemetry/opentelemetry.go @@ -90,7 +90,7 @@ func (t TracingBucket) Attributes(ctx context.Context, name string) (_ objstore. return t.bkt.Attributes(ctx, name) } -func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { +func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { ctx, span := t.tracer.Start(ctx, "bucket_upload") defer span.End() span.SetAttributes(attribute.String("name", name)) diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index 543fbfdd..8b99e304 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -96,10 +96,10 @@ func (t TracingBucket) Attributes(ctx context.Context, name string) (attrs objst return } -func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { +func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { doWithSpan(ctx, "bucket_upload", func(spanCtx context.Context, span opentracing.Span) { span.LogKV("name", name) - writtenBytes, err = t.bkt.Upload(spanCtx, name, r) + err = t.bkt.Upload(spanCtx, name, r) }) return } From 361e530e86bfe85f16a73557eec580ab0f4cab17 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 31 Jul 2023 16:36:30 +0100 Subject: [PATCH 020/130] [FEAT] Clean up Signed-off-by: rita.canavarro --- go.mod | 10 +++++----- go.sum | 12 ++++++++++++ objstore_test.go | 2 +- providers/s3/s3_e2e_test.go | 2 +- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 81d39061..ff862035 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/oracle/oci-go-sdk/v65 v65.41.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.12.2 - github.com/prometheus/common v0.36.0 + github.com/prometheus/client_golang v1.16.0 + github.com/prometheus/common v0.44.0 github.com/tencentyun/cos-go-sdk-v5 v0.7.40 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 @@ -71,7 +71,7 @@ require ( github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect @@ -80,8 +80,8 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect diff --git a/go.sum b/go.sum index a6266e2c..b453b431 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfR github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 h1:Oj853U9kG+RLTCQXpjvOnrv0WaZHxgmZz1TlLywgOPY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= @@ -261,6 +262,8 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI= @@ -303,17 +306,26 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.36.0 h1:78hJTing+BLYLjhXE+Z2BubeEymH5Lr0/Mt8FKkxxYo= github.com/prometheus/common v0.36.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= diff --git a/objstore_test.go b/objstore_test.go index 370d53fc..934dbb50 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -227,7 +227,7 @@ func TestDownloadDir_CleanUp(t *testing.T) { // We exapect the third Get to fail testutil.NotOk(t, DownloadDir(context.Background(), log.NewNopLogger(), b, "dir/", "dir/", tempDir)) - _, err = os.Stat(tempDir) + _, err := os.Stat(tempDir) testutil.Assert(t, os.IsNotExist(err)) } diff --git a/providers/s3/s3_e2e_test.go b/providers/s3/s3_e2e_test.go index 2bb3a851..4b75a014 100644 --- a/providers/s3/s3_e2e_test.go +++ b/providers/s3/s3_e2e_test.go @@ -50,6 +50,6 @@ func BenchmarkUpload(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - testutil.Ok(bkt.Upload(ctx, "test", strings.NewReader(str)), err) + testutil.Ok(b, bkt.Upload(ctx, "test", strings.NewReader(str))) } } From 8b22c6fd29c6c756c3994e33fa469e1ea0364f2f Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 31 Jul 2023 16:42:16 +0100 Subject: [PATCH 021/130] [FEAT] Clean up Signed-off-by: rita.canavarro --- inmem.go | 3 +-- providers/azure/azure.go | 5 ----- providers/filesystem/filesystem.go | 3 +-- providers/filesystem/filesystem_test.go | 9 +++------ providers/gcs/gcs.go | 3 +-- providers/s3/s3.go | 5 ++--- providers/swift/swift.go | 3 +-- 7 files changed, 9 insertions(+), 22 deletions(-) diff --git a/inmem.go b/inmem.go index 84e27780..3f6f35e9 100644 --- a/inmem.go +++ b/inmem.go @@ -183,9 +183,8 @@ func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) error return err } b.objects[name] = body - size := int64(len(body)) b.attrs[name] = ObjectAttributes{ - Size: size, + Size: int64(len(body)), LastModified: time.Now(), } return nil diff --git a/providers/azure/azure.go b/providers/azure/azure.go index bbddac64..f28ab025 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -309,11 +309,6 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { // Upload the contents of the reader as an object into the bucket. func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { level.Debug(b.logger).Log("msg", "uploading blob", "blob", name) - _, err := objstore.TryToGetSize(r) - if err != nil { - return errors.Wrapf(err, "failed to get size apriori to upload %s", name) - } - blobClient := b.containerClient.NewBlockBlobClient(name) opts := &blockblob.UploadStreamOptions{ BlockSize: 3 * 1024 * 1024, diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 5d5085e9..cb0a094e 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -207,8 +207,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err erro } defer errcapture.Do(&err, f.Close, "close") - _, err = io.Copy(f, r) - if err != nil { + if _, err = io.Copy(f, r); err != nil { return errors.Wrapf(err, "copy to %s", file) } return nil diff --git a/providers/filesystem/filesystem_test.go b/providers/filesystem/filesystem_test.go index d8a0a91d..fb9a7d3d 100644 --- a/providers/filesystem/filesystem_test.go +++ b/providers/filesystem/filesystem_test.go @@ -23,10 +23,8 @@ func TestDelete_EmptyDirDeletionRaceCondition(t *testing.T) { testutil.Ok(t, err) // Upload 2 objects in a subfolder. - _, err = b.Upload(ctx, "subfolder/first", strings.NewReader("first")) - testutil.Ok(t, err) - _, err = b.Upload(ctx, "subfolder/second", strings.NewReader("second")) - testutil.Ok(t, err) + testutil.Ok(t, b.Upload(ctx, "subfolder/first", strings.NewReader("first"))) + testutil.Ok(t, b.Upload(ctx, "subfolder/second", strings.NewReader("second"))) // Prepare goroutines to concurrently delete the 2 objects (each one deletes a different object) start := make(chan struct{}) @@ -118,8 +116,7 @@ func TestUpload_CancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content"))) - testutil.NotOk(t, err) + testutil.NotOk(t, b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content")))) testutil.Equals(t, context.Canceled, err) } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index df902ed4..bee75fa3 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -173,8 +173,7 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { // Upload writes the file specified in src to remote GCS location specified as target. func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { w := b.bkt.Object(name).NewWriter(ctx) - _, err := io.Copy(w, r) - if err != nil { + if _, err := io.Copy(w, r); err != nil { return err } return w.Close() diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 0e5e24e1..f92d3973 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -492,7 +492,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { if size < int64(partSize) { partSize = 0 } - _, err = b.client.PutObject( + if _, err := b.client.PutObject( ctx, b.name, name, @@ -508,8 +508,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { // TODO(bwplotka): Consider adjusting this number to GOMAXPROCS or to expose this in config if it becomes bottleneck. NumThreads: 4, }, - ) - if err != nil { + ); err != nil { return errors.Wrap(err, "upload s3 object") } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index b5bbd903..ad7a90fa 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -327,8 +327,7 @@ func (c *Container) Upload(_ context.Context, name string, r io.Reader) (err err } } defer errcapture.Do(&err, file.Close, "upload object close") - _, err = io.Copy(file, r) - if err != nil { + if _, err = io.Copy(file, r); err != nil { return errors.Wrap(err, "uploading object") } return nil From 18e198e8eb88be015e2b53a3ff10ab76ec3cb224 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 31 Jul 2023 17:43:32 +0100 Subject: [PATCH 022/130] [FIX] Filesystem test Signed-off-by: rita.canavarro --- providers/filesystem/filesystem_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/filesystem/filesystem_test.go b/providers/filesystem/filesystem_test.go index fb9a7d3d..c3621fe0 100644 --- a/providers/filesystem/filesystem_test.go +++ b/providers/filesystem/filesystem_test.go @@ -116,7 +116,8 @@ func TestUpload_CancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - testutil.NotOk(t, b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content")))) + err = b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content"))) + testutil.NotOk(t, err) testutil.Equals(t, context.Canceled, err) } From 66868e8746f23c13119890b9a7b7ee42970a01f2 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 1 Aug 2023 09:10:44 +0100 Subject: [PATCH 023/130] [FIX] Nit pick Signed-off-by: rita.canavarro --- providers/filesystem/filesystem.go | 2 +- providers/gcs/gcs.go | 1 + providers/swift/swift.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index cb0a094e..21c70485 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -207,7 +207,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err erro } defer errcapture.Do(&err, f.Close, "close") - if _, err = io.Copy(f, r); err != nil { + if _, err := io.Copy(f, r); err != nil { return errors.Wrapf(err, "copy to %s", file) } return nil diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index bee75fa3..5ea45c7e 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -173,6 +173,7 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { // Upload writes the file specified in src to remote GCS location specified as target. func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { w := b.bkt.Object(name).NewWriter(ctx) + if _, err := io.Copy(w, r); err != nil { return err } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index ad7a90fa..9bfa3cf8 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -327,7 +327,7 @@ func (c *Container) Upload(_ context.Context, name string, r io.Reader) (err err } } defer errcapture.Do(&err, file.Close, "upload object close") - if _, err = io.Copy(file, r); err != nil { + if _, err := io.Copy(file, r); err != nil { return errors.Wrap(err, "uploading object") } return nil From 200a5149a2c7817334252b804f0be73e152de6b6 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 1 Aug 2023 09:56:43 +0100 Subject: [PATCH 024/130] [FEAT] Upload written bytes histogram Signed-off-by: rita.canavarro --- objstore.go | 15 ++++++--------- objstore_test.go | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/objstore.go b/objstore.go index 14ba8c47..aae04e6e 100644 --- a/objstore.go +++ b/objstore.go @@ -444,10 +444,11 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu ConstLabels: prometheus.Labels{"bucket": name}, }), - opsWrittenBytes: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ - Name: "objstore_bucket_operation_written_bytes_total", - Help: "Total number of bytes uploaded from TSDB block, per operation.", + opsWrittenBytes: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Name: "objstore_bucket_operation_written_bytes", + Help: "Number of bytes uploaded from TSDB block, per operation.", ConstLabels: prometheus.Labels{"bucket": name}, + Buckets: prometheus.ExponentialBuckets(50<<100, 2, 10), }, []string{"operation"}), } for _, op := range []string{ @@ -485,7 +486,7 @@ type metricBucket struct { opsFetchedBytes *prometheus.CounterVec opsTransferredBytes *prometheus.HistogramVec - opsWrittenBytes *prometheus.CounterVec + opsWrittenBytes *prometheus.HistogramVec opsDuration *prometheus.HistogramVec lastSuccessfulUploadTime prometheus.Gauge } @@ -556,7 +557,6 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, - b.opsWrittenBytes, ), nil } @@ -579,7 +579,6 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, - b.opsWrittenBytes, ), nil } @@ -663,10 +662,9 @@ type timingReadCloser struct { isFailureExpected IsOpFailureExpectedFunc fetchedBytes *prometheus.CounterVec transferredBytes *prometheus.HistogramVec - writtenBytes *prometheus.CounterVec } -func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec, writtenBytes *prometheus.CounterVec) *timingReadCloser { +func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -683,7 +681,6 @@ func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramV fetchedBytes: fetchedBytes, transferredBytes: transferredBytes, readBytes: 0, - writtenBytes: writtenBytes, } } diff --git a/objstore_test.go b/objstore_test.go index 934dbb50..4b105281 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -194,7 +194,7 @@ func TestTimingTracingReader(t *testing.T) { tr := NopCloserWithSize(r) tr = newTimingReadCloser(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { return false - }, m.opsFetchedBytes, m.opsTransferredBytes, m.opsWrittenBytes) + }, m.opsFetchedBytes, m.opsTransferredBytes) size, err := TryToGetSize(tr) From 7fb13ca24614f997f7d41ed775a7477f21cccee1 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 1 Aug 2023 11:40:25 +0100 Subject: [PATCH 025/130] [FEAT] Remove written bytes histogram Signed-off-by: rita.canavarro --- objstore.go | 1 - 1 file changed, 1 deletion(-) diff --git a/objstore.go b/objstore.go index aae04e6e..7eadcc39 100644 --- a/objstore.go +++ b/objstore.go @@ -464,7 +464,6 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsFailures.WithLabelValues(op) bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) - bkt.opsWrittenBytes.WithLabelValues(op) } // fetched bytes only relevant for get and getrange for _, op := range []string{ From 0548f4f14ab54eb4117f6f248443c8ecf5572952 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 10:33:18 +0100 Subject: [PATCH 026/130] [FEAT] Read transferred bytes per upload Signed-off-by: rita.canavarro --- objstore.go | 21 +++++++++++++++++++-- objstore_test.go | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/objstore.go b/objstore.go index 7eadcc39..19e40ea2 100644 --- a/objstore.go +++ b/objstore.go @@ -465,11 +465,11 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) } - // fetched bytes only relevant for get and getrange + // fetched bytes only relevant for get, getrange and upload for _, op := range []string{ OpGet, OpGetRange, - // TODO: Add uploads + OpUpload, } { bkt.opsTransferredBytes.WithLabelValues(op) } @@ -611,6 +611,23 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err } b.lastSuccessfulUploadTime.SetToCurrentTime() b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) + + rc := nopCloserWithObjectSize{ + Reader: r, + } + + trc := newTimingReadCloser( + rc, + op, + b.opsDuration, + b.opsFailures, + b.isOpFailureExpected, + b.opsFetchedBytes, + b.opsTransferredBytes, + ) + + trc.Close() + return nil } diff --git a/objstore_test.go b/objstore_test.go index 4b105281..6c893ab0 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -170,6 +170,25 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="+Inf"} 0 objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="get_range"} 0 objstore_bucket_operation_transferred_bytes_count{bucket="",operation="get_range"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="32768"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="65536"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="131072"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="262144"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="524288"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.048576e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="2.097152e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="4.194304e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="8.388608e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.6777216e+07"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="3.3554432e+07"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="6.7108864e+07"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.34217728e+08"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="2.68435456e+08"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="5.36870912e+08"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.073741824e+09"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="+Inf"} 3 + objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="upload"} 0 + objstore_bucket_operation_transferred_bytes_count{bucket="",operation="upload"} 3 `), `objstore_bucket_operation_transferred_bytes`)) testutil.Ok(t, UploadDir(context.Background(), log.NewNopLogger(), m, tempDir, "/dir-copy", WithUploadConcurrency(10))) From 8f5c7b5e75327206f996860d8773ce0b0146a3a4 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 10:50:19 +0100 Subject: [PATCH 027/130] [FIX] Conflicts after rebase Signed-off-by: rita.canavarro --- go.mod | 10 +++++----- go.sum | 35 +++++++++++++++++++++++++++++++++++ objstore.go | 7 ------- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index ff862035..5bc96bf2 100644 --- a/go.mod +++ b/go.mod @@ -24,8 +24,8 @@ require ( go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.4.0 - golang.org/x/sync v0.1.0 + golang.org/x/oauth2 v0.8.0 + golang.org/x/sync v0.3.0 google.golang.org/api v0.103.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 @@ -36,7 +36,7 @@ require ( cloud.google.com/go/compute v1.15.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.8.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect @@ -59,7 +59,7 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.4 // indirect @@ -94,7 +94,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index b453b431..5465ea17 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,7 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible h1:9gWa46nstkJ9miBReJcN8Gq34cBFbzSpQZVVT9N09TM= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go-v2 v1.16.0 h1:cBAYjiiexRAg9v2z9vb6IdxAa7ef4KCtjW7w7e3GxGo= github.com/aws/aws-sdk-go-v2 v1.16.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI= github.com/aws/aws-sdk-go-v2/config v1.15.1 h1:hTIZFepYESYyowQUBo47lu69WSxsYqGUILY9Nu8+7pY= @@ -111,6 +112,11 @@ github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -125,9 +131,12 @@ github.com/efficientgo/e2e v0.13.1-0.20220922081603-45de9fc588a8/go.mod h1:Hi+sz github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -179,6 +188,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -203,6 +214,7 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -223,6 +235,7 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= @@ -332,6 +345,9 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -352,6 +368,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -365,6 +382,7 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7y github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -376,6 +394,7 @@ go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -418,6 +437,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -447,10 +467,12 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -462,6 +484,7 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -471,8 +494,11 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -514,6 +540,7 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -577,6 +604,7 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -632,6 +660,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -652,7 +681,10 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -669,6 +701,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -681,6 +715,7 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/objstore.go b/objstore.go index 19e40ea2..cbe49ecf 100644 --- a/objstore.go +++ b/objstore.go @@ -443,13 +443,6 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu Help: "Second timestamp of the last successful upload to the bucket.", ConstLabels: prometheus.Labels{"bucket": name}, }), - - opsWrittenBytes: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ - Name: "objstore_bucket_operation_written_bytes", - Help: "Number of bytes uploaded from TSDB block, per operation.", - ConstLabels: prometheus.Labels{"bucket": name}, - Buckets: prometheus.ExponentialBuckets(50<<100, 2, 10), - }, []string{"operation"}), } for _, op := range []string{ OpIter, From 72636c357fb92c99c71387df897ab332e17b3b80 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 10:59:41 +0100 Subject: [PATCH 028/130] [FIX] Dependencies Signed-off-by: rita.canavarro --- go.mod | 30 +++++++++++++++--------------- go.sum | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 5bc96bf2..fcbc4bb8 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/thanos-io/objstore go 1.18 require ( - cloud.google.com/go/storage v1.27.0 + cloud.google.com/go/storage v1.29.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go-v2 v1.16.0 github.com/aws/aws-sdk-go-v2/config v1.15.1 @@ -24,18 +24,18 @@ require ( go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.8.0 + golang.org/x/oauth2 v0.11.0 golang.org/x/sync v0.3.0 - google.golang.org/api v0.103.0 + google.golang.org/api v0.126.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.107.0 // indirect - cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go v0.110.2 // indirect + cloud.google.com/go/compute v1.20.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.8.0 // indirect + cloud.google.com/go/iam v0.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect @@ -64,8 +64,8 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect @@ -86,15 +86,15 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.12.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/grpc v1.55.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) @@ -105,5 +105,5 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.11.0 // indirect + golang.org/x/crypto v0.12.0 // indirect ) diff --git a/go.sum b/go.sum index 5465ea17..a30a4d66 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -23,12 +25,15 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg= +cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -41,6 +46,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= @@ -231,10 +237,12 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -407,6 +415,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -475,6 +484,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -485,6 +496,8 @@ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -547,6 +560,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -559,6 +573,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -629,6 +644,7 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -669,6 +685,7 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -687,6 +704,7 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -703,6 +721,8 @@ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From ae27ef19f98d0aac57363dc475fd74d718828f0e Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 11:02:08 +0100 Subject: [PATCH 029/130] [FIX] Dependencies Signed-off-by: rita.canavarro --- go.mod | 2 ++ go.sum | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/go.mod b/go.mod index fcbc4bb8..9d8e4993 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,8 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index a30a4d66..c1a657db 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,7 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -46,6 +47,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= @@ -237,11 +239,13 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -415,6 +419,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -560,6 +565,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -573,6 +579,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -644,6 +651,7 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -685,7 +693,12 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -704,6 +717,7 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 510968f7723310f4bdf1d5ed2f648f74fca577c9 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 11:04:19 +0100 Subject: [PATCH 030/130] [FIX] Remove opsWrittenBytes Signed-off-by: rita.canavarro --- objstore.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/objstore.go b/objstore.go index cbe49ecf..fa4e8ab8 100644 --- a/objstore.go +++ b/objstore.go @@ -478,7 +478,6 @@ type metricBucket struct { opsFetchedBytes *prometheus.CounterVec opsTransferredBytes *prometheus.HistogramVec - opsWrittenBytes *prometheus.HistogramVec opsDuration *prometheus.HistogramVec lastSuccessfulUploadTime prometheus.Gauge } @@ -490,7 +489,6 @@ func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { opsFailures: b.opsFailures, opsFetchedBytes: b.opsFetchedBytes, opsTransferredBytes: b.opsTransferredBytes, - opsWrittenBytes: b.opsWrittenBytes, isOpFailureExpected: fn, opsDuration: b.opsDuration, lastSuccessfulUploadTime: b.lastSuccessfulUploadTime, From 4fe2701f2bae382db3bd3e31d61f4e00462d92e2 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 15:04:18 +0100 Subject: [PATCH 031/130] [FEAT] Clean up Signed-off-by: rita.canavarro --- objstore.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/objstore.go b/objstore.go index fa4e8ab8..e130215e 100644 --- a/objstore.go +++ b/objstore.go @@ -603,12 +603,10 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.lastSuccessfulUploadTime.SetToCurrentTime() b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) - rc := nopCloserWithObjectSize{ - Reader: r, - } - trc := newTimingReadCloser( - rc, + nopCloserWithObjectSize{ + Reader: r, + }, op, b.opsDuration, b.opsFailures, From 62c969d6b3758566f9e5ebe6472d0d8900065e69 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 8 Aug 2023 12:54:56 +0100 Subject: [PATCH 032/130] [FIX] Wrapping of reader Signed-off-by: rita.canavarro --- objstore.go | 19 +++++++++---------- objstore_test.go | 14 +++++++------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/objstore.go b/objstore.go index e130215e..54d4eccd 100644 --- a/objstore.go +++ b/objstore.go @@ -593,16 +593,6 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.ops.WithLabelValues(op).Inc() start := time.Now() - err := b.bkt.Upload(ctx, name, r) - if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { - b.opsFailures.WithLabelValues(op).Inc() - } - return err - } - b.lastSuccessfulUploadTime.SetToCurrentTime() - b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) - trc := newTimingReadCloser( nopCloserWithObjectSize{ Reader: r, @@ -614,6 +604,15 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.opsFetchedBytes, b.opsTransferredBytes, ) + err := b.bkt.Upload(ctx, name, trc) + if err != nil { + if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.opsFailures.WithLabelValues(op).Inc() + } + return err + } + b.lastSuccessfulUploadTime.SetToCurrentTime() + b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) trc.Close() diff --git a/objstore_test.go b/objstore_test.go index 6c893ab0..aa73b2bd 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -126,7 +126,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_fetched_bytes_total{bucket="",operation="get"} 1.048578e+06 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="get_range"} 0 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="iter"} 0 - objstore_bucket_operation_fetched_bytes_total{bucket="",operation="upload"} 0 + objstore_bucket_operation_fetched_bytes_total{bucket="",operation="upload"} 1.048578e+06 `), `objstore_bucket_operation_fetched_bytes_total`)) testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` @@ -170,11 +170,11 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="+Inf"} 0 objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="get_range"} 0 objstore_bucket_operation_transferred_bytes_count{bucket="",operation="get_range"} 0 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="32768"} 3 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="65536"} 3 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="131072"} 3 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="262144"} 3 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="524288"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="32768"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="65536"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="131072"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="262144"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="524288"} 2 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.048576e+06"} 3 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="2.097152e+06"} 3 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="4.194304e+06"} 3 @@ -187,7 +187,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="5.36870912e+08"} 3 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.073741824e+09"} 3 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="+Inf"} 3 - objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="upload"} 0 + objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="upload"} 1.048578e+06 objstore_bucket_operation_transferred_bytes_count{bucket="",operation="upload"} 3 `), `objstore_bucket_operation_transferred_bytes`)) From db8d21c070dec982ed72c22c7e07599b6ccadaa6 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 8 Aug 2023 14:12:24 +0100 Subject: [PATCH 033/130] [FEAT] Minor updates Signed-off-by: rita.canavarro --- objstore.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/objstore.go b/objstore.go index 54d4eccd..049ceddc 100644 --- a/objstore.go +++ b/objstore.go @@ -594,9 +594,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err start := time.Now() trc := newTimingReadCloser( - nopCloserWithObjectSize{ - Reader: r, - }, + io.NopCloser(r), op, b.opsDuration, b.opsFailures, @@ -609,6 +607,8 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } + + trc.Close() return err } b.lastSuccessfulUploadTime.SetToCurrentTime() From 3554ffd1569c54730b646e6edad88aef22ebaa6a Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 8 Aug 2023 14:22:55 +0100 Subject: [PATCH 034/130] [REVERT] GO mod Signed-off-by: rita.canavarro --- go.mod | 51 ++++++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 9d8e4993..dee14caa 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/thanos-io/objstore go 1.18 require ( - cloud.google.com/go/storage v1.29.0 + cloud.google.com/go/storage v1.27.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go-v2 v1.16.0 github.com/aws/aws-sdk-go-v2/config v1.15.1 @@ -18,25 +18,25 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/oracle/oci-go-sdk/v65 v65.41.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.16.0 - github.com/prometheus/common v0.44.0 + github.com/prometheus/client_golang v1.12.2 + github.com/prometheus/common v0.36.0 github.com/tencentyun/cos-go-sdk-v5 v0.7.40 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.11.0 - golang.org/x/sync v0.3.0 - google.golang.org/api v0.126.0 + golang.org/x/oauth2 v0.4.0 + golang.org/x/sync v0.1.0 + google.golang.org/api v0.103.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.110.2 // indirect - cloud.google.com/go/compute v1.20.1 // indirect + cloud.google.com/go v0.107.0 // indirect + cloud.google.com/go/compute v1.15.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect + cloud.google.com/go/iam v0.8.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect @@ -59,19 +59,18 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.11.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect @@ -80,32 +79,30 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/grpc v1.55.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/grpc v1.53.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.12.0 // indirect + golang.org/x/crypto v0.11.0 // indirect ) From 7e4dd6ac0bbf73c423d7e44a485076d03ee33f4f Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 8 Aug 2023 16:13:13 +0100 Subject: [PATCH 035/130] [FEAT] Changelog Signed-off-by: rita.canavarro --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7231f68a..b1803c3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. -- [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_written_bytes_total` that counts the number of total bytes per written to the bucket operation. +- [#69](https://github.com/thanos-io/objstore/pull/69) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange. +- [#66](https://github.com/thanos-io/objstore/pull/66) Made `objstore_bucket_operation_transferred_bytes` also count the number of total bytes per written to the bucket operation Upload. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. From 96b3e99a690e5337b6146a51d1383b1fd5da2b48 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 8 Aug 2023 17:51:00 +0100 Subject: [PATCH 036/130] [FEAT] Improvements Signed-off-by: rita.canavarro --- CHANGELOG.md | 2 +- go.sum | 93 ------------------------------------------------ objstore.go | 8 +++-- objstore_test.go | 2 +- 4 files changed, 8 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1803c3a..cf10fe59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. - [#69](https://github.com/thanos-io/objstore/pull/69) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange. -- [#66](https://github.com/thanos-io/objstore/pull/66) Made `objstore_bucket_operation_transferred_bytes` also count the number of total bytes per written to the bucket operation Upload. +- [#69](https://github.com/thanos-io/objstore/pull/69) [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange and also counts the number of total bytes per written to the bucket operation Upload. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. diff --git a/go.sum b/go.sum index c1a657db..860aca72 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,6 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= -cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= -cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -25,16 +23,12 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= -cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg= -cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -47,19 +41,13 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= -cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 h1:Oj853U9kG+RLTCQXpjvOnrv0WaZHxgmZz1TlLywgOPY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= @@ -77,7 +65,6 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible h1:9gWa46nstkJ9miBReJcN8Gq34cBFbzSpQZVVT9N09TM= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go-v2 v1.16.0 h1:cBAYjiiexRAg9v2z9vb6IdxAa7ef4KCtjW7w7e3GxGo= github.com/aws/aws-sdk-go-v2 v1.16.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI= github.com/aws/aws-sdk-go-v2/config v1.15.1 h1:hTIZFepYESYyowQUBo47lu69WSxsYqGUILY9Nu8+7pY= @@ -120,11 +107,6 @@ github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -139,12 +121,9 @@ github.com/efficientgo/e2e v0.13.1-0.20220922081603-45de9fc588a8/go.mod h1:Hi+sz github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -163,7 +142,6 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -196,8 +174,6 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -222,7 +198,6 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -231,23 +206,16 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= -github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= -github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= @@ -287,8 +255,6 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI= @@ -304,7 +270,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -331,35 +296,23 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.36.0 h1:78hJTing+BLYLjhXE+Z2BubeEymH5Lr0/Mt8FKkxxYo= github.com/prometheus/common v0.36.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -380,7 +333,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -394,7 +346,6 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7y github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -406,7 +357,6 @@ go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -415,12 +365,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -451,7 +397,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -481,16 +426,11 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -500,9 +440,6 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -512,11 +449,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -558,15 +492,11 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -576,11 +506,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -626,7 +553,6 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -651,8 +577,6 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= -google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= -google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -684,7 +608,6 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -693,12 +616,6 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -711,14 +628,9 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -733,10 +645,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -749,7 +657,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/objstore.go b/objstore.go index 049ceddc..24e5d8a1 100644 --- a/objstore.go +++ b/objstore.go @@ -458,6 +458,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) } + // fetched bytes only relevant for get, getrange and upload for _, op := range []string{ OpGet, @@ -599,7 +600,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.opsDuration, b.opsFailures, b.isOpFailureExpected, - b.opsFetchedBytes, + nil, b.opsTransferredBytes, ) err := b.bkt.Upload(ctx, name, trc) @@ -707,7 +708,10 @@ func (rc *timingReadCloser) Close() error { func (rc *timingReadCloser) Read(b []byte) (n int, err error) { n, err = rc.ReadCloser.Read(b) - rc.fetchedBytes.WithLabelValues(rc.op).Add(float64(n)) + if rc.fetchedBytes != nil { + rc.fetchedBytes.WithLabelValues(rc.op).Add(float64(n)) + } + rc.readBytes += int64(n) // Report metric just once. if !rc.alreadyGotErr && err != nil && err != io.EOF { diff --git a/objstore_test.go b/objstore_test.go index aa73b2bd..4b737c16 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -126,7 +126,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_fetched_bytes_total{bucket="",operation="get"} 1.048578e+06 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="get_range"} 0 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="iter"} 0 - objstore_bucket_operation_fetched_bytes_total{bucket="",operation="upload"} 1.048578e+06 + objstore_bucket_operation_fetched_bytes_total{bucket="",operation="upload"} 0 `), `objstore_bucket_operation_fetched_bytes_total`)) testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` From 4253402e19b907624de33ba46c55adb831c8d60c Mon Sep 17 00:00:00 2001 From: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> Date: Wed, 9 Aug 2023 09:07:36 +0100 Subject: [PATCH 037/130] [FEAT] Clean whitespace Co-authored-by: Christian Simon Signed-off-by: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> --- objstore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objstore_test.go b/objstore_test.go index 4b737c16..5feb089b 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -170,7 +170,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="+Inf"} 0 objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="get_range"} 0 objstore_bucket_operation_transferred_bytes_count{bucket="",operation="get_range"} 0 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="32768"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="32768"} 2 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="65536"} 2 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="131072"} 2 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="262144"} 2 From 1e55b009c378eaa669aac1233bc0402b9dec0fa4 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Wed, 9 Aug 2023 09:10:10 +0100 Subject: [PATCH 038/130] [FEAT] Clean up Signed-off-by: rita.canavarro --- CHANGELOG.md | 1 - objstore.go | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf10fe59..42b77fa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. -- [#69](https://github.com/thanos-io/objstore/pull/69) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange. - [#69](https://github.com/thanos-io/objstore/pull/69) [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange and also counts the number of total bytes per written to the bucket operation Upload. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile diff --git a/objstore.go b/objstore.go index 24e5d8a1..f772d51a 100644 --- a/objstore.go +++ b/objstore.go @@ -593,7 +593,6 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err const op = OpUpload b.ops.WithLabelValues(op).Inc() - start := time.Now() trc := newTimingReadCloser( io.NopCloser(r), op, @@ -603,6 +602,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err nil, b.opsTransferredBytes, ) + defer trc.Close() err := b.bkt.Upload(ctx, name, trc) if err != nil { if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { @@ -613,9 +613,6 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err return err } b.lastSuccessfulUploadTime.SetToCurrentTime() - b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) - - trc.Close() return nil } From 35b9ac6b3c062c675e4a43a6224f3f532a424162 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Wed, 9 Aug 2023 09:17:34 +0100 Subject: [PATCH 039/130] [FEAT] Clean up Signed-off-by: rita.canavarro --- providers/azure/azure.go | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/azure/azure.go b/providers/azure/azure.go index f28ab025..a72f3052 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -314,7 +314,6 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { BlockSize: 3 * 1024 * 1024, Concurrency: 4, } - if _, err := blobClient.UploadStream(ctx, r, opts); err != nil { return errors.Wrapf(err, "cannot upload Azure blob, address: %s", name) } From d5dfb062a64e58f35a23f1f5ccebb54ed832fbdf Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Wed, 16 Aug 2023 13:07:27 +0100 Subject: [PATCH 040/130] [FEAT] Clean up Signed-off-by: rita.canavarro --- objstore.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/objstore.go b/objstore.go index f772d51a..5bcc55f1 100644 --- a/objstore.go +++ b/objstore.go @@ -608,8 +608,6 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } - - trc.Close() return err } b.lastSuccessfulUploadTime.SetToCurrentTime() From 47a0da286d6afa0e4208bf2f9880c0abfc5b8559 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Thu, 7 Sep 2023 15:29:33 +0100 Subject: [PATCH 041/130] [FEAT] Update changelog Signed-off-by: rita.canavarro --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42b77fa8..4b9387df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. -- [#69](https://github.com/thanos-io/objstore/pull/69) [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange and also counts the number of total bytes per written to the bucket operation Upload. +- [#69](https://github.com/thanos-io/objstore/pull/69) [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange and also counts the number of total bytes written to the bucket operation Upload. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. From 25673b61e4d9514f3f5d244fc755b5ff224a488f Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Tue, 12 Sep 2023 13:38:16 -0400 Subject: [PATCH 042/130] GCS: use Query.SetAttrSelection when listing Signed-off-by: Andy Asp --- providers/gcs/gcs.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 5ea45c7e..ad305d6e 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -108,10 +108,16 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt delimiter = "" } - it := b.bkt.Objects(ctx, &storage.Query{ + query := &storage.Query{ Prefix: dir, Delimiter: delimiter, - }) + } + err := query.SetAttrSelection([]string{"Name"}) + if err != nil { + return err + } + + it := b.bkt.Objects(ctx, query) for { select { case <-ctx.Done(): From 02d54dc163e2e0bddcc35b52c5e69bb1e33cbd85 Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Tue, 12 Sep 2023 16:09:45 -0400 Subject: [PATCH 043/130] Add changelog entry Signed-off-by: Andy Asp --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b9387df..72f8f180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. +- [#76](https://github.com/thanos-io/objstore/pull/76) GCS: Query for object names only in `Iter` to possibly improve performance when listing objects. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From 0cc6ab5601c744400c2540f52b397897334804ac Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Wed, 20 Sep 2023 12:42:20 -0400 Subject: [PATCH 044/130] Fix metricBucket losing object size in Upload Signed-off-by: Andy Asp --- objstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objstore.go b/objstore.go index 5bcc55f1..20723704 100644 --- a/objstore.go +++ b/objstore.go @@ -594,7 +594,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.ops.WithLabelValues(op).Inc() trc := newTimingReadCloser( - io.NopCloser(r), + NopCloserWithSize(r), op, b.opsDuration, b.opsFailures, From 6175521d7140b1b76cbd7ae53436f09dccb43b4c Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Wed, 20 Sep 2023 13:42:53 -0400 Subject: [PATCH 045/130] Add changelog entry Signed-off-by: Andy Asp --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72f8f180..e26b1548 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#33](https://github.com/thanos-io/objstore/pull/33) Tracing: Add `ContextWithTracer()` to inject the tracer into the context. - [#34](https://github.com/thanos-io/objstore/pull/34) Fix ignored options when creating shared credential Azure client. - [#62](https://github.com/thanos-io/objstore/pull/62) S3: Fix ignored context cancellation in `Iter` method. +- [#77](https://github.com/thanos-io/objstore/pull/77) Fix buckets wrapped with metrics from being unable to determine object sizes in `Upload`. ### Added - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. From ff39b162a6d3bbfc5a9f46097f7d6e7a224fe2fb Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Wed, 20 Sep 2023 12:31:49 -0400 Subject: [PATCH 046/130] S3: clone putUserMetadata map to prevent concurrent modification Signed-off-by: Andy Asp --- providers/s3/s3.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/providers/s3/s3.go b/providers/s3/s3.go index f92d3973..83e3a2de 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -492,6 +492,13 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { if size < int64(partSize) { partSize = 0 } + + // Cloning map since minio may modify it + userMetadata := make(map[string]string, len(b.putUserMetadata)) + for k, v := range b.putUserMetadata { + userMetadata[k] = v + } + if _, err := b.client.PutObject( ctx, b.name, @@ -501,7 +508,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { minio.PutObjectOptions{ PartSize: partSize, ServerSideEncryption: sse, - UserMetadata: b.putUserMetadata, + UserMetadata: userMetadata, StorageClass: b.storageClass, // 4 is what minio-go have as the default. To be certain we do micro benchmark before any changes we // ensure we pin this number to four. From a557e75bcb9ea67a663b6329571b5a1d0794969c Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Wed, 20 Sep 2023 14:16:35 -0400 Subject: [PATCH 047/130] Add changelog entry Signed-off-by: Andy Asp --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e26b1548..0dc65033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#34](https://github.com/thanos-io/objstore/pull/34) Fix ignored options when creating shared credential Azure client. - [#62](https://github.com/thanos-io/objstore/pull/62) S3: Fix ignored context cancellation in `Iter` method. - [#77](https://github.com/thanos-io/objstore/pull/77) Fix buckets wrapped with metrics from being unable to determine object sizes in `Upload`. +- [#78](https://github.com/thanos-io/objstore/pull/78) S3: Fix possible concurrent modification of the PutUserMetadata map. ### Added - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. From 753479c8c8e770adb88b9637f83a4f27a965508f Mon Sep 17 00:00:00 2001 From: gotjosh Date: Tue, 26 Sep 2023 12:27:30 +0100 Subject: [PATCH 048/130] Metrics: Wire up `objstore_bucket_operation_duration_seconds` in `iter` operations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trying to get some statistics in Mimir about the time we're spending in our `iter` operations I realised that despite these operations showing up as part of the labels their value was always 0. Soon enough and with the help of Peter, I realised that these were not wired up in `objstore.go`. This PR does exactly that and measures the duration of the whole `iter` operation. There's one big caveat though, `iter` operations take a callback as an argument which is executed per each item returned. The computed duration will include whatever time we spend on each of these callbacks which might an undesirable effect for users of the library. Sadly, I can't get around this without changing the interface for `iter` to (e.g.) return JUST the time the network operation took. In the meantime, I've done the best I've could without incurring in a big change and documented the behaviour as part of the Histogram's HELP. Co-Authored-By: Peter Štibraný <895919+pstibrany@users.noreply.github.com> Signed-off-by: gotjosh --- go.mod | 2 +- go.sum | 1 - objstore.go | 4 +++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dee14caa..d66fc2fe 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( golang.org/x/oauth2 v0.4.0 golang.org/x/sync v0.1.0 google.golang.org/api v0.103.0 + google.golang.org/grpc v1.53.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) @@ -92,7 +93,6 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.53.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 860aca72..45756f80 100644 --- a/go.sum +++ b/go.sum @@ -491,7 +491,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/objstore.go b/objstore.go index 20723704..3648b878 100644 --- a/objstore.go +++ b/objstore.go @@ -433,7 +433,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu opsDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ Name: "objstore_bucket_operation_duration_seconds", - Help: "Duration of successful operations against the bucket", + Help: "Duration of successful operations against the bucket per operation - iter operations include time spent on each function call.", ConstLabels: prometheus.Labels{"bucket": name}, Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, }, []string{"operation"}), @@ -504,12 +504,14 @@ func (b *metricBucket) Iter(ctx context.Context, dir string, f func(name string) const op = OpIter b.ops.WithLabelValues(op).Inc() + start := time.Now() err := b.bkt.Iter(ctx, dir, f, options...) if err != nil { if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } } + b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return err } From abdf84053ece3533b033aeaf3bd192a7c487ebb8 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Tue, 26 Sep 2023 12:32:02 +0100 Subject: [PATCH 049/130] better words Signed-off-by: gotjosh --- objstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objstore.go b/objstore.go index 3648b878..219a9572 100644 --- a/objstore.go +++ b/objstore.go @@ -433,7 +433,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu opsDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ Name: "objstore_bucket_operation_duration_seconds", - Help: "Duration of successful operations against the bucket per operation - iter operations include time spent on each function call.", + Help: "Duration of successful operations against the bucket per operation - iter operations include time spent on each callback.", ConstLabels: prometheus.Labels{"bucket": name}, Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, }, []string{"operation"}), From 934414af4b0de0ed9ef3bf6aca251c611940d98a Mon Sep 17 00:00:00 2001 From: gotjosh Date: Tue, 26 Sep 2023 12:34:20 +0100 Subject: [PATCH 050/130] Add a changelog entry Signed-off-by: gotjosh --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dc65033..46b1410e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#62](https://github.com/thanos-io/objstore/pull/62) S3: Fix ignored context cancellation in `Iter` method. - [#77](https://github.com/thanos-io/objstore/pull/77) Fix buckets wrapped with metrics from being unable to determine object sizes in `Upload`. - [#78](https://github.com/thanos-io/objstore/pull/78) S3: Fix possible concurrent modification of the PutUserMetadata map. +- [#79](https://github.com/thanos-io/objstore/pull/79) Metrics: Fix `objstore_bucket_operation_duration_seconds` for `iter` operations. ### Added - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. From 07527328bec35a02b9bf1454f01df4711a402c62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:40:34 +0000 Subject: [PATCH 051/130] Bump google.golang.org/grpc from 1.53.0 to 1.56.3 Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.53.0 to 1.56.3. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.53.0...v1.56.3) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 24 ++++++++++++------------ go.sum | 51 ++++++++++++++++++++++++++------------------------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index d66fc2fe..51afcdd0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/thanos-io/objstore go 1.18 require ( - cloud.google.com/go/storage v1.27.0 + cloud.google.com/go/storage v1.28.1 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go-v2 v1.16.0 github.com/aws/aws-sdk-go-v2/config v1.15.1 @@ -24,19 +24,19 @@ require ( go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.4.0 + golang.org/x/oauth2 v0.7.0 golang.org/x/sync v0.1.0 - google.golang.org/api v0.103.0 - google.golang.org/grpc v1.53.0 + google.golang.org/api v0.114.0 + google.golang.org/grpc v1.56.3 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.107.0 // indirect - cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go v0.110.0 // indirect + cloud.google.com/go/compute v1.19.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.8.0 // indirect + cloud.google.com/go/iam v0.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect @@ -60,12 +60,12 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.7.1 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect @@ -92,8 +92,8 @@ require ( golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 45756f80..be5146a4 100644 --- a/go.sum +++ b/go.sum @@ -13,23 +13,23 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= -cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= -cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= -cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -39,8 +39,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= -cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= @@ -172,8 +172,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -197,7 +198,7 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -210,12 +211,12 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= -github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= @@ -438,8 +439,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -574,8 +575,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= -google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -613,8 +614,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -628,8 +629,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -642,8 +643,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 03bc50625ff6c00f383eb6dd025100c09c033fb3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:51:22 +0000 Subject: [PATCH 052/130] Bump golang.org/x/net from 0.12.0 to 0.17.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.12.0 to 0.17.0. - [Commits](https://github.com/golang/net/compare/v0.12.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 51afcdd0..727acbfc 100644 --- a/go.mod +++ b/go.mod @@ -86,9 +86,9 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.12.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -104,5 +104,5 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.11.0 // indirect + golang.org/x/crypto v0.14.0 // indirect ) diff --git a/go.sum b/go.sum index be5146a4..10727b62 100644 --- a/go.sum +++ b/go.sum @@ -366,8 +366,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -430,8 +430,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -495,8 +495,8 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -506,8 +506,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 5ce143530dbecc1a16eac58f49e204309754844b Mon Sep 17 00:00:00 2001 From: sinkingpoint Date: Wed, 8 Nov 2023 11:36:03 +1100 Subject: [PATCH 053/130] Allow turning off CRC32 checksums in S3 Puts https://github.com/thanos-io/thanos/pull/6746 bumped objstore, which bumped minio, which made Thanos incompatible with otherwise compliant S3 backends that do not support the x-amz-checksum header. This adds in a `PutWithMD5` (open to naming changes) config entry to the S3 bucket config that allows reverting back to MD5 checksums which _are_ supported. Signed-off-by: sinkingpoint --- CHANGELOG.md | 1 + providers/s3/s3.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46b1410e..60d9fa90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. - [#76](https://github.com/thanos-io/objstore/pull/76) GCS: Query for object names only in `Iter` to possibly improve performance when listing objects. +- [#85](https://github.com/thanos-io/objstore/pull/85) S3: Allow checksum algorithm to be configured ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 83e3a2de..507b550b 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -113,6 +113,7 @@ var DefaultConfig = Config{ }, PartSize: 1024 * 1024 * 64, // 64MB. BucketLookupType: AutoLookup, + SendContentMd5: true, // Default to using MD5. } // HTTPConfig exists here only because Cortex depends on it, and we depend on Cortex. @@ -136,6 +137,7 @@ type Config struct { TraceConfig TraceConfig `yaml:"trace"` ListObjectsVersion string `yaml:"list_objects_version"` BucketLookupType BucketLookupType `yaml:"bucket_lookup_type"` + SendContentMd5 bool `yaml:"send_content_md5"` // PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize. // NOTE we need to make sure this number does not produce more parts than 10 000. PartSize uint64 `yaml:"part_size"` @@ -166,6 +168,7 @@ type Bucket struct { storageClass string partSize uint64 listObjectsV1 bool + sendContentMd5 bool } // parseConfig unmarshals a buffer into a Config with default values. @@ -334,6 +337,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B storageClass: storageClass, partSize: config.PartSize, listObjectsV1: config.ListObjectsVersion == "v1", + sendContentMd5: config.SendContentMd5, } return bkt, nil } @@ -510,6 +514,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { ServerSideEncryption: sse, UserMetadata: userMetadata, StorageClass: b.storageClass, + SendContentMd5: b.sendContentMd5, // 4 is what minio-go have as the default. To be certain we do micro benchmark before any changes we // ensure we pin this number to four. // TODO(bwplotka): Consider adjusting this number to GOMAXPROCS or to expose this in config if it becomes bottleneck. From 156474886f0ff3af5d24d71aff8aa2ce57b6554e Mon Sep 17 00:00:00 2001 From: Rikhil Shah Date: Fri, 10 Nov 2023 17:31:09 +0000 Subject: [PATCH 054/130] Add support for Azure Workload Identity authentication Signed-off-by: Rikhil Shah --- go.mod | 12 ++++++------ go.sum | 26 +++++++++++++------------- providers/azure/helpers.go | 15 ++++++++++++--- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 727acbfc..b0045140 100644 --- a/go.mod +++ b/go.mod @@ -37,8 +37,8 @@ require ( cloud.google.com/go/compute v1.19.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.13.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.11.0 // indirect @@ -58,12 +58,12 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/gofrs/flock v0.8.1 // indirect - github.com/golang-jwt/jwt/v4 v4.4.3 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.7.1 // indirect github.com/jpillora/backoff v1.0.0 // indirect @@ -99,8 +99,8 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect diff --git a/go.sum b/go.sum index 10727b62..27256767 100644 --- a/go.sum +++ b/go.sum @@ -42,16 +42,16 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 h1:Oj853U9kG+RLTCQXpjvOnrv0WaZHxgmZz1TlLywgOPY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= @@ -111,7 +111,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4 h1:rydBwnBoywKQMjWF0z8SriYtQ+uUcaFsxuijMjJr5PI= @@ -142,8 +142,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= -github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -209,8 +209,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 7b4a5fbe..846394a0 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -63,18 +63,27 @@ func getContainerClient(conf Config) (*container.Client, error) { return containerClient, nil } - // Use MSI for authentication. - msiOpt := &azidentity.ManagedIdentityCredentialOptions{} + // Otherwise use a token credential + var cred azcore.TokenCredential + + // Use Managed Identity Credential if a user assigned ID is set if conf.UserAssignedID != "" { + msiOpt := &azidentity.ManagedIdentityCredentialOptions{} msiOpt.ID = azidentity.ClientID(conf.UserAssignedID) + cred, err = azidentity.NewManagedIdentityCredential(msiOpt) + } else { + // Otherwise use Default Azure Credential + cred, err = azidentity.NewDefaultAzureCredential(nil) } - cred, err := azidentity.NewManagedIdentityCredential(msiOpt) + if err != nil { return nil, err } + containerClient, err := container.NewClient(containerURL, cred, opt) if err != nil { return nil, err } + return containerClient, nil } From 761604ac79929a755ca2f714fafa4a3e4b556132 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 15 Nov 2023 08:06:04 +0100 Subject: [PATCH 055/130] Update google storage library This commit updates cloud.google.com/go/storage to 1.35. Signed-off-by: Filip Petkovski --- go.mod | 44 +++++++++++++------------ go.sum | 100 ++++++++++++++++++++++++++++++++------------------------- 2 files changed, 80 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index b0045140..982eb44e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/thanos-io/objstore go 1.18 require ( - cloud.google.com/go/storage v1.28.1 + cloud.google.com/go/storage v1.35.1 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go-v2 v1.16.0 github.com/aws/aws-sdk-go-v2/config v1.15.1 @@ -24,19 +24,19 @@ require ( go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.7.0 - golang.org/x/sync v0.1.0 - google.golang.org/api v0.114.0 - google.golang.org/grpc v1.56.3 + golang.org/x/oauth2 v0.14.0 + golang.org/x/sync v0.5.0 + google.golang.org/api v0.150.0 + google.golang.org/grpc v1.59.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.19.1 // indirect + cloud.google.com/go v0.110.10 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect + cloud.google.com/go/iam v1.1.5 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect @@ -61,11 +61,11 @@ require ( github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/uuid v1.3.1 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.1 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.4.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect @@ -86,14 +86,16 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.4.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) @@ -104,5 +106,5 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.15.0 // indirect ) diff --git a/go.sum b/go.sum index 27256767..5703ca6c 100644 --- a/go.sum +++ b/go.sum @@ -13,23 +13,22 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= -cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -39,8 +38,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= -cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w= +cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw= @@ -189,8 +188,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= @@ -207,16 +205,18 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= @@ -347,6 +347,7 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7y github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -366,8 +367,9 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -398,6 +400,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -427,11 +430,13 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -439,8 +444,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= +golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -450,8 +455,9 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -492,11 +498,13 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -506,13 +514,14 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydthsj6oOd6l9bpuFcNoUvTs= -golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= +golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -553,12 +562,13 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -575,16 +585,16 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= -google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.150.0 h1:Z9k22qD289SZ8gCJrk4DrWXkNjtfvKAUo/l1ma8eBYE= +google.golang.org/api v0.150.0/go.mod h1:ccy+MJ6nrYFgE3WgRx/AMXOxOmU8Q4hSa+jjibzhxcg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -614,8 +624,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -629,8 +643,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -643,8 +657,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 4898cea91aef971f6d186a8ad3ede20ad8862989 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 15 Nov 2023 08:07:45 +0100 Subject: [PATCH 056/130] Add CHANGELOG entry Signed-off-by: Filip Petkovski --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60d9fa90..dfb2c11b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,4 +44,5 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#65](https://github.com/thanos-io/objstore/pull/65) *: Upgrade minio-go version to `v7.0.61`. - [#70](https://github.com/thanos-io/objstore/pull/70) GCS: Update cloud.google.com/go/storage version to `v1.27.0`. - [#71](https://github.com/thanos-io/objstore/pull/71) Replace method `IsCustomerManagedKeyError` for a more generic `IsAccessDeniedErr` on the bucket interface. +- [#89](https://github.com/thanos-io/objstore/pull/89) GCS: Upgrade cloud.google.com/go/storage version to `v1.35.1`. ### Removed From 16876bca6cb5ac8945f39bdcf8711b76e8fcbccd Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 15 Nov 2023 08:10:59 +0100 Subject: [PATCH 057/130] Update go version Signed-off-by: Filip Petkovski --- .bingo/Variables.mk | 6 +- .bingo/copyright.mod | 2 +- .bingo/golangci-lint.mod | 2 +- .bingo/golangci-lint.sum | 397 ++++++++++++++++++++++++++ .bingo/mdox.mod | 2 +- .bingo/variables.env | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/docs.yaml | 2 +- .go-version | 2 +- Makefile | 2 +- go.mod | 2 +- 11 files changed, 409 insertions(+), 12 deletions(-) diff --git a/.bingo/Variables.mk b/.bingo/Variables.mk index 8f8a0263..7e9e075b 100644 --- a/.bingo/Variables.mk +++ b/.bingo/Variables.mk @@ -41,11 +41,11 @@ $(GOIMPORTS): $(BINGO_DIR)/goimports.mod @echo "(re)installing $(GOBIN)/goimports-v0.0.0-20200526224456-8b020aee10d2" @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=goimports.mod -o=$(GOBIN)/goimports-v0.0.0-20200526224456-8b020aee10d2 "golang.org/x/tools/cmd/goimports" -GOLANGCI_LINT := $(GOBIN)/golangci-lint-v1.46.2 +GOLANGCI_LINT := $(GOBIN)/golangci-lint-v1.54.1 $(GOLANGCI_LINT): $(BINGO_DIR)/golangci-lint.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. - @echo "(re)installing $(GOBIN)/golangci-lint-v1.46.2" - @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=golangci-lint.mod -o=$(GOBIN)/golangci-lint-v1.46.2 "github.com/golangci/golangci-lint/cmd/golangci-lint" + @echo "(re)installing $(GOBIN)/golangci-lint-v1.54.1" + @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=golangci-lint.mod -o=$(GOBIN)/golangci-lint-v1.54.1 "github.com/golangci/golangci-lint/cmd/golangci-lint" MDOX := $(GOBIN)/mdox-v0.9.0 $(MDOX): $(BINGO_DIR)/mdox.mod diff --git a/.bingo/copyright.mod b/.bingo/copyright.mod index 474b10bc..90433b44 100644 --- a/.bingo/copyright.mod +++ b/.bingo/copyright.mod @@ -1,5 +1,5 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT -go 1.18 +go 1.20 require github.com/efficientgo/tools/copyright v0.0.0-20220225185207-fe763185946b diff --git a/.bingo/golangci-lint.mod b/.bingo/golangci-lint.mod index 230e52b5..95a2175e 100644 --- a/.bingo/golangci-lint.mod +++ b/.bingo/golangci-lint.mod @@ -2,4 +2,4 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT go 1.14 -require github.com/golangci/golangci-lint v1.46.2 // cmd/golangci-lint +require github.com/golangci/golangci-lint v1.54.1 // cmd/golangci-lint diff --git a/.bingo/golangci-lint.sum b/.bingo/golangci-lint.sum index dd377b37..5750abd4 100644 --- a/.bingo/golangci-lint.sum +++ b/.bingo/golangci-lint.sum @@ -1,5 +1,9 @@ +4d63.com/gocheckcompilerdirectives v1.2.1 h1:AHcMYuw56NPjq/2y615IGg2kYkBdTvOaojYCBcRE7MA= +4d63.com/gocheckcompilerdirectives v1.2.1/go.mod h1:yjDJSxmDTtIHHCqX0ufRYZDL6vQtMG7tJdKVeWwsqvs= 4d63.com/gochecknoglobals v0.1.0 h1:zeZSRqj5yCg28tCkIV/z/lWbwvNm5qnKVS15PI8nhD0= 4d63.com/gochecknoglobals v0.1.0/go.mod h1:wfdC5ZjKSPr7CybKEcgJhUOgeAQW1+7WcyK8OvUilfo= +4d63.com/gochecknoglobals v0.2.1 h1:1eiorGsgHOFOuoOiJDy2psSrQbRdIHrlge0IJIkUgDc= +4d63.com/gochecknoglobals v0.2.1/go.mod h1:KRE8wtJB3CXCsb1xy421JfTHIIbmT3U5ruxw2Qu8fSU= bitbucket.org/creachadair/shell v0.0.6/go.mod h1:8Qqi/cYk7vPnsOePHroKXDJYmb5x7ENhtiFtfZq8K+M= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -43,6 +47,8 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= @@ -60,20 +66,33 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/4meepo/tagalign v1.3.2 h1:1idD3yxlRGV18VjqtDbqYvQ5pXqQS0wO2dn6M3XstvI= +github.com/4meepo/tagalign v1.3.2/go.mod h1:Q9c1rYMZJc9dPRkbQPpcBNCLEmY2njbAsXhQOZFE2dE= +github.com/Abirdcfly/dupword v0.0.12 h1:56NnOyrXzChj07BDFjeRA+IUzSz01jmzEq+G4kEgFhc= +github.com/Abirdcfly/dupword v0.0.12/go.mod h1:+us/TGct/nI9Ndcbcp3rgNcQzctTj68pq7TcgNpLfdI= github.com/Antonboom/errname v0.1.6 h1:LzIJZlyLOCSu51o3/t2n9Ck7PcoP9wdbrdaW6J8fX24= github.com/Antonboom/errname v0.1.6/go.mod h1:7lz79JAnuoMNDAWE9MeeIr1/c/VpSUWatBv2FH9NYpI= +github.com/Antonboom/errname v0.1.10 h1:RZ7cYo/GuZqjr1nuJLNe8ZH+a+Jd9DaZzttWzak9Bls= +github.com/Antonboom/errname v0.1.10/go.mod h1:xLeiCIrvVNpUtsN0wxAh05bNIZpqE22/qDMnTBTttiA= github.com/Antonboom/nilnil v0.1.1 h1:PHhrh5ANKFWRBh7TdYmyyq2gyT2lotnvFvvFbylF81Q= github.com/Antonboom/nilnil v0.1.1/go.mod h1:L1jBqoWM7AOeTD+tSquifKSesRHs4ZdaxvZR+xdJEaI= +github.com/Antonboom/nilnil v0.1.5 h1:X2JAdEVcbPaOom2TUa1FxZ3uyuUlex0XMLGYMemu6l0= +github.com/Antonboom/nilnil v0.1.5/go.mod h1:I24toVuBKhfP5teihGWctrRiPbRKHwZIFOvc6v3HZXk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 h1:sHglBQTwgx+rWPdisA5ynNEsoARbiCBOyGcJM4/OzsM= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/GaijinEntertainment/go-exhaustruct/v2 v2.1.0 h1:LAPPhJ4KR5Z8aKVZF5S48csJkxL5RMKmE/98fMs1u5M= github.com/GaijinEntertainment/go-exhaustruct/v2 v2.1.0/go.mod h1:LGOGuvEgCfCQsy3JF2tRmpGDpzA53iZfyGEWSPwQ6/4= +github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0 h1:3ZBs7LAezy8gh0uECsA6CGU43FF3zsx5f4eah5FxTMA= +github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0/go.mod h1:rZLTje5A9kFBe0pzhpe2TdhRniBF++PRHQuRpR8esVc= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= @@ -83,13 +102,19 @@ github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuN github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenPeeDeeP/depguard v1.1.0 h1:pjK9nLPS1FwQYGGpPxoMYpe7qACHOhAWQMQzV71i49o= github.com/OpenPeeDeeP/depguard v1.1.0/go.mod h1:JtAMzWkmFEzDPyAd+W0NHl1lvpQKTvT9jnRVsohBKpc= +github.com/OpenPeeDeeP/depguard/v2 v2.1.0 h1:aQl70G173h/GZYhWf36aE5H0KaujXfVMnn/f1kSDVYY= +github.com/OpenPeeDeeP/depguard/v2 v2.1.0/go.mod h1:PUBgk35fX4i7JDmwzlJwJ+GMe6NfO1723wmJMgPThNQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alexkohler/nakedret/v2 v2.0.2 h1:qnXuZNvv3/AxkAb22q/sEsEpcA99YxLFACDtEw9TPxE= +github.com/alexkohler/nakedret/v2 v2.0.2/go.mod h1:2b8Gkk0GsOrqQv/gPWjNLDSKwG8I5moSXG1K4VIBcTQ= github.com/alexkohler/prealloc v1.0.0 h1:Hbq0/3fJPQhNkN0dR95AVrr6R7tou91y0uHG5pOcUuw= github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE= +github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw= +github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= @@ -103,11 +128,14 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/ashanbrown/forbidigo v1.3.0 h1:VkYIwb/xxdireGAdJNZoo24O4lmnEWkactplBlWTShc= github.com/ashanbrown/forbidigo v1.3.0/go.mod h1:vVW7PEdqEFqapJe95xHkTfB1+XvZXBFg8t0sG2FIxmI= +github.com/ashanbrown/forbidigo v1.6.0 h1:D3aewfM37Yb3pxHujIPSpTf6oQk9sc9WZi8gerOIVIY= +github.com/ashanbrown/forbidigo v1.6.0/go.mod h1:Y8j9jy9ZYAEHXdu723cUlraTqbzjKF1MUyfOKL+AjcU= github.com/ashanbrown/makezero v1.1.1 h1:iCQ87C0V0vSyO+M9E/FZYbu65auqH0lnsOkf5FcB28s= github.com/ashanbrown/makezero v1.1.1/go.mod h1:i1bJLCRSCHOcOa9Y6MyF2FTfMZMFdHvxKHxgO5Z1axI= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.36.30/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -115,16 +143,28 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bkielbasa/cyclop v1.2.0 h1:7Jmnh0yL2DjKfw28p86YTd/B4lRGcNuu12sKE35sM7A= github.com/bkielbasa/cyclop v1.2.0/go.mod h1:qOI0yy6A7dYC4Zgsa72Ppm9kONl0RoIlPbzot9mhmeI= +github.com/bkielbasa/cyclop v1.2.1 h1:AeF71HZDob1P2/pRm1so9cd1alZnrpyc4q2uP2l0gJY= +github.com/bkielbasa/cyclop v1.2.1/go.mod h1:K/dT/M0FPAiYjBgQGau7tz+3TMh4FWAEqlMhzFWCrgM= github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= github.com/bombsimon/wsl/v3 v3.3.0 h1:Mka/+kRLoQJq7g2rggtgQsjuI/K5Efd87WX96EWFxjM= github.com/bombsimon/wsl/v3 v3.3.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= +github.com/bombsimon/wsl/v3 v3.4.0 h1:RkSxjT3tmlptwfgEgTgU+KYKLI35p/tviNXNXiL2aNU= +github.com/bombsimon/wsl/v3 v3.4.0/go.mod h1:KkIB+TXkqy6MvK9BDZVbZxKNYsE1/oLRJbIFtf14qqo= github.com/breml/bidichk v0.2.3 h1:qe6ggxpTfA8E75hdjWPZ581sY3a2lnl0IRxLQFelECI= github.com/breml/bidichk v0.2.3/go.mod h1:8u2C6DnAy0g2cEq+k/A2+tr9O1s+vHGxWn0LTc70T2A= +github.com/breml/bidichk v0.2.4 h1:i3yedFWWQ7YzjdZJHnPo9d/xURinSq3OM+gyM43K4/8= +github.com/breml/bidichk v0.2.4/go.mod h1:7Zk0kRFt1LIZxtQdl9W9JwGAcLTTkOs+tN7wuEYGJ3s= github.com/breml/errchkjson v0.3.0 h1:YdDqhfqMT+I1vIxPSas44P+9Z9HzJwCeAzjB8PxP1xw= github.com/breml/errchkjson v0.3.0/go.mod h1:9Cogkyv9gcT8HREpzi3TiqBxCqDzo8awa92zSDFcofU= +github.com/breml/errchkjson v0.3.1 h1:hlIeXuspTyt8Y/UmP5qy1JocGNR00KQHgfaNtRAjoxQ= +github.com/breml/errchkjson v0.3.1/go.mod h1:XroxrzKjdiutFyW3nWhw34VGg7kiMsDQox73yWCGI2U= github.com/butuzov/ireturn v0.1.1 h1:QvrO2QF2+/Cx1WA/vETCIYBKtRjc30vesdoPUNo1EbY= github.com/butuzov/ireturn v0.1.1/go.mod h1:Wh6Zl3IMtTpaIKbmwzqi6olnM9ptYQxxVacMsOEFPoc= +github.com/butuzov/ireturn v0.2.0 h1:kCHi+YzC150GE98WFuZQu9yrTn6GEydO2AuPLbTgnO4= +github.com/butuzov/ireturn v0.2.0/go.mod h1:Wh6Zl3IMtTpaIKbmwzqi6olnM9ptYQxxVacMsOEFPoc= +github.com/butuzov/mirror v1.1.0 h1:ZqX54gBVMXu78QLoiqdwpl2mgmoOJTk7s4p4o+0avZI= +github.com/butuzov/mirror v1.1.0/go.mod h1:8Q0BdQU6rC6WILDiBM60DBfvV78OLJmMmixe7GF45AE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -134,8 +174,12 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charithe/durationcheck v0.0.9 h1:mPP4ucLrf/rKZiIG/a9IPXHGlh8p4CzgpyTy6EEutYk= github.com/charithe/durationcheck v0.0.9/go.mod h1:SSbRIBVfMjCi/kEB6K65XEA83D6prSM8ap1UCpNKtgg= +github.com/charithe/durationcheck v0.0.10 h1:wgw73BiocdBDQPik+zcEoBG/ob8uyBHf2iyoHGPf5w4= +github.com/charithe/durationcheck v0.0.10/go.mod h1:bCWXb7gYRysD1CU3C+u4ceO49LoGOY1C1L6uouGNreQ= github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4 h1:tFXjAxje9thrTF4h57Ckik+scJjTWdwAtZqZPtOT48M= github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4/go.mod h1:W8EnPSQ8Nv4fUjc/v1/8tHFqhuOJXnRub0dTfuAQktU= +github.com/chavacava/garif v0.0.0-20230227094218-b8c73b2037b8 h1:W9o46d2kbNL06lq7UNDPV0zYLzkrde/bjIqO02eoll0= +github.com/chavacava/garif v0.0.0-20230227094218-b8c73b2037b8/go.mod h1:gakxgyXaaPkxvLw1XQxNGK4I37ys9iBRzNUx/B7pUCo= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -166,10 +210,16 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cristalhq/acmd v0.11.1/go.mod h1:LG5oa43pE/BbxtfMoImHCQN++0Su7dzipdgBjMCBVDQ= +github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo= +github.com/curioswitch/go-reassign v0.2.0/go.mod h1:x6OpXuWvgfQaMGks2BZybTngWjT84hqJfKoO8Tt/Roc= github.com/daixiang0/gci v0.3.3 h1:55xJKH7Gl9Vk6oQ1cMkwrDWjAkT1D+D1G9kNmRcAIY4= github.com/daixiang0/gci v0.3.3/go.mod h1:1Xr2bxnQbDxCqqulUOv8qpGqkgRw9RSCGGjEC2LjF8o= +github.com/daixiang0/gci v0.11.0 h1:XeQbFKkCRxvVyn06EOuNY6LPGBLVuB/W130c8FrnX6A= +github.com/daixiang0/gci v0.11.0/go.mod h1:xtHP9N7AHdNvtRNfcx9gwTDfw7FRJx4bZUsiEfiNNAI= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -188,6 +238,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.0.14/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= @@ -200,11 +251,17 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/firefart/nonamedreturns v1.0.1 h1:fSvcq6ZpK/uBAgJEGMvzErlzyM4NELLqqdTofVjVNag= github.com/firefart/nonamedreturns v1.0.1/go.mod h1:D3dpIBojGGNh5UfElmwPu73SwDCm+VKhHYqwlNOk2uQ= +github.com/firefart/nonamedreturns v1.0.4 h1:abzI1p7mAEPYuR4A+VLKn4eNDOycjYo2phmY9sfv40Y= +github.com/firefart/nonamedreturns v1.0.4/go.mod h1:TDhe/tjI1BXo48CmYbUduTV7BdIga8MAO/xbKdcVsGI= github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -213,9 +270,13 @@ github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmV github.com/fullstorydev/grpcurl v1.6.0/go.mod h1:ZQ+ayqbKMJNhzLmbpCiurTVlaK2M/3nqZCxaQ2Ze/sM= github.com/fzipp/gocyclo v0.5.1 h1:L66amyuYogbxl0j2U+vGqJXusPF2IkduvXLnYD5TFgw= github.com/fzipp/gocyclo v0.5.1/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= +github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= +github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-critic/go-critic v0.6.3 h1:abibh5XYBTASawfTQ0rA7dVtQT+6KzpGqb/J+DxRDaw= github.com/go-critic/go-critic v0.6.3/go.mod h1:c6b3ZP1MQ7o6lPR7Rv3lEf7pYQUmAcx8ABHgdZCQt/k= +github.com/go-critic/go-critic v0.9.0 h1:Pmys9qvU3pSML/3GEQ2Xd9RZ/ip+aXHKILuxczKGV/U= +github.com/go-critic/go-critic v0.9.0/go.mod h1:5P8tdXL7m/6qnyG6oRAlYLORvoXH0WDypYgAEmagT40= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -225,30 +286,55 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= +github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU= github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= +github.com/go-toolsmith/astcopy v1.0.2/go.mod h1:4TcEdbElGc9twQEYpVo/aieIXfHhiuLh4aLAck6dO7Y= +github.com/go-toolsmith/astcopy v1.1.0 h1:YGwBN0WM+ekI/6SS6+52zLDEf8Yvp3n2seZITCUBt5s= +github.com/go-toolsmith/astcopy v1.1.0/go.mod h1:hXM6gan18VA1T/daUEHCFcYiW8Ai1tIwIzHY6srfEAw= github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= github.com/go-toolsmith/astequal v1.0.1 h1:JbSszi42Jiqu36Gnf363HWS9MTEAz67vTQLponh3Moc= github.com/go-toolsmith/astequal v1.0.1/go.mod h1:4oGA3EZXTVItV/ipGiOx7NWkY5veFfcsOJVS2YxltLw= +github.com/go-toolsmith/astequal v1.0.2/go.mod h1:9Ai4UglvtR+4up+bAD4+hCj7iTo4m/OXVTSLnCyTAx4= +github.com/go-toolsmith/astequal v1.0.3/go.mod h1:9Ai4UglvtR+4up+bAD4+hCj7iTo4m/OXVTSLnCyTAx4= +github.com/go-toolsmith/astequal v1.1.0 h1:kHKm1AWqClYn15R0K1KKE4RG614D46n+nqUQ06E1dTw= +github.com/go-toolsmith/astequal v1.1.0/go.mod h1:sedf7VIdCL22LD8qIvv7Nn9MuWJruQA/ysswh64lffQ= github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k= github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= +github.com/go-toolsmith/astfmt v1.1.0 h1:iJVPDPp6/7AaeLJEruMsBUlOYCmvg0MoCfJprsOmcco= +github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlNMV634mhwuQ4= github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg= github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= +github.com/go-toolsmith/astp v1.1.0 h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA= +github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA= github.com/go-toolsmith/pkgload v1.0.2-0.20220101231613-e814995d17c5/go.mod h1:3NAwwmD4uY/yggRxoEjk/S00MIV3A+H7rrE3i87eYxM= +github.com/go-toolsmith/pkgload v1.2.2/go.mod h1:R2hxLNRKuAsiXCo2i5J6ZQPhnPMOVtU+f0arbFPWCus= github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4= github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw= +github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= github.com/go-toolsmith/typep v1.0.2 h1:8xdsa1+FSIH/RhEkgnD1j2CJOy5mNllW1Q9tRiYwvlk= github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus= +github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b h1:khEcpUM4yFcxg4/FHQWkvVRmgijNXRfzkIDHh23ggEo= github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= +github.com/go-xmlfmt/xmlfmt v1.1.2 h1:Nea7b4icn8s57fTx1M5AI4qQT5HEM3rVUO8MuE6g80U= +github.com/go-xmlfmt/xmlfmt v1.1.2/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -293,6 +379,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= @@ -302,16 +390,24 @@ github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe h1:6RGUuS7EGotKx6 github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe/go.mod h1:gjqyPShc/m8pEMpk0a3SeagVb0kaqvhscv+i9jI5ZhQ= github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks= github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/gofmt v0.0.0-20220901101216-f2edd75033f2 h1:amWTbTGqOZ71ruzrdA+Nx5WA3tV1N0goTspwmKCQvBY= +github.com/golangci/gofmt v0.0.0-20220901101216-f2edd75033f2/go.mod h1:9wOXstvyDRshQ9LggQuzBCGysxs3b6Uo/1MvYCR2NMs= github.com/golangci/golangci-lint v1.46.2 h1:o90t/Xa6dhJbvy8Bz2RpzUXqrkigp19DLStMolTZbyo= github.com/golangci/golangci-lint v1.46.2/go.mod h1:3DkdHnxn9eoTTrpT2gB0TEv8KSziuoqe9FitgQLHvAY= +github.com/golangci/golangci-lint v1.54.1 h1:0qMrH1gkeIBqCZaaAm5Fwq4xys9rO/lJofHfZURIFFk= +github.com/golangci/golangci-lint v1.54.1/go.mod h1:JK47+qksV/t2mAz9YvndwT0ZLW4A1rvDljOs3g9jblo= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= github.com/golangci/misspell v0.3.5 h1:pLzmVdl3VxTOncgzHcvLOKirdvcx/TydsClUQXTehjo= github.com/golangci/misspell v0.3.5/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/misspell v0.4.1 h1:+y73iSicVy2PqyX7kmUefHusENlrP9YwuHZHPLGQj/g= +github.com/golangci/misspell v0.4.1/go.mod h1:9mAN1quEo3DlpbaIKKyEvRxK1pwqR9s/Sea1bJCtlNI= github.com/golangci/revgrep v0.0.0-20210930125155-c22e5001d4f2 h1:SgM7GDZTxtTTQPU84heOxy34iG5Du7F2jcoZnvp+fXI= github.com/golangci/revgrep v0.0.0-20210930125155-c22e5001d4f2/go.mod h1:LK+zW4MpyytAWQRz0M4xnzEk50lSvqDQKfx304apFkY= +github.com/golangci/revgrep v0.0.0-20220804021717-745bb2f7c2e6 h1:DIPQnGy2Gv2FSA4B/hh8Q7xx3B7AIDk3DAMeHclH1vQ= +github.com/golangci/revgrep v0.0.0-20220804021717-745bb2f7c2e6/go.mod h1:0AKcRCkMoKvUvlf89F6O7H2LYdhr1zBh736mBItOdRs= github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys= github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -332,6 +428,9 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -367,11 +466,15 @@ github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pf github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= +github.com/gookit/color v1.5.3/go.mod h1:NUzwzeehUfl7GIb36pqId+UGmRfQcU/WiiyTTeNjHtE= github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= github.com/gordonklaus/ineffassign v0.0.0-20210914165742-4cc7213b9bc8 h1:PVRE9d4AQKmbelZ7emNig1+NT27DUmKZn5qXxfio54U= github.com/gordonklaus/ineffassign v0.0.0-20210914165742-4cc7213b9bc8/go.mod h1:Qcp2HIAYhR7mNUVSIxZww3Guk4it82ghYcEXIAk+QT0= +github.com/gordonklaus/ineffassign v0.0.0-20230610083614-0e73809eb601 h1:mrEEilTAUmaAORhssPPkxj84TsHrPMLBGW2Z4SoTxm8= +github.com/gordonklaus/ineffassign v0.0.0-20230610083614-0e73809eb601/go.mod h1:Qcp2HIAYhR7mNUVSIxZww3Guk4it82ghYcEXIAk+QT0= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -426,6 +529,8 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4= github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -450,6 +555,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jgautheron/goconst v1.5.1 h1:HxVbL1MhydKs8R8n/HE5NPvzfaYmQJA3o879lE4+WcM= github.com/jgautheron/goconst v1.5.1/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4= github.com/jhump/protoreflect v1.6.1/go.mod h1:RZQ/lnuN+zqeRVpQigTwO6o0AJUkxbnSnpuG7toUTG4= @@ -461,6 +568,7 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/txtarfs v0.0.0-20210218200122-0702f000015a/go.mod h1:izVPOvVRsHiKkeGCT6tYBNWyDVuzj9wAaBb5R9qamfw= @@ -484,8 +592,12 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.6.0 h1:YTDO4pNy7AUN/021p+JGHycQyYNIyMoenM1YDVK6RlY= github.com/kisielk/errcheck v1.6.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/errcheck v1.6.3 h1:dEKh+GLHcWm2oN34nMvDzn1sqI0i0WxPvrgiJA5JuM8= +github.com/kisielk/errcheck v1.6.3/go.mod h1:nXw/i/MfnvRHqXa7XXmQMUB0oNFGuBrNI8d8NLy0LPw= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkHAIKE/contextcheck v1.1.4 h1:B6zAaLhOEEcjvUgIYEqystmnFk1Oemn8bvJhbt0GMb8= +github.com/kkHAIKE/contextcheck v1.1.4/go.mod h1:1+i/gWqokIa+dm31mqGLZhZJ7Uh44DJGZVmr6QRBNJg= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -496,39 +608,59 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kulti/thelper v0.6.2 h1:K4xulKkwOCnT1CDms6Ex3uG1dvSMUUQe9zxgYQgbRXs= github.com/kulti/thelper v0.6.2/go.mod h1:DsqKShOvP40epevkFrvIwkCMNYxMeTNjdWL4dqWHZ6I= +github.com/kulti/thelper v0.6.3 h1:ElhKf+AlItIu+xGnI990no4cE2+XaSu1ULymV2Yulxs= +github.com/kulti/thelper v0.6.3/go.mod h1:DsqKShOvP40epevkFrvIwkCMNYxMeTNjdWL4dqWHZ6I= github.com/kunwardeep/paralleltest v1.0.3 h1:UdKIkImEAXjR1chUWLn+PNXqWUGs//7tzMeWuP7NhmI= github.com/kunwardeep/paralleltest v1.0.3/go.mod h1:vLydzomDFpk7yu5UX02RmP0H8QfRPOV/oFhWN85Mjb4= +github.com/kunwardeep/paralleltest v1.0.8 h1:Ul2KsqtzFxTlSU7IP0JusWlLiNqQaloB9vguyjbE558= +github.com/kunwardeep/paralleltest v1.0.8/go.mod h1:2C7s65hONVqY7Q5Efj5aLzRCNLjw2h4eMc9EcypGjcY= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kyoh86/exportloopref v0.1.8 h1:5Ry/at+eFdkX9Vsdw3qU4YkvGtzuVfzT4X7S77LoN/M= github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg= +github.com/kyoh86/exportloopref v0.1.11 h1:1Z0bcmTypkL3Q4k+IDHMWTcnCliEZcaPiIe0/ymEyhQ= +github.com/kyoh86/exportloopref v0.1.11/go.mod h1:qkV4UF1zGl6EkF1ox8L5t9SwyeBAZ3qLMd6up458uqA= github.com/ldez/gomoddirectives v0.2.3 h1:y7MBaisZVDYmKvt9/l1mjNCiSA1BVn34U0ObUcJwlhA= github.com/ldez/gomoddirectives v0.2.3/go.mod h1:cpgBogWITnCfRq2qGoDkKMEVSaarhdBr6g8G04uz6d0= github.com/ldez/tagliatelle v0.3.1 h1:3BqVVlReVUZwafJUwQ+oxbx2BEX2vUG4Yu/NOfMiKiM= github.com/ldez/tagliatelle v0.3.1/go.mod h1:8s6WJQwEYHbKZDsp/LjArytKOG8qaMrKQQ3mFukHs88= +github.com/ldez/tagliatelle v0.5.0 h1:epgfuYt9v0CG3fms0pEgIMNPuFf/LpPIfjk4kyqSioo= +github.com/ldez/tagliatelle v0.5.0/go.mod h1:rj1HmWiL1MiKQuOONhd09iySTEkUuE/8+5jtPYz9xa4= github.com/leonklingele/grouper v1.1.0 h1:tC2y/ygPbMFSBOs3DcyaEMKnnwH7eYKzohOtRrf0SAg= github.com/leonklingele/grouper v1.1.0/go.mod h1:uk3I3uDfi9B6PeUjsCKi6ndcf63Uy7snXgR4yDYQVDY= +github.com/leonklingele/grouper v1.1.1 h1:suWXRU57D4/Enn6pXR0QVqqWWrnJ9Osrz+5rjt8ivzU= +github.com/leonklingele/grouper v1.1.1/go.mod h1:uk3I3uDfi9B6PeUjsCKi6ndcf63Uy7snXgR4yDYQVDY= github.com/letsencrypt/pkcs11key/v4 v4.0.0/go.mod h1:EFUvBDay26dErnNb70Nd0/VW3tJiIbETBPTl9ATXQag= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lufeee/execinquery v1.2.1 h1:hf0Ems4SHcUGBxpGN7Jz78z1ppVkP/837ZlETPCEtOM= github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/maratori/testableexamples v1.0.0 h1:dU5alXRrD8WKSjOUnmJZuzdxWOEQ57+7s93SLMxb2vI= +github.com/maratori/testableexamples v1.0.0/go.mod h1:4rhjL1n20TUTT4vdh3RDqSizKLyXp7K2u6HgraZCGzE= github.com/maratori/testpackage v1.0.1 h1:QtJ5ZjqapShm0w5DosRjg0PRlSdAdlx+W6cCKoALdbQ= github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU= +github.com/maratori/testpackage v1.1.1 h1:S58XVV5AD7HADMmD0fNnziNHqKvSdDuEKdPD1rNTU04= +github.com/maratori/testpackage v1.1.1/go.mod h1:s4gRK/ym6AMrqpOa/kEbQTV4Q4jb7WeLZzVhVVVOQMc= github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 h1:pWxk9e//NbPwfxat7RXkts09K+dEBJWakUWwICVqYbA= github.com/matoous/godox v0.0.0-20210227103229-6504466cf951/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= +github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26 h1:gWg6ZQ4JhDfJPqlo2srm/LN17lpybq15AryXIRcWYLE= +github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -537,6 +669,8 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -545,12 +679,16 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mbilski/exhaustivestruct v1.2.0 h1:wCBmUnSYufAHO6J4AVWY6ff+oxWxsVFrwgOdMUQePUo= @@ -558,6 +696,8 @@ github.com/mbilski/exhaustivestruct v1.2.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aks github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= github.com/mgechev/revive v1.2.1 h1:GjFml7ZsoR0IrQ2E2YIvWFNS5GPDV7xNwvA5GM1HZC4= github.com/mgechev/revive v1.2.1/go.mod h1:+Ro3wqY4vakcYNtkBWdZC7dBg1xSB6sp054wWwmeFm0= +github.com/mgechev/revive v1.3.2 h1:Wb8NQKBaALBJ3xrrj4zpwJwqwNA6nDpyJSEQWcCka6U= +github.com/mgechev/revive v1.3.2/go.mod h1:UCLtc7o5vg5aXCwdUTU1kEBQ1v+YXPAkYDIDXbrs5I0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -585,6 +725,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/moricho/tparallel v0.2.1 h1:95FytivzT6rYzdJLdtfn6m1bfFJylOJK41+lgv/EHf4= github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k= +github.com/moricho/tparallel v0.3.1 h1:fQKD4U1wRMAYNngDonW5XupoB/ZGJHdpzrWqgyg9krA= +github.com/moricho/tparallel v0.3.1/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI= github.com/mozilla/scribe v0.0.0-20180711195314-fb71baf557c1/go.mod h1:FIczTrinKo8VaLxe6PWTPEXRXDIHz2QAwiaBaP5/4a8= github.com/mozilla/tls-observatory v0.0.0-20210609171429-7bc42856d2e5/go.mod h1:FUqVoUPHSEdDR0MnFM3Dh8AU0pZHLXUD127SAJGER/s= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -598,9 +740,13 @@ github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/exhaustive v0.7.11 h1:xV/WU3Vdwh5BUH4N06JNUznb6d5zhRPOnlgCrpNYNKA= github.com/nishanths/exhaustive v0.7.11/go.mod h1:gX+MP7DWMKJmNa1HfMozK+u04hQd3na9i0hyqf3/dOI= +github.com/nishanths/exhaustive v0.11.0 h1:T3I8nUGhl/Cwu5Z2hfc92l0e04D2GEW6e0l8pzda2l0= +github.com/nishanths/exhaustive v0.11.0/go.mod h1:RqwDsZ1xY0dNdqHho2z6X+bgzizwbLYOWnZbbl2wLB4= github.com/nishanths/predeclared v0.0.0-20190419143655-18a43bb90ffc/go.mod h1:62PewwiQTlm/7Rj+cxVYqZvDIUc+JjZq6GHAC1fsObQ= github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= +github.com/nunnatsa/ginkgolinter v0.13.3 h1:wEvjrzSMfDdnoWkctignX9QTf4rT9f4GkQ3uVoXBmiU= +github.com/nunnatsa/ginkgolinter v0.13.3/go.mod h1:aTKXo8WddENYxNEFT+4ZxEgWXqlD9uMD3w9Bfw/ABEc= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -614,10 +760,32 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= +github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= +github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= +github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= +github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= +github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= +github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= @@ -632,6 +800,10 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.0-beta.8/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.0.0/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.0.2/go.mod h1:MovirKjgVRESsAvNZlAjtFwV867yGuwRkXbG66OzopI= +github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= +github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA= github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= @@ -647,6 +819,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polyfloyd/go-errorlint v1.0.0 h1:pDrQG0lrh68e602Wfp68BlUTRFoHn8PZYAjLgt2LFsM= github.com/polyfloyd/go-errorlint v1.0.0/go.mod h1:KZy4xxPJyy88/gldCe5OdW6OQRtNO3EZE7hXzmnebgA= +github.com/polyfloyd/go-errorlint v1.4.3 h1:P6NALOLV8BrWhm6PsqOraUK05E5h8IZnpXYJ+CIg+0U= +github.com/polyfloyd/go-errorlint v1.4.3/go.mod h1:VPlWPh6hB/wruVG803SuNpLuTGNjLHYlvcdSy4RhdPA= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= @@ -655,6 +829,7 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -680,15 +855,22 @@ github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3 github.com/quasilyte/go-ruleguard v0.3.1-0.20210203134552-1b5a410e1cc8/go.mod h1:KsAh3x0e7Fkpgs+Q9pNLS5XpFSvYCEVl5gP9Pp1xp30= github.com/quasilyte/go-ruleguard v0.3.16-0.20220213074421-6aa060fab41a h1:sWFavxtIctGrVs5SYZ5Ml1CvrDAs8Kf5kx2PI3C41dA= github.com/quasilyte/go-ruleguard v0.3.16-0.20220213074421-6aa060fab41a/go.mod h1:VMX+OnnSw4LicdiEGtRSD/1X8kW7GuEscjYNr4cOIT4= +github.com/quasilyte/go-ruleguard v0.4.0 h1:DyM6r+TKL+xbKB4Nm7Afd1IQh9kEUKQs2pboWGKtvQo= +github.com/quasilyte/go-ruleguard v0.4.0/go.mod h1:Eu76Z/R8IXtViWUIHkE3p8gdH3/PKk1eh3YGfaEof10= github.com/quasilyte/go-ruleguard/dsl v0.3.0/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= github.com/quasilyte/go-ruleguard/dsl v0.3.16/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= github.com/quasilyte/go-ruleguard/dsl v0.3.19/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= +github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= github.com/quasilyte/go-ruleguard/rules v0.0.0-20201231183845-9e62ed36efe1/go.mod h1:7JTjp89EGyU1d6XfBiXihJNG37wB2VRkd125Q1u7Plc= github.com/quasilyte/go-ruleguard/rules v0.0.0-20211022131956-028d6511ab71/go.mod h1:4cgAphtvu7Ftv7vOT2ZOYhC6CvBxZixcasr8qIOTA50= github.com/quasilyte/gogrep v0.0.0-20220120141003-628d8b3623b5 h1:PDWGei+Rf2bBiuZIbZmM20J2ftEy9IeUCHA8HbQqed8= github.com/quasilyte/gogrep v0.0.0-20220120141003-628d8b3623b5/go.mod h1:wSEyW6O61xRV6zb6My3HxrQ5/8ke7NE2OayqCHa3xRM= +github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= +github.com/quasilyte/gogrep v0.5.0/go.mod h1:Cm9lpz9NZjEoL1tgZ2OgeUKPIxL1meE7eo60Z6Sk+Ng= github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 h1:L8QM9bvf68pVdQ3bCFZMDmnt9yqcMBro1pC7F+IPYMY= github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= +github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl980XxGFEZSS6KlBGIV0diGdySzxATTWoqaU= +github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= github.com/remyoudompheng/go-dbus v0.0.0-20121104212943-b7232d34b1d5/go.mod h1:+u151txRmLpwxBmpYn9z3d1sdJdjRPQpsXuYeY9jNls= @@ -699,26 +881,44 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryancurrah/gomodguard v1.2.3 h1:ww2fsjqocGCAFamzvv/b8IsRduuHHeK2MHTcTxZTQX8= github.com/ryancurrah/gomodguard v1.2.3/go.mod h1:rYbA/4Tg5c54mV1sv4sQTP5WOPBcoLtnBZ7/TEhXAbg= +github.com/ryancurrah/gomodguard v1.3.0 h1:q15RT/pd6UggBXVBuLps8BXRvl5GPBcwVA7BJHMLuTw= +github.com/ryancurrah/gomodguard v1.3.0/go.mod h1:ggBxb3luypPEzqVtq33ee7YSN35V28XeGnid8dnni50= github.com/ryanrolds/sqlclosecheck v0.3.0 h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8OUZI9xFw= github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA= +github.com/ryanrolds/sqlclosecheck v0.4.0 h1:i8SX60Rppc1wRuyQjMciLqIzV3xnoHB7/tXbr6RGYNI= +github.com/ryanrolds/sqlclosecheck v0.4.0/go.mod h1:TBRRjzL31JONc9i4XMinicuo+s+E8yKZ5FN8X3G6CKQ= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sagikazarmark/crypt v0.5.0/go.mod h1:l+nzl7KWh51rpzp2h7t4MZWyiEWdhNpOAnclKvg+mdA= +github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= github.com/sanposhiho/wastedassign/v2 v2.0.6 h1:+6/hQIHKNJAUixEj6EmOngGIisyeI+T3335lYTyxRoA= github.com/sanposhiho/wastedassign/v2 v2.0.6/go.mod h1:KyZ0MWTwxxBmfwn33zh3k1dmsbF2ud9pAAGfoLfjhtI= +github.com/sanposhiho/wastedassign/v2 v2.0.7 h1:J+6nrY4VW+gC9xFzUc+XjPD3g3wF3je/NsJFwFK7Uxc= +github.com/sanposhiho/wastedassign/v2 v2.0.7/go.mod h1:KyZ0MWTwxxBmfwn33zh3k1dmsbF2ud9pAAGfoLfjhtI= +github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tMEOsumirXcOJqAw= +github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= +github.com/sashamelentyev/usestdlibvars v1.23.0 h1:01h+/2Kd+NblNItNeux0veSL5cBF1jbEOPrEhDzGYq0= +github.com/sashamelentyev/usestdlibvars v1.23.0/go.mod h1:YPwr/Y1LATzHI93CqoPUN/2BzGQ/6N/cl/KwgR0B/aU= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/securego/gosec/v2 v2.11.0 h1:+PDkpzR41OI2jrw1q6AdXZCbsNGNGT7pQjal0H0cArI= github.com/securego/gosec/v2 v2.11.0/go.mod h1:SX8bptShuG8reGC0XS09+a4H2BoWSJi+fscA+Pulbpo= +github.com/securego/gosec/v2 v2.16.0 h1:Pi0JKoasQQ3NnoRao/ww/N/XdynIB9NRYYZT5CyOs5U= +github.com/securego/gosec/v2 v2.16.0/go.mod h1:xvLcVZqUfo4aAQu56TNv7/Ltz6emAOQAEsrZrt7uGlI= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= github.com/shirou/gopsutil/v3 v3.22.4/go.mod h1:D01hZJ4pVHPpCTZ3m3T2+wDF2YAGfd+H4ifUguaQzHM= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -728,15 +928,27 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sivchari/containedctx v1.0.2 h1:0hLQKpgC53OVF1VT7CeoFHk9YKstur1XOgfYIc1yrHI= github.com/sivchari/containedctx v1.0.2/go.mod h1:PwZOeqm4/DLoJOqMSIJs3aKqXRX4YO+uXww087KZ7Bw= +github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+Wwfd0XE= +github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4= +github.com/sivchari/nosnakecase v1.7.0 h1:7QkpWIRMe8x25gckkFd2A5Pi6Ymo0qgr4JrhGt95do8= +github.com/sivchari/nosnakecase v1.7.0/go.mod h1:CwDzrzPea40/GB6uynrNLiorAlgFRvRbFSgJx2Gs+QY= github.com/sivchari/tenv v1.5.0 h1:wxW0mFpKI6DIb3s6m1jCDYvkWXCskrimXMuGd0K/kSQ= github.com/sivchari/tenv v1.5.0/go.mod h1:64yStXKSOxDfX47NlhVwND4dHwfZDdbp2Lyl018Icvg= +github.com/sivchari/tenv v1.7.1 h1:PSpuD4bu6fSmtWMxSGWcvqUUgIn7k3yOJhOIzVWn8Ak= +github.com/sivchari/tenv v1.7.1/go.mod h1:64yStXKSOxDfX47NlhVwND4dHwfZDdbp2Lyl018Icvg= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sonatard/noctx v0.0.1 h1:VC1Qhl6Oxx9vvWo3UDgrGXYCeKCe3Wbw7qAWL6FrmTY= github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI= +github.com/sonatard/noctx v0.0.2 h1:L7Dz4De2zDQhW8S0t+KUjY0MAQJd6SgVwhzNIc4ok00= +github.com/sonatard/noctx v0.0.2/go.mod h1:kzFz+CzWSjQ2OzIm46uJZoXuBpa2+0y3T36U18dWqIo= github.com/sourcegraph/go-diff v0.6.1 h1:hmA1LzxW0n1c3Q4YbrFgg4P99GSnebYa3x8gr0HZqLQ= github.com/sourcegraph/go-diff v0.6.1/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= +github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0= +github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= @@ -746,11 +958,16 @@ github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfA github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -762,6 +979,8 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/spf13/viper v1.11.0 h1:7OX/1FS6n7jHD1zGrZTM7WtY13ZELRyosK4k93oPr44= github.com/spf13/viper v1.11.0/go.mod h1:djo0X/bA5+tYVoCn+C7cAYJGcVn/qYLFTG8gdUsX7Zk= +github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= +github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stbenjam/no-sprintf-host-port v0.1.1 h1:tYugd/yrm1O0dV+ThCbaKZh195Dfm07ysF0U6JQXczc= @@ -769,6 +988,9 @@ github.com/stbenjam/no-sprintf-host-port v0.1.1/go.mod h1:TLhvtIvONRzdmkFiio4O8L github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -779,43 +1001,73 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= +github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/sylvia7788/contextcheck v1.0.4 h1:MsiVqROAdr0efZc/fOCt0c235qm9XJqHtWwM+2h2B04= github.com/sylvia7788/contextcheck v1.0.4/go.mod h1:vuPKJMQ7MQ91ZTqfdyreNKwZjyUg6KO+IebVyQDedZQ= +github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c/go.mod h1:SbErYREK7xXdsRiigaQiQkI9McGRzYMvlKYaP3Nimdk= github.com/tdakkota/asciicheck v0.1.1 h1:PKzG7JUTUmVspQTDqtkX9eSiLGossXTybutHwTXuO0A= github.com/tdakkota/asciicheck v0.1.1/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= +github.com/tdakkota/asciicheck v0.2.0 h1:o8jvnUANo0qXtnslk2d3nMKTFNlOnJjRrNcj0j9qkHM= +github.com/tdakkota/asciicheck v0.2.0/go.mod h1:Qb7Y9EgjCLJGup51gDHFzbI08/gbGhL/UVhYIPWG2rg= github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= github.com/tetafro/godot v1.4.11 h1:BVoBIqAf/2QdbFmSwAWnaIqDivZdOV0ZRwEm6jivLKw= github.com/tetafro/godot v1.4.11/go.mod h1:LR3CJpxDVGlYOWn3ZZg1PgNZdTUvzsZWu8xaEohUpn8= github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 h1:kl4KhGNsJIbDHS9/4U9yQo1UcPQM0kOMJHn29EoH/Ro= github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 h1:quvGphlmUVU+nhpFa4gg4yJyTRJ13reZMDHrKwYw53M= +github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966/go.mod h1:27bSVNWSBOHm+qRp1T9qzaIpsWEP6TbUnei/43HK+PQ= +github.com/timonwong/loggercheck v0.9.4 h1:HKKhqrjcVj8sxL7K77beXh0adEm6DLjV/QOGeMXEVi4= +github.com/timonwong/loggercheck v0.9.4/go.mod h1:caz4zlPcgvpEkXgVnAJGowHAMW2NwHaNlpS8xDbVhTg= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tomarrell/wrapcheck/v2 v2.6.1 h1:Cf4a/iwuMp9s7kKrh74GTgijRVim0wEpKjgAsT7Wctw= github.com/tomarrell/wrapcheck/v2 v2.6.1/go.mod h1:Eo+Opt6pyMW1b6cNllOcDSSoHO0aTJ+iF6BfCUbHltA= +github.com/tomarrell/wrapcheck/v2 v2.8.1 h1:HxSqDSN0sAt0yJYsrcYVoEeyM4aI9yAm3KQpIXDJRhQ= +github.com/tomarrell/wrapcheck/v2 v2.8.1/go.mod h1:/n2Q3NZ4XFT50ho6Hbxg+RV1uyo2Uow/Vdm9NQcl5SE= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= github.com/tommy-muehle/go-mnd/v2 v2.5.0 h1:iAj0a8e6+dXSL7Liq0aXPox36FiN1dBbjA6lt9fl65s= github.com/tommy-muehle/go-mnd/v2 v2.5.0/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= +github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw= +github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA= github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/ultraware/funlen v0.1.0 h1:BuqclbkY6pO+cvxoq7OsktIXZpgBSkYTQtmwhAK81vI= +github.com/ultraware/funlen v0.1.0/go.mod h1:XJqmOQja6DpxarLj6Jj1U7JuoS8PvL4nEqDaQhy22p4= github.com/ultraware/whitespace v0.0.5 h1:hh+/cpIcopyMYbZNVov9iSxvJU3OYQg78Sfaqzi/CzI= github.com/ultraware/whitespace v0.0.5/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/uudashr/gocognit v1.0.5 h1:rrSex7oHr3/pPLQ0xoWq108XMU8s678FJcQ+aSfOHa4= github.com/uudashr/gocognit v1.0.5/go.mod h1:wgYz0mitoKOTysqxTDMOUXg+Jb5SvtihkfmugIZYpEA= +github.com/uudashr/gocognit v1.0.7 h1:e9aFXgKgUJrQ5+bs61zBigmj7bFJ/5cC6HmMahVzuDo= +github.com/uudashr/gocognit v1.0.7/go.mod h1:nAIUuVBnYU7pcninia3BHOvQkpQCeO76Uscky5BOwcY= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= github.com/valyala/quicktemplate v1.7.0/go.mod h1:sqKJnoaOF88V07vkO+9FL8fb9uZg/VPSJnLYn+LmLk8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= +github.com/xen0n/gosmopolitan v1.2.1 h1:3pttnTuFumELBRSh+KQs1zcz4fN6Zy7aB0xlnQSn1Iw= +github.com/xen0n/gosmopolitan v1.2.1/go.mod h1:JsHq/Brs1o050OOdmzHeOr0N7OtlnKRAGAsElF8xBQA= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -823,6 +1075,8 @@ github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= github.com/yeya24/promlinter v0.2.0 h1:xFKDQ82orCU5jQujdaD8stOHiv8UN68BSdn2a8u8Y3o= github.com/yeya24/promlinter v0.2.0/go.mod h1:u54lkmBOZrpEbQQ6gox2zWKKLKu2SGe+2KOiextY+IA= +github.com/ykadowak/zerologlint v0.1.3 h1:TLy1dTW3Nuc+YE3bYRPToG1Q9Ej78b5UUN6bjbGdxPE= +github.com/ykadowak/zerologlint v0.1.3/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= @@ -833,18 +1087,27 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= gitlab.com/bosi/decorder v0.2.1 h1:ehqZe8hI4w7O4b1vgsDZw1YU1PE7iJXrQWFMsocbQ1w= gitlab.com/bosi/decorder v0.2.1/go.mod h1:6C/nhLSbF6qZbYD8bRmISBwc6vcWdNsiIBkRvjJFrH0= +gitlab.com/bosi/decorder v0.4.0 h1:HWuxAhSxIvsITcXeP+iIRg9d1cVfvVkmlF7M68GaoDY= +gitlab.com/bosi/decorder v0.4.0/go.mod h1:xarnteyUoJiOTEldDysquWKTVDCKo2TOIOIibSuWqOg= +go-simpler.org/assert v0.5.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20200513171258-e048e166ab9c/go.mod h1:xCI7ZzBfRuGgBXyXO6yfWfDmlWd35khcWpUa4L0xI/k= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.etcd.io/etcd/client/v2 v2.305.2/go.mod h1:2D7ZejHVMIfog1221iLSYlQRzrtECw3kz4I4VAQm3qI= +go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU= +go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= go.mozilla.org/mozlog v0.0.0-20170222151521-4bb13139d403/go.mod h1:jHoPAGnDrCy6kaI2tAze5Prf0Nr0w/oNkROt2lw3n3o= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -854,18 +1117,25 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.tmz.dev/musttag v0.7.1 h1:9lFmeSFnFfPuMq4IksHGomItE6NgKMNW2Nt2FPOhCfU= +go.tmz.dev/musttag v0.7.1/go.mod h1:oJLkpR56EsIryktZJk/B0IroSMi37YWver47fibGh5U= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -885,6 +1155,11 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220313003712-b769efc7c000/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -897,8 +1172,15 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5 h1:FR+oGxGfbQu1d+jglI3rCkjAjUnhRSZcUxr+DqlDLNo= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e h1:qyrTQ++p1afMkO4DPEeLGq/3oTsdlvdH4vqZUBWzUKM= golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20230307190834-24139beb5833 h1:jWGQJV4niP+CCmFW9ekjA9Zx8vYORzOUH2/Nl5WPuLQ= +golang.org/x/exp/typeparams v0.0.0-20230307190834-24139beb5833/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -928,6 +1210,15 @@ golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -986,6 +1277,21 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1019,6 +1325,12 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1119,9 +1431,40 @@ golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1132,6 +1475,17 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1238,12 +1592,31 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.11-0.20220316014157-77aa08bb151a h1:ofrrl6c6NG5/IOSx/R1cyiQxxjqlur0h/TvbUhkH0II= golang.org/x/tools v0.1.11-0.20220316014157-77aa08bb151a/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.4.1-0.20221208213631-3f74d914ae6d/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.0/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.9.2/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1282,6 +1655,9 @@ google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQ google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.81.0/go.mod h1:FA6Mb/bZxj706H2j+j2d6mHEEaHBmbbWnkfvmorOCko= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1370,6 +1746,12 @@ google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1403,6 +1785,8 @@ google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1432,6 +1816,9 @@ gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -1449,6 +1836,9 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1458,15 +1848,22 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.3.1 h1:1kJlrWJLkaGXgcaeosRXViwviqjI7nkBvU2+sZW0AYc= honnef.co/go/tools v0.3.1/go.mod h1:vlRD9XErLMGT+mDuofSr0mMMquscM/1nQqtRSsh6m70= +honnef.co/go/tools v0.4.3 h1:o/n5/K5gXqk8Gozvs2cnL0F2S1/g1vcGCAx2vETjITw= +honnef.co/go/tools v0.4.3/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA= mvdan.cc/gofumpt v0.3.1 h1:avhhrOmv0IuvQVK7fvwV91oFSGAk5/6Po8GXTzICeu8= mvdan.cc/gofumpt v0.3.1/go.mod h1:w3ymliuxvzVx8DAutBnVyDqYb1Niy/yCJt/lk821YCE= +mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E= +mvdan.cc/gofumpt v0.5.0/go.mod h1:HBeVDtMKRZpXyxFciAirzdKklDlGu8aAy1wEbH5Y9js= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5 h1:Jh3LAeMt1eGpxomyu3jVkmVZWW2MxZ1qIIV2TZ/nRio= mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5/go.mod h1:b8RRCBm0eeiWR8cfN88xeq2G5SG3VKGO+5UPWi5FSOY= +mvdan.cc/unparam v0.0.0-20221223090309-7455f1af531d h1:3rvTIIM22r9pvXk+q3swxUQAQOxksVMGK7sml4nG57w= +mvdan.cc/unparam v0.0.0-20221223090309-7455f1af531d/go.mod h1:IeHQjmn6TOD+e4Z3RFiZMMsLVL+A96Nvptar8Fj71is= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/.bingo/mdox.mod b/.bingo/mdox.mod index f5c5f148..0464270e 100644 --- a/.bingo/mdox.mod +++ b/.bingo/mdox.mod @@ -1,5 +1,5 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT -go 1.18 +go 1.20 require github.com/bwplotka/mdox v0.9.0 diff --git a/.bingo/variables.env b/.bingo/variables.env index 749af6c8..e3ebc904 100644 --- a/.bingo/variables.env +++ b/.bingo/variables.env @@ -16,7 +16,7 @@ FAILLINT="${GOBIN}/faillint-v1.11.0" GOIMPORTS="${GOBIN}/goimports-v0.0.0-20200526224456-8b020aee10d2" -GOLANGCI_LINT="${GOBIN}/golangci-lint-v1.46.2" +GOLANGCI_LINT="${GOBIN}/golangci-lint-v1.54.1" MDOX="${GOBIN}/mdox-v0.9.0" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d2aacfb3..46610347 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -40,7 +40,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.18 + go-version: 1.20.x # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 8560c1a7..6be30e4d 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -20,7 +20,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.18.x + go-version: 1.20.x - uses: actions/cache@v1 with: diff --git a/.go-version b/.go-version index adc97d8e..d8c94e3e 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.18 +1.20.x diff --git a/Makefile b/Makefile index 09fd0d3e..ed5aaf4b 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ check-docs: $(MDOX) .PHONY: deps deps: ## Ensures fresh go.mod and go.sum. - @go mod tidy -compat=1.18 + @go mod tidy -compat=1.20 @go mod verify .PHONY: format diff --git a/go.mod b/go.mod index 982eb44e..b7b8cffc 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/thanos-io/objstore -go 1.18 +go 1.20 require ( cloud.google.com/go/storage v1.35.1 From f01e268612c24e6f24db14ee6d836ceb28c937b2 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 15 Nov 2023 09:24:30 +0100 Subject: [PATCH 058/130] Fix lint Signed-off-by: Filip Petkovski --- README.md | 4 ++-- client/factory_test.go | 8 ++++---- providers/obs/obs.go | 2 +- tracing/opentracing/opentracing.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 2a155271..8037f550 100644 --- a/README.md +++ b/README.md @@ -88,8 +88,7 @@ type BucketReader interface { // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. IsObjNotFoundErr(err error) bool - // IsAccessDeniedErr returns true if access to object is denied. - IsAccessDeniedErr(err error) bool + // IsAccessDeniedErr returns true if acces to object is denied. ``` Those interfaces represent the object storage operations your code can use from `objstore` clients. @@ -176,6 +175,7 @@ config: enable: false list_objects_version: "" bucket_lookup_type: auto + send_content_md5: true part_size: 67108864 sse_config: type: "" diff --git a/client/factory_test.go b/client/factory_test.go index ae1e980e..dc9e46f0 100644 --- a/client/factory_test.go +++ b/client/factory_test.go @@ -6,7 +6,7 @@ package client import ( "context" "fmt" - "io/ioutil" + "os" "github.com/go-kit/log" "go.opentelemetry.io/otel/trace" @@ -17,7 +17,7 @@ import ( func ExampleBucket() { // Read the configuration file. - confContentYaml, err := ioutil.ReadFile("testconf/filesystem.conf.yml") + confContentYaml, err := os.ReadFile("testconf/filesystem.conf.yml") if err != nil { panic(err) } @@ -40,7 +40,7 @@ func ExampleBucket() { func ExampleTracingBucketUsingOpenTracing() { //nolint:govet // Read the configuration file. - confContentYaml, err := ioutil.ReadFile("testconf/filesystem.conf.yml") + confContentYaml, err := os.ReadFile("testconf/filesystem.conf.yml") if err != nil { panic(err) } @@ -66,7 +66,7 @@ func ExampleTracingBucketUsingOpenTracing() { //nolint:govet func ExampleTracingBucketUsingOpenTelemetry() { //nolint:govet // Read the configuration file. - confContentYaml, err := ioutil.ReadFile("testconf/filesystem.conf.yml") + confContentYaml, err := os.ReadFile("testconf/filesystem.conf.yml") if err != nil { panic(err) } diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 35a04c80..7bd9666b 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -280,7 +280,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return b.getRange(ctx, name, off, length) } -func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { +func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io.ReadCloser, error) { if strings.TrimSpace(name) == "" { return nil, errors.New("object name cannot be empty") } diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index 8b99e304..0a26ceeb 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -209,8 +209,8 @@ func startSpan(ctx context.Context, operationName string, opts ...opentracing.St // doWithSpan executes function doFn inside new span with `operationName` name and hooking as child to a span found within given context if any. // It uses opentracing.Tracer propagated in context. If no found, it uses noop tracer notification. -func doWithSpan(ctx context.Context, operationName string, doFn func(context.Context, Span), opts ...opentracing.StartSpanOption) { - span, newCtx := startSpan(ctx, operationName, opts...) +func doWithSpan(ctx context.Context, operationName string, doFn func(context.Context, Span), _ ...opentracing.StartSpanOption) { + span, newCtx := startSpan(ctx, operationName) defer span.Finish() doFn(newCtx, span) } From 844a043c6e82b025e4c26bdefd3f06a6b0032634 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 15 Nov 2023 12:14:33 +0100 Subject: [PATCH 059/130] Fix typo Signed-off-by: Filip Petkovski --- README.md | 2 +- objstore.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8037f550..3d8c63ee 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ type BucketReader interface { // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. IsObjNotFoundErr(err error) bool - // IsAccessDeniedErr returns true if acces to object is denied. + // IsAccessDeniedErr returns true if access to object is denied. ``` Those interfaces represent the object storage operations your code can use from `objstore` clients. diff --git a/objstore.go b/objstore.go index 219a9572..8f71011a 100644 --- a/objstore.go +++ b/objstore.go @@ -85,7 +85,7 @@ type BucketReader interface { // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. IsObjNotFoundErr(err error) bool - // IsAccessDeniedErr returns true if acces to object is denied. + // IsAccessDeniedErr returns true if access to object is denied. IsAccessDeniedErr(err error) bool // Attributes returns information about the specified object. From 10c8b523b814d4ccfd2ab68d8e4f644defee046b Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 22 Nov 2023 09:24:41 +0100 Subject: [PATCH 060/130] Extend mdox range Signed-off-by: Filip Petkovski --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3d8c63ee..e473ecff 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ type Bucket interface { All [provider implementations](providers) have to implement `Bucket` interface that allows common read and write operations that all supported by all object providers. If you want to limit the code that will do bucket operation to only read access (smart idea, allowing to limit access permissions), you can use the [`BucketReader` interface](objstore.go): -```go mdox-exec="sed -n '68,88p' objstore.go" +```go mdox-exec="sed -n '68,93p' objstore.go" // BucketReader provides read access to an object storage bucket. type BucketReader interface { @@ -89,6 +89,11 @@ type BucketReader interface { IsObjNotFoundErr(err error) bool // IsAccessDeniedErr returns true if access to object is denied. + IsAccessDeniedErr(err error) bool + + // Attributes returns information about the specified object. + Attributes(ctx context.Context, name string) (ObjectAttributes, error) +} ``` Those interfaces represent the object storage operations your code can use from `objstore` clients. From 008a3c92d11799d6a3a68cf4dd6095e4c9d4e891 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 6 Dec 2023 09:01:58 +0100 Subject: [PATCH 061/130] Enable using gRPC client for GCS GCP has an experimental feature which allows using gRPC when talking to GCS. For the time being, it requires buckets to be explicitly added to an allowlist, but eventually the feature will become GA. This commit allows using a gRPC client when creating a GCS bucket by toggling a boolean flag in the bucket config. Signed-off-by: Filip Petkovski --- providers/gcs/gcs.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index ad305d6e..0c9b3bb1 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -33,6 +33,7 @@ const DirDelim = "/" type Config struct { Bucket string `yaml:"bucket"` ServiceAccount string `yaml:"service_account"` + UseGRPC bool `yaml:"use_grpc"` } // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. @@ -75,7 +76,19 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())), ) - gcsClient, err := storage.NewClient(ctx, opts...) + return newBucket(ctx, logger, gc, opts) +} + +func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option.ClientOption) (*Bucket, error) { + var ( + err error + gcsClient *storage.Client + ) + if gc.UseGRPC { + gcsClient, err = storage.NewGRPCClient(ctx, opts...) + } else { + gcsClient, err = storage.NewClient(ctx, opts...) + } if err != nil { return nil, err } From c035baa67b5cb18e22502763a4ba0caa38b38e3e Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 6 Dec 2023 09:06:23 +0100 Subject: [PATCH 062/130] Add CHANGELOG Signed-off-by: Filip Petkovski --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfb2c11b..9f2e035e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. - [#76](https://github.com/thanos-io/objstore/pull/76) GCS: Query for object names only in `Iter` to possibly improve performance when listing objects. - [#85](https://github.com/thanos-io/objstore/pull/85) S3: Allow checksum algorithm to be configured +- [#92](https://github.com/thanos-io/objstore/pull/92) GCS: Allow using a gRPC client. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From 28a0b946789dcc2146950dee569fc10598923953 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 6 Dec 2023 09:43:09 +0100 Subject: [PATCH 063/130] Expose grpc connection pool size Signed-off-by: Filip Petkovski --- providers/gcs/gcs.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 0c9b3bb1..0ff6243b 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -19,6 +19,7 @@ import ( "golang.org/x/oauth2/google" "google.golang.org/api/iterator" "google.golang.org/api/option" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "gopkg.in/yaml.v2" @@ -31,9 +32,10 @@ const DirDelim = "/" // Config stores the configuration for gcs bucket. type Config struct { - Bucket string `yaml:"bucket"` - ServiceAccount string `yaml:"service_account"` - UseGRPC bool `yaml:"use_grpc"` + Bucket string `yaml:"bucket"` + ServiceAccount string `yaml:"service_account"` + UseGRPC bool `yaml:"use_grpc"` + GRPCConnPoolSize int `yaml:"grpc_conn_pool_size"` } // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. @@ -85,6 +87,10 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option. gcsClient *storage.Client ) if gc.UseGRPC { + opts = append(opts, + option.WithGRPCDialOption(grpc.WithRecvBufferPool(grpc.NewSharedBufferPool())), + option.WithGRPCConnectionPool(gc.GRPCConnPoolSize), + ) gcsClient, err = storage.NewGRPCClient(ctx, opts...) } else { gcsClient, err = storage.NewClient(ctx, opts...) From a70cbf63c797d0dce1521f2522138b63867f8b94 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Mon, 18 Dec 2023 07:55:41 +0100 Subject: [PATCH 064/130] Add docs for GRPCConnPoolSize Signed-off-by: Filip Petkovski --- README.md | 2 ++ providers/gcs/gcs.go | 12 ++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e473ecff..a42ab726 100644 --- a/README.md +++ b/README.md @@ -352,6 +352,8 @@ type: GCS config: bucket: "" service_account: "" + use_grpc: false + grpc_conn_pool_size: 0 prefix: "" ``` diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 0ff6243b..1d717b77 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -32,10 +32,14 @@ const DirDelim = "/" // Config stores the configuration for gcs bucket. type Config struct { - Bucket string `yaml:"bucket"` - ServiceAccount string `yaml:"service_account"` - UseGRPC bool `yaml:"use_grpc"` - GRPCConnPoolSize int `yaml:"grpc_conn_pool_size"` + Bucket string `yaml:"bucket"` + ServiceAccount string `yaml:"service_account"` + UseGRPC bool `yaml:"use_grpc"` + // GRPCConnPoolSize controls the size of the gRPC connection pool and should only be used + // when direct path is not enabled. + // See https://pkg.go.dev/cloud.google.com/go/storage#hdr-Experimental_gRPC_API for more details + // on how to enable direct path. + GRPCConnPoolSize int `yaml:"grpc_conn_pool_size"` } // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. From e59dd9c3f50cebb3f368cd4cb0461f752e0204e6 Mon Sep 17 00:00:00 2001 From: Daniel Deluiggi Date: Fri, 22 Dec 2023 16:11:42 -0800 Subject: [PATCH 065/130] Allow timingReadCloser to be seeker Signed-off-by: Daniel Deluiggi --- CHANGELOG.md | 1 + objstore.go | 22 ++++++++++++++++++++-- objstore_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f2e035e..6d6aa563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#76](https://github.com/thanos-io/objstore/pull/76) GCS: Query for object names only in `Iter` to possibly improve performance when listing objects. - [#85](https://github.com/thanos-io/objstore/pull/85) S3: Allow checksum algorithm to be configured - [#92](https://github.com/thanos-io/objstore/pull/92) GCS: Allow using a gRPC client. +- [#94](https://github.com/thanos-io/objstore/pull/94) Allow timingReadCloser to be seeker ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/objstore.go b/objstore.go index 8f71011a..df43035f 100644 --- a/objstore.go +++ b/objstore.go @@ -649,6 +649,10 @@ func (b *metricBucket) Name() string { return b.bkt.Name() } +type timingReadSeekCloser struct { + timingReadCloser +} + type timingReadCloser struct { io.ReadCloser objSize int64 @@ -666,12 +670,13 @@ type timingReadCloser struct { transferredBytes *prometheus.HistogramVec } -func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { +func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) objSize, objSizeErr := TryToGetSize(rc) - return &timingReadCloser{ + + trc := timingReadCloser{ ReadCloser: rc, objSize: objSize, objSizeErr: objSizeErr, @@ -684,6 +689,15 @@ func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramV transferredBytes: transferredBytes, readBytes: 0, } + + _, ok := rc.(io.Seeker) + if ok { + return &timingReadSeekCloser{ + timingReadCloser: trc, + } + } + + return &trc } func (t *timingReadCloser) ObjectSize() (int64, error) { @@ -719,3 +733,7 @@ func (rc *timingReadCloser) Read(b []byte) (n int, err error) { } return n, err } + +func (rsc *timingReadSeekCloser) Seek(offset int64, whence int) (int64, error) { + return (rsc.ReadCloser).(io.Seeker).Seek(offset, whence) +} diff --git a/objstore_test.go b/objstore_test.go index 5feb089b..431d6af8 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -232,6 +232,35 @@ func TestTimingTracingReader(t *testing.T) { testutil.Equals(t, int64(11), size) } +func TestTimingTracingReaderSeeker(t *testing.T) { + m := WrapWithMetrics(NewInMemBucket(), nil, "") + r := bytes.NewReader([]byte("hello world")) + + tr := nopSeekerCloserWithSize(r).(io.ReadCloser) + tr = newTimingReadCloser(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { + return false + }, m.opsFetchedBytes, m.opsTransferredBytes) + + size, err := TryToGetSize(tr) + + testutil.Ok(t, err) + testutil.Equals(t, int64(11), size) + + smallBuf := make([]byte, 4) + n, err := io.ReadFull(tr, smallBuf) + testutil.Ok(t, err) + testutil.Equals(t, 4, n) + + // Verify that size is still the same, after reading 4 bytes. + size, err = TryToGetSize(tr) + + testutil.Ok(t, err) + testutil.Equals(t, int64(11), size) + + _, ok := tr.(io.Seeker) + testutil.Equals(t, true, ok) +} + func TestDownloadDir_CleanUp(t *testing.T) { b := unreliableBucket{ Bucket: NewInMemBucket(), @@ -264,3 +293,16 @@ func (b unreliableBucket) Get(ctx context.Context, name string) (io.ReadCloser, } return b.Bucket.Get(ctx, name) } + +type nopSeekerCloserWithObjectSize struct{ io.Reader } + +func (n nopSeekerCloserWithObjectSize) Seek(offset int64, whence int) (int64, error) { + return 0, nil +} + +func (nopSeekerCloserWithObjectSize) Close() error { return nil } +func (n nopSeekerCloserWithObjectSize) ObjectSize() (int64, error) { return TryToGetSize(n.Reader) } + +func nopSeekerCloserWithSize(r io.Reader) io.ReadSeekCloser { + return nopSeekerCloserWithObjectSize{r} +} From 84b5fa388de162cb8b8e9167bd9bac82dc3b7239 Mon Sep 17 00:00:00 2001 From: Daniel Deluiggi Date: Thu, 11 Jan 2024 12:35:01 -0800 Subject: [PATCH 066/130] Add seeker option for nopCloserWithObjectSize Signed-off-by: Daniel Deluiggi --- objstore.go | 23 ++++++++++++++++++++++- objstore_test.go | 36 +++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/objstore.go b/objstore.go index df43035f..a83dacdf 100644 --- a/objstore.go +++ b/objstore.go @@ -232,6 +232,19 @@ func NopCloserWithSize(r io.Reader) io.ReadCloser { return nopCloserWithObjectSize{r} } +type nopSeekerCloserWithObjectSize struct{ io.Reader } + +func (nopSeekerCloserWithObjectSize) Close() error { return nil } +func (n nopSeekerCloserWithObjectSize) ObjectSize() (int64, error) { return TryToGetSize(n.Reader) } + +func (n nopSeekerCloserWithObjectSize) Seek(offset int64, whence int) (int64, error) { + return n.Reader.(io.Seeker).Seek(offset, whence) +} + +func nopSeekerCloserWithSize(r io.Reader) io.ReadSeekCloser { + return nopSeekerCloserWithObjectSize{r} +} + // UploadDir uploads all files in srcdir to the bucket with into a top-level directory // named dstdir. It is a caller responsibility to clean partial upload in case of failure. func UploadDir(ctx context.Context, logger log.Logger, bkt Bucket, srcdir, dstdir string, options ...UploadOption) error { @@ -595,8 +608,16 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err const op = OpUpload b.ops.WithLabelValues(op).Inc() + _, ok := r.(io.Seeker) + var nopR io.ReadCloser + if ok { + nopR = nopSeekerCloserWithSize(r) + } else { + nopR = NopCloserWithSize(r) + } + trc := newTimingReadCloser( - NopCloserWithSize(r), + nopR, op, b.opsDuration, b.opsFailures, diff --git a/objstore_test.go b/objstore_test.go index 431d6af8..bb3fbf5b 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -232,6 +232,29 @@ func TestTimingTracingReader(t *testing.T) { testutil.Equals(t, int64(11), size) } +func TestUploadKeepsSeekerObj(t *testing.T) { + r := prometheus.NewRegistry() + m := seekerTestBucket{ + Bucket: WrapWithMetrics(NewInMemBucket(), r, ""), + } + + testutil.Ok(t, m.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1")))) +} + +// seekerBucket implements Bucket and checks if io.Reader is still seekable. +type seekerTestBucket struct { + Bucket +} + +func (b seekerTestBucket) Upload(ctx context.Context, name string, r io.Reader) error { + _, ok := r.(io.Seeker) + if !ok { + return errors.New("Reader was supposed to be seekable") + } + + return nil +} + func TestTimingTracingReaderSeeker(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) @@ -293,16 +316,3 @@ func (b unreliableBucket) Get(ctx context.Context, name string) (io.ReadCloser, } return b.Bucket.Get(ctx, name) } - -type nopSeekerCloserWithObjectSize struct{ io.Reader } - -func (n nopSeekerCloserWithObjectSize) Seek(offset int64, whence int) (int64, error) { - return 0, nil -} - -func (nopSeekerCloserWithObjectSize) Close() error { return nil } -func (n nopSeekerCloserWithObjectSize) ObjectSize() (int64, error) { return TryToGetSize(n.Reader) } - -func nopSeekerCloserWithSize(r io.Reader) io.ReadSeekCloser { - return nopSeekerCloserWithObjectSize{r} -} From ab3da4d0a526e4fce2dd5e29fae9c1ba89468526 Mon Sep 17 00:00:00 2001 From: Daniel Deluiggi Date: Thu, 11 Jan 2024 12:40:28 -0800 Subject: [PATCH 067/130] Add changelog Signed-off-by: Daniel Deluiggi --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d6aa563..90b29a88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#85](https://github.com/thanos-io/objstore/pull/85) S3: Allow checksum algorithm to be configured - [#92](https://github.com/thanos-io/objstore/pull/92) GCS: Allow using a gRPC client. - [#94](https://github.com/thanos-io/objstore/pull/94) Allow timingReadCloser to be seeker +- [#96](https://github.com/thanos-io/objstore/pull/96) Allow nopCloserWithObjectSize to be seeker ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From 476df5bf574956423d8b3c35cdd2db6e139de1dd Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 26 Jan 2024 14:30:23 +0100 Subject: [PATCH 068/130] Fix S3 upload performance regression Signed-off-by: Marco Pracucci --- objstore.go | 128 ++++++++++++++++++++++++----------------------- objstore_test.go | 116 +++++++++++++++++++++++++++++++++++------- 2 files changed, 165 insertions(+), 79 deletions(-) diff --git a/objstore.go b/objstore.go index a83dacdf..548c91e3 100644 --- a/objstore.go +++ b/objstore.go @@ -232,19 +232,6 @@ func NopCloserWithSize(r io.Reader) io.ReadCloser { return nopCloserWithObjectSize{r} } -type nopSeekerCloserWithObjectSize struct{ io.Reader } - -func (nopSeekerCloserWithObjectSize) Close() error { return nil } -func (n nopSeekerCloserWithObjectSize) ObjectSize() (int64, error) { return TryToGetSize(n.Reader) } - -func (n nopSeekerCloserWithObjectSize) Seek(offset int64, whence int) (int64, error) { - return n.Reader.(io.Seeker).Seek(offset, whence) -} - -func nopSeekerCloserWithSize(r io.Reader) io.ReadSeekCloser { - return nopSeekerCloserWithObjectSize{r} -} - // UploadDir uploads all files in srcdir to the bucket with into a top-level directory // named dstdir. It is a caller responsibility to clean partial upload in case of failure. func UploadDir(ctx context.Context, logger log.Logger, bkt Bucket, srcdir, dstdir string, options ...UploadOption) error { @@ -555,7 +542,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err } return nil, err } - return newTimingReadCloser( + return newTimingReader( rc, op, b.opsDuration, @@ -577,7 +564,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in } return nil, err } - return newTimingReadCloser( + return newTimingReader( rc, op, b.opsDuration, @@ -608,16 +595,8 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err const op = OpUpload b.ops.WithLabelValues(op).Inc() - _, ok := r.(io.Seeker) - var nopR io.ReadCloser - if ok { - nopR = nopSeekerCloserWithSize(r) - } else { - nopR = NopCloserWithSize(r) - } - - trc := newTimingReadCloser( - nopR, + trc := newTimingReader( + r, op, b.opsDuration, b.opsFailures, @@ -670,12 +649,8 @@ func (b *metricBucket) Name() string { return b.bkt.Name() } -type timingReadSeekCloser struct { - timingReadCloser -} - -type timingReadCloser struct { - io.ReadCloser +type timingReader struct { + io.Reader objSize int64 objSizeErr error @@ -691,14 +666,14 @@ type timingReadCloser struct { transferredBytes *prometheus.HistogramVec } -func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { +func newTimingReader(r io.Reader, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) - objSize, objSizeErr := TryToGetSize(rc) + objSize, objSizeErr := TryToGetSize(r) - trc := timingReadCloser{ - ReadCloser: rc, + trc := timingReader{ + Reader: r, objSize: objSize, objSizeErr: objSizeErr, start: time.Now(), @@ -711,50 +686,79 @@ func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramV readBytes: 0, } - _, ok := rc.(io.Seeker) - if ok { - return &timingReadSeekCloser{ - timingReadCloser: trc, - } + _, isSeeker := r.(io.Seeker) + _, isReaderAt := r.(io.ReaderAt) + + if isSeeker && isReaderAt { + // The assumption is that in most cases when io.ReaderAt() is implemented then + // io.Seeker is implemented too (e.g. os.File). + return &timingReaderSeekerReaderAt{timingReaderSeeker: timingReaderSeeker{timingReader: trc}} + } + if isSeeker { + return &timingReaderSeeker{timingReader: trc} } return &trc } -func (t *timingReadCloser) ObjectSize() (int64, error) { - return t.objSize, t.objSizeErr +func (r *timingReader) ObjectSize() (int64, error) { + return r.objSize, r.objSizeErr } -func (rc *timingReadCloser) Close() error { - err := rc.ReadCloser.Close() - if !rc.alreadyGotErr && err != nil { - rc.failed.WithLabelValues(rc.op).Inc() +func (r *timingReader) Close() error { + var closeErr error + + // Call the wrapped reader if it implements Close(). + if closer, ok := r.Reader.(io.Closer); ok { + closeErr = closer.Close() + + if !r.alreadyGotErr && closeErr != nil { + r.failed.WithLabelValues(r.op).Inc() + r.alreadyGotErr = true + } } - if !rc.alreadyGotErr && err == nil { - rc.duration.WithLabelValues(rc.op).Observe(time.Since(rc.start).Seconds()) - rc.transferredBytes.WithLabelValues(rc.op).Observe(float64(rc.readBytes)) - rc.alreadyGotErr = true + + // Track duration and transferred bytes only if no error occurred. + if !r.alreadyGotErr { + r.duration.WithLabelValues(r.op).Observe(time.Since(r.start).Seconds()) + r.transferredBytes.WithLabelValues(r.op).Observe(float64(r.readBytes)) + + // Trick to tracking metrics multiple times in case Close() gets called again. + r.alreadyGotErr = true } - return err + + return closeErr } -func (rc *timingReadCloser) Read(b []byte) (n int, err error) { - n, err = rc.ReadCloser.Read(b) - if rc.fetchedBytes != nil { - rc.fetchedBytes.WithLabelValues(rc.op).Add(float64(n)) +func (r *timingReader) Read(b []byte) (n int, err error) { + n, err = r.Reader.Read(b) + if r.fetchedBytes != nil { + r.fetchedBytes.WithLabelValues(r.op).Add(float64(n)) } - rc.readBytes += int64(n) + r.readBytes += int64(n) // Report metric just once. - if !rc.alreadyGotErr && err != nil && err != io.EOF { - if !rc.isFailureExpected(err) { - rc.failed.WithLabelValues(rc.op).Inc() + if !r.alreadyGotErr && err != nil && err != io.EOF { + if !r.isFailureExpected(err) { + r.failed.WithLabelValues(r.op).Inc() } - rc.alreadyGotErr = true + r.alreadyGotErr = true } return n, err } -func (rsc *timingReadSeekCloser) Seek(offset int64, whence int) (int64, error) { - return (rsc.ReadCloser).(io.Seeker).Seek(offset, whence) +type timingReaderSeeker struct { + timingReader +} + +func (rsc *timingReaderSeeker) Seek(offset int64, whence int) (int64, error) { + return (rsc.Reader).(io.Seeker).Seek(offset, whence) +} + +type timingReaderSeekerReaderAt struct { + timingReaderSeeker +} + +func (rsc *timingReaderSeekerReaderAt) ReadAt(p []byte, off int64) (int, error) { + return (rsc.Reader).(io.ReaderAt).ReadAt(p, off) } diff --git a/objstore_test.go b/objstore_test.go index bb3fbf5b..8ca4e352 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -8,6 +8,7 @@ import ( "context" "io" "os" + "path/filepath" "strings" "testing" @@ -206,12 +207,12 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { `), `objstore_bucket_operations_total`)) } -func TestTimingTracingReader(t *testing.T) { +func TestTimingReader(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) tr := NopCloserWithSize(r) - tr = newTimingReadCloser(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { + tr = newTimingReader(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) @@ -232,35 +233,103 @@ func TestTimingTracingReader(t *testing.T) { testutil.Equals(t, int64(11), size) } -func TestUploadKeepsSeekerObj(t *testing.T) { - r := prometheus.NewRegistry() - m := seekerTestBucket{ - Bucket: WrapWithMetrics(NewInMemBucket(), r, ""), +func TestTimingReader_ShouldCorrectlyWrapFile(t *testing.T) { + // Create a test file. + testFilepath := filepath.Join(t.TempDir(), "test") + testutil.Ok(t, os.WriteFile(testFilepath, []byte("test"), os.ModePerm)) + + // Open the file (it will be used as io.Reader). + file, err := os.Open(testFilepath) + testutil.Ok(t, err) + t.Cleanup(func() { + testutil.Ok(t, file.Close()) + }) + + m := WrapWithMetrics(NewInMemBucket(), nil, "") + r := newTimingReader(file, "", m.opsDuration, m.opsFailures, func(err error) bool { + return false + }, m.opsFetchedBytes, m.opsTransferredBytes) + + // It must both implement io.Seeker and io.ReaderAt. + _, isSeeker := r.(io.Seeker) + testutil.Assert(t, isSeeker) + + _, isReaderAt := r.(io.ReaderAt) + testutil.Assert(t, isReaderAt) +} + +func TestWrapWithMetrics_UploadShouldPreserveReaderFeatures(t *testing.T) { + tests := map[string]struct { + reader io.Reader + expectedIsSeeker bool + expectedIsReaderAt bool + }{ + "bytes.Reader": { + reader: bytes.NewReader([]byte("1")), + expectedIsSeeker: true, + expectedIsReaderAt: true, + }, + "bytes.Buffer": { + reader: bytes.NewBuffer([]byte("1")), + expectedIsSeeker: false, + expectedIsReaderAt: false, + }, + "os.File": { + reader: func() io.Reader { + // Create a test file. + testFilepath := filepath.Join(t.TempDir(), "test") + testutil.Ok(t, os.WriteFile(testFilepath, []byte("test"), os.ModePerm)) + + // Open the file (it will be used as io.Reader). + file, err := os.Open(testFilepath) + testutil.Ok(t, err) + t.Cleanup(func() { + testutil.Ok(t, file.Close()) + }) + + return file + }(), + expectedIsSeeker: true, + expectedIsReaderAt: true, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + m := &uploadTrackerTestBucket{ + Bucket: WrapWithMetrics(NewInMemBucket(), nil, ""), + } + + testutil.Ok(t, m.Upload(context.Background(), "dir/obj1", testData.reader)) + + _, isSeeker := m.uploadReader.(io.Seeker) + testutil.Equals(t, testData.expectedIsSeeker, isSeeker) + + _, isReaderAt := m.uploadReader.(io.ReaderAt) + testutil.Equals(t, testData.expectedIsReaderAt, isReaderAt) + }) } - testutil.Ok(t, m.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1")))) } -// seekerBucket implements Bucket and checks if io.Reader is still seekable. -type seekerTestBucket struct { +// seekerBucket implements Bucket and keeps a reference of the io.Reader passed to Upload(). +type uploadTrackerTestBucket struct { Bucket -} -func (b seekerTestBucket) Upload(ctx context.Context, name string, r io.Reader) error { - _, ok := r.(io.Seeker) - if !ok { - return errors.New("Reader was supposed to be seekable") - } + uploadReader io.Reader +} +func (b *uploadTrackerTestBucket) Upload(ctx context.Context, name string, r io.Reader) error { + b.uploadReader = r return nil } -func TestTimingTracingReaderSeeker(t *testing.T) { +func TestTimingReader_ShouldWrapSeeker(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) tr := nopSeekerCloserWithSize(r).(io.ReadCloser) - tr = newTimingReadCloser(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { + tr = newTimingReader(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) @@ -316,3 +385,16 @@ func (b unreliableBucket) Get(ctx context.Context, name string) (io.ReadCloser, } return b.Bucket.Get(ctx, name) } + +type nopSeekerCloserWithObjectSize struct{ io.Reader } + +func (nopSeekerCloserWithObjectSize) Close() error { return nil } +func (n nopSeekerCloserWithObjectSize) ObjectSize() (int64, error) { return TryToGetSize(n.Reader) } + +func (n nopSeekerCloserWithObjectSize) Seek(offset int64, whence int) (int64, error) { + return n.Reader.(io.Seeker).Seek(offset, whence) +} + +func nopSeekerCloserWithSize(r io.Reader) io.ReadSeekCloser { + return nopSeekerCloserWithObjectSize{r} +} From 00f0c4c7bace8a273fef1097fba32e5442f9a39b Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 26 Jan 2024 14:34:22 +0100 Subject: [PATCH 069/130] Simplified unit tests Signed-off-by: Marco Pracucci --- objstore_test.go | 53 ++++++++---------------------------------------- 1 file changed, 8 insertions(+), 45 deletions(-) diff --git a/objstore_test.go b/objstore_test.go index 8ca4e352..f971e04e 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -210,9 +210,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { func TestTimingReader(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) - - tr := NopCloserWithSize(r) - tr = newTimingReader(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { + tr := newTimingReader(r, "", m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) @@ -231,6 +229,13 @@ func TestTimingReader(t *testing.T) { testutil.Ok(t, err) testutil.Equals(t, int64(11), size) + + // Given the reader was bytes.Reader it should both implement io.Seeker and io.ReaderAt. + _, isSeeker := tr.(io.Seeker) + testutil.Assert(t, isSeeker) + + _, isReaderAt := tr.(io.ReaderAt) + testutil.Assert(t, isReaderAt) } func TestTimingReader_ShouldCorrectlyWrapFile(t *testing.T) { @@ -324,35 +329,6 @@ func (b *uploadTrackerTestBucket) Upload(ctx context.Context, name string, r io. return nil } -func TestTimingReader_ShouldWrapSeeker(t *testing.T) { - m := WrapWithMetrics(NewInMemBucket(), nil, "") - r := bytes.NewReader([]byte("hello world")) - - tr := nopSeekerCloserWithSize(r).(io.ReadCloser) - tr = newTimingReader(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { - return false - }, m.opsFetchedBytes, m.opsTransferredBytes) - - size, err := TryToGetSize(tr) - - testutil.Ok(t, err) - testutil.Equals(t, int64(11), size) - - smallBuf := make([]byte, 4) - n, err := io.ReadFull(tr, smallBuf) - testutil.Ok(t, err) - testutil.Equals(t, 4, n) - - // Verify that size is still the same, after reading 4 bytes. - size, err = TryToGetSize(tr) - - testutil.Ok(t, err) - testutil.Equals(t, int64(11), size) - - _, ok := tr.(io.Seeker) - testutil.Equals(t, true, ok) -} - func TestDownloadDir_CleanUp(t *testing.T) { b := unreliableBucket{ Bucket: NewInMemBucket(), @@ -385,16 +361,3 @@ func (b unreliableBucket) Get(ctx context.Context, name string) (io.ReadCloser, } return b.Bucket.Get(ctx, name) } - -type nopSeekerCloserWithObjectSize struct{ io.Reader } - -func (nopSeekerCloserWithObjectSize) Close() error { return nil } -func (n nopSeekerCloserWithObjectSize) ObjectSize() (int64, error) { return TryToGetSize(n.Reader) } - -func (n nopSeekerCloserWithObjectSize) Seek(offset int64, whence int) (int64, error) { - return n.Reader.(io.Seeker).Seek(offset, whence) -} - -func nopSeekerCloserWithSize(r io.Reader) io.ReadSeekCloser { - return nopSeekerCloserWithObjectSize{r} -} From d8fa7c8ef0bf3947b628fb601d437f8ef456c668 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 26 Jan 2024 15:00:05 +0100 Subject: [PATCH 070/130] Do not close the reader in Upload() Signed-off-by: Marco Pracucci --- objstore.go | 15 ++++++++++++--- objstore_test.go | 26 ++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/objstore.go b/objstore.go index 548c91e3..b1d7d09a 100644 --- a/objstore.go +++ b/objstore.go @@ -544,6 +544,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err } return newTimingReader( rc, + true, op, b.opsDuration, b.opsFailures, @@ -566,6 +567,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in } return newTimingReader( rc, + true, op, b.opsDuration, b.opsFailures, @@ -597,6 +599,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err trc := newTimingReader( r, + false, op, b.opsDuration, b.opsFailures, @@ -651,6 +654,11 @@ func (b *metricBucket) Name() string { type timingReader struct { io.Reader + + // closeReader holds whether the wrapper io.Reader should be closed when + // Close() is called on the timingReader. + closeReader bool + objSize int64 objSizeErr error @@ -666,7 +674,7 @@ type timingReader struct { transferredBytes *prometheus.HistogramVec } -func newTimingReader(r io.Reader, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { +func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -674,6 +682,7 @@ func newTimingReader(r io.Reader, op string, dur *prometheus.HistogramVec, faile trc := timingReader{ Reader: r, + closeReader: closeReader, objSize: objSize, objSizeErr: objSizeErr, start: time.Now(), @@ -708,8 +717,8 @@ func (r *timingReader) ObjectSize() (int64, error) { func (r *timingReader) Close() error { var closeErr error - // Call the wrapped reader if it implements Close(). - if closer, ok := r.Reader.(io.Closer); ok { + // Call the wrapped reader if it implements Close(), only if we've been asked to close it. + if closer, ok := r.Reader.(io.Closer); r.closeReader && ok { closeErr = closer.Close() if !r.alreadyGotErr && closeErr != nil { diff --git a/objstore_test.go b/objstore_test.go index f971e04e..908c866c 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -210,7 +210,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { func TestTimingReader(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) - tr := newTimingReader(r, "", m.opsDuration, m.opsFailures, func(err error) bool { + tr := newTimingReader(r, true, "", m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) @@ -251,7 +251,7 @@ func TestTimingReader_ShouldCorrectlyWrapFile(t *testing.T) { }) m := WrapWithMetrics(NewInMemBucket(), nil, "") - r := newTimingReader(file, "", m.opsDuration, m.opsFailures, func(err error) bool { + r := newTimingReader(file, true, "", m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) @@ -314,7 +314,18 @@ func TestWrapWithMetrics_UploadShouldPreserveReaderFeatures(t *testing.T) { testutil.Equals(t, testData.expectedIsReaderAt, isReaderAt) }) } +} + +func TestWrapWithMetrics_UploadShouldNotCloseTheReader(t *testing.T) { + reader := &closeTrackerReader{ + Reader: bytes.NewBuffer([]byte("test")), + } + bucket := WrapWithMetrics(NewInMemBucket(), nil, "") + testutil.Ok(t, bucket.Upload(context.Background(), "dir/obj1", reader)) + + // Should not call Close() on the reader. + testutil.Assert(t, !reader.closeCalled) } // seekerBucket implements Bucket and keeps a reference of the io.Reader passed to Upload(). @@ -361,3 +372,14 @@ func (b unreliableBucket) Get(ctx context.Context, name string) (io.ReadCloser, } return b.Bucket.Get(ctx, name) } + +// closeTrackerReader is a io.ReadCloser which keeps track whether Close() has been called. +type closeTrackerReader struct { + io.Reader + closeCalled bool +} + +func (r *closeTrackerReader) Close() error { + r.closeCalled = true + return nil +} From 499af24cb3dcf009963ffdcbb9f9cdb63cf2dbab Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 26 Jan 2024 15:15:20 +0100 Subject: [PATCH 071/130] Improved TestWrapWithMetrics_CloseReader Signed-off-by: Marco Pracucci --- objstore_test.go | 114 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 94 insertions(+), 20 deletions(-) diff --git a/objstore_test.go b/objstore_test.go index 908c866c..560636f3 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -301,43 +301,87 @@ func TestWrapWithMetrics_UploadShouldPreserveReaderFeatures(t *testing.T) { for testName, testData := range tests { t.Run(testName, func(t *testing.T) { - m := &uploadTrackerTestBucket{ + var uploadReader io.Reader + + m := &testBucket{ Bucket: WrapWithMetrics(NewInMemBucket(), nil, ""), + upload: func(ctx context.Context, name string, r io.Reader) error { + uploadReader = r + return nil + }, } testutil.Ok(t, m.Upload(context.Background(), "dir/obj1", testData.reader)) - _, isSeeker := m.uploadReader.(io.Seeker) + _, isSeeker := uploadReader.(io.Seeker) testutil.Equals(t, testData.expectedIsSeeker, isSeeker) - _, isReaderAt := m.uploadReader.(io.ReaderAt) + _, isReaderAt := uploadReader.(io.ReaderAt) testutil.Equals(t, testData.expectedIsReaderAt, isReaderAt) }) } } -func TestWrapWithMetrics_UploadShouldNotCloseTheReader(t *testing.T) { - reader := &closeTrackerReader{ - Reader: bytes.NewBuffer([]byte("test")), - } +func TestWrapWithMetrics_CloseReader(t *testing.T) { + const objPath = "dir/obj1" - bucket := WrapWithMetrics(NewInMemBucket(), nil, "") - testutil.Ok(t, bucket.Upload(context.Background(), "dir/obj1", reader)) + t.Run("Upload() should not close the input Reader", func(t *testing.T) { + reader := &closeTrackerReader{ + Reader: bytes.NewBuffer([]byte("test")), + } - // Should not call Close() on the reader. - testutil.Assert(t, !reader.closeCalled) -} + bucket := WrapWithMetrics(NewInMemBucket(), nil, "") + testutil.Ok(t, bucket.Upload(context.Background(), objPath, reader)) -// seekerBucket implements Bucket and keeps a reference of the io.Reader passed to Upload(). -type uploadTrackerTestBucket struct { - Bucket + // Should not call Close() on the reader. + testutil.Assert(t, !reader.closeCalled) - uploadReader io.Reader -} + // An explicit call to Close() should close it. + testutil.Ok(t, reader.Close()) + testutil.Assert(t, reader.closeCalled) + }) -func (b *uploadTrackerTestBucket) Upload(ctx context.Context, name string, r io.Reader) error { - b.uploadReader = r - return nil + t.Run("Get() should return a wrapper io.ReadCloser that correctly Close the wrapped one", func(t *testing.T) { + origReader := &closeTrackerReader{ + Reader: bytes.NewBuffer([]byte("test")), + } + + bucket := WrapWithMetrics(&testBucket{ + get: func(_ context.Context, _ string) (io.ReadCloser, error) { + return origReader, nil + }, + }, nil, "") + + wrappedReader, err := bucket.Get(context.Background(), objPath) + testutil.Ok(t, err) + testutil.Assert(t, origReader != wrappedReader) + + // Calling Close() to the wrappedReader should close origReader. + testutil.Assert(t, !origReader.closeCalled) + testutil.Ok(t, wrappedReader.Close()) + testutil.Assert(t, origReader.closeCalled) + }) + + t.Run("GetRange() should return a wrapper io.ReadCloser that correctly Close the wrapped one", func(t *testing.T) { + origReader := &closeTrackerReader{ + Reader: bytes.NewBuffer([]byte("test")), + } + + bucket := WrapWithMetrics(&testBucket{ + getRange: func(_ context.Context, _ string, _, _ int64) (io.ReadCloser, error) { + return origReader, nil + }, + }, nil, "") + + wrappedReader, err := bucket.GetRange(context.Background(), objPath, 0, 1) + testutil.Ok(t, err) + testutil.Assert(t, origReader != wrappedReader) + + // Calling Close() to the wrappedReader should close origReader. + testutil.Assert(t, !origReader.closeCalled) + testutil.Ok(t, wrappedReader.Close()) + testutil.Assert(t, origReader.closeCalled) + }) } func TestDownloadDir_CleanUp(t *testing.T) { @@ -383,3 +427,33 @@ func (r *closeTrackerReader) Close() error { r.closeCalled = true return nil } + +// testBucket implements Bucket and allows to customize the functions. +type testBucket struct { + Bucket + + upload func(ctx context.Context, name string, r io.Reader) error + get func(ctx context.Context, name string) (io.ReadCloser, error) + getRange func(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) +} + +func (b *testBucket) Upload(ctx context.Context, name string, r io.Reader) error { + if b.upload != nil { + return b.upload(ctx, name, r) + } + return errors.New("Upload has not been mocked") +} + +func (b *testBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { + if b.get != nil { + return b.get(ctx, name) + } + return nil, errors.New("Get has not been mocked") +} + +func (b *testBucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { + if b.getRange != nil { + return b.getRange(ctx, name, off, length) + } + return nil, errors.New("GetRange has not been mocked") +} From c83effb52e5d57b4f2ac15bca7b9ff2671a2b42a Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 26 Jan 2024 15:48:57 +0100 Subject: [PATCH 072/130] Added TestMetricBucket_ReaderCloseError Signed-off-by: Marco Pracucci --- objstore_test.go | 346 +++++++++++++++++++++++++++++------------------ 1 file changed, 215 insertions(+), 131 deletions(-) diff --git a/objstore_test.go b/objstore_test.go index 560636f3..ababe62c 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -81,6 +81,208 @@ func TestMetricBucket_Multiple_Clients(t *testing.T) { WrapWithMetrics(NewInMemBucket(), reg, "def") } +func TestMetricBucket_UploadShouldPreserveReaderFeatures(t *testing.T) { + tests := map[string]struct { + reader io.Reader + expectedIsSeeker bool + expectedIsReaderAt bool + }{ + "bytes.Reader": { + reader: bytes.NewReader([]byte("1")), + expectedIsSeeker: true, + expectedIsReaderAt: true, + }, + "bytes.Buffer": { + reader: bytes.NewBuffer([]byte("1")), + expectedIsSeeker: false, + expectedIsReaderAt: false, + }, + "os.File": { + reader: func() io.Reader { + // Create a test file. + testFilepath := filepath.Join(t.TempDir(), "test") + testutil.Ok(t, os.WriteFile(testFilepath, []byte("test"), os.ModePerm)) + + // Open the file (it will be used as io.Reader). + file, err := os.Open(testFilepath) + testutil.Ok(t, err) + t.Cleanup(func() { + testutil.Ok(t, file.Close()) + }) + + return file + }(), + expectedIsSeeker: true, + expectedIsReaderAt: true, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + var uploadReader io.Reader + + m := &mockBucket{ + Bucket: WrapWithMetrics(NewInMemBucket(), nil, ""), + upload: func(ctx context.Context, name string, r io.Reader) error { + uploadReader = r + return nil + }, + } + + testutil.Ok(t, m.Upload(context.Background(), "dir/obj1", testData.reader)) + + _, isSeeker := uploadReader.(io.Seeker) + testutil.Equals(t, testData.expectedIsSeeker, isSeeker) + + _, isReaderAt := uploadReader.(io.ReaderAt) + testutil.Equals(t, testData.expectedIsReaderAt, isReaderAt) + }) + } +} + +func TestMetricBucket_ReaderClose(t *testing.T) { + const objPath = "dir/obj1" + + t.Run("Upload() should not close the input Reader", func(t *testing.T) { + closeCalled := false + + reader := &mockReader{ + Reader: bytes.NewBuffer([]byte("test")), + close: func() error { + closeCalled = true + return nil + }, + } + + bucket := WrapWithMetrics(NewInMemBucket(), nil, "") + testutil.Ok(t, bucket.Upload(context.Background(), objPath, reader)) + + // Should not call Close() on the reader. + testutil.Assert(t, !closeCalled) + + // An explicit call to Close() should close it. + testutil.Ok(t, reader.Close()) + testutil.Assert(t, closeCalled) + + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpUpload))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpUpload))) + }) + + t.Run("Get() should return a wrapper io.ReadCloser that correctly Close the wrapped one", func(t *testing.T) { + closeCalled := false + + origReader := &mockReader{ + Reader: bytes.NewBuffer([]byte("test")), + close: func() error { + closeCalled = true + return nil + }, + } + + bucket := WrapWithMetrics(&mockBucket{ + get: func(_ context.Context, _ string) (io.ReadCloser, error) { + return origReader, nil + }, + }, nil, "") + + wrappedReader, err := bucket.Get(context.Background(), objPath) + testutil.Ok(t, err) + testutil.Assert(t, origReader != wrappedReader) + + // Calling Close() to the wrappedReader should close origReader. + testutil.Assert(t, !closeCalled) + testutil.Ok(t, wrappedReader.Close()) + testutil.Assert(t, closeCalled) + + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpGet))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpGet))) + }) + + t.Run("GetRange() should return a wrapper io.ReadCloser that correctly Close the wrapped one", func(t *testing.T) { + closeCalled := false + + origReader := &mockReader{ + Reader: bytes.NewBuffer([]byte("test")), + close: func() error { + closeCalled = true + return nil + }, + } + + bucket := WrapWithMetrics(&mockBucket{ + getRange: func(_ context.Context, _ string, _, _ int64) (io.ReadCloser, error) { + return origReader, nil + }, + }, nil, "") + + wrappedReader, err := bucket.GetRange(context.Background(), objPath, 0, 1) + testutil.Ok(t, err) + testutil.Assert(t, origReader != wrappedReader) + + // Calling Close() to the wrappedReader should close origReader. + testutil.Assert(t, !closeCalled) + testutil.Ok(t, wrappedReader.Close()) + testutil.Assert(t, closeCalled) + + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpGetRange))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpGetRange))) + }) +} + +func TestMetricBucket_ReaderCloseError(t *testing.T) { + origReader := &mockReader{ + Reader: bytes.NewBuffer([]byte("test")), + close: func() error { + return errors.New("mocked error") + }, + } + + t.Run("Get() should track failure if reader Close() returns error", func(t *testing.T) { + bucket := WrapWithMetrics(&mockBucket{ + get: func(ctx context.Context, name string) (io.ReadCloser, error) { + return origReader, nil + }, + }, nil, "") + + testutil.NotOk(t, bucket.Upload(context.Background(), "test", origReader)) + + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpUpload))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpUpload))) + }) + + t.Run("Get() should track failure if reader Close() returns error", func(t *testing.T) { + bucket := WrapWithMetrics(&mockBucket{ + get: func(ctx context.Context, name string) (io.ReadCloser, error) { + return origReader, nil + }, + }, nil, "") + + reader, err := bucket.Get(context.Background(), "test") + testutil.Ok(t, err) + testutil.NotOk(t, reader.Close()) + testutil.NotOk(t, reader.Close()) // Called twice to ensure metrics are not tracked twice. + + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpGet))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpGet))) + }) + + t.Run("GetRange() should track failure if reader Close() returns error", func(t *testing.T) { + bucket := WrapWithMetrics(&mockBucket{ + getRange: func(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { + return origReader, nil + }, + }, nil, "") + + reader, err := bucket.GetRange(context.Background(), "test", 0, 1) + testutil.Ok(t, err) + testutil.NotOk(t, reader.Close()) + testutil.NotOk(t, reader.Close()) // Called twice to ensure metrics are not tracked twice. + + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpGetRange))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpGetRange))) + }) +} + func TestDownloadUploadDirConcurrency(t *testing.T) { r := prometheus.NewRegistry() m := WrapWithMetrics(NewInMemBucket(), r, "") @@ -263,127 +465,6 @@ func TestTimingReader_ShouldCorrectlyWrapFile(t *testing.T) { testutil.Assert(t, isReaderAt) } -func TestWrapWithMetrics_UploadShouldPreserveReaderFeatures(t *testing.T) { - tests := map[string]struct { - reader io.Reader - expectedIsSeeker bool - expectedIsReaderAt bool - }{ - "bytes.Reader": { - reader: bytes.NewReader([]byte("1")), - expectedIsSeeker: true, - expectedIsReaderAt: true, - }, - "bytes.Buffer": { - reader: bytes.NewBuffer([]byte("1")), - expectedIsSeeker: false, - expectedIsReaderAt: false, - }, - "os.File": { - reader: func() io.Reader { - // Create a test file. - testFilepath := filepath.Join(t.TempDir(), "test") - testutil.Ok(t, os.WriteFile(testFilepath, []byte("test"), os.ModePerm)) - - // Open the file (it will be used as io.Reader). - file, err := os.Open(testFilepath) - testutil.Ok(t, err) - t.Cleanup(func() { - testutil.Ok(t, file.Close()) - }) - - return file - }(), - expectedIsSeeker: true, - expectedIsReaderAt: true, - }, - } - - for testName, testData := range tests { - t.Run(testName, func(t *testing.T) { - var uploadReader io.Reader - - m := &testBucket{ - Bucket: WrapWithMetrics(NewInMemBucket(), nil, ""), - upload: func(ctx context.Context, name string, r io.Reader) error { - uploadReader = r - return nil - }, - } - - testutil.Ok(t, m.Upload(context.Background(), "dir/obj1", testData.reader)) - - _, isSeeker := uploadReader.(io.Seeker) - testutil.Equals(t, testData.expectedIsSeeker, isSeeker) - - _, isReaderAt := uploadReader.(io.ReaderAt) - testutil.Equals(t, testData.expectedIsReaderAt, isReaderAt) - }) - } -} - -func TestWrapWithMetrics_CloseReader(t *testing.T) { - const objPath = "dir/obj1" - - t.Run("Upload() should not close the input Reader", func(t *testing.T) { - reader := &closeTrackerReader{ - Reader: bytes.NewBuffer([]byte("test")), - } - - bucket := WrapWithMetrics(NewInMemBucket(), nil, "") - testutil.Ok(t, bucket.Upload(context.Background(), objPath, reader)) - - // Should not call Close() on the reader. - testutil.Assert(t, !reader.closeCalled) - - // An explicit call to Close() should close it. - testutil.Ok(t, reader.Close()) - testutil.Assert(t, reader.closeCalled) - }) - - t.Run("Get() should return a wrapper io.ReadCloser that correctly Close the wrapped one", func(t *testing.T) { - origReader := &closeTrackerReader{ - Reader: bytes.NewBuffer([]byte("test")), - } - - bucket := WrapWithMetrics(&testBucket{ - get: func(_ context.Context, _ string) (io.ReadCloser, error) { - return origReader, nil - }, - }, nil, "") - - wrappedReader, err := bucket.Get(context.Background(), objPath) - testutil.Ok(t, err) - testutil.Assert(t, origReader != wrappedReader) - - // Calling Close() to the wrappedReader should close origReader. - testutil.Assert(t, !origReader.closeCalled) - testutil.Ok(t, wrappedReader.Close()) - testutil.Assert(t, origReader.closeCalled) - }) - - t.Run("GetRange() should return a wrapper io.ReadCloser that correctly Close the wrapped one", func(t *testing.T) { - origReader := &closeTrackerReader{ - Reader: bytes.NewBuffer([]byte("test")), - } - - bucket := WrapWithMetrics(&testBucket{ - getRange: func(_ context.Context, _ string, _, _ int64) (io.ReadCloser, error) { - return origReader, nil - }, - }, nil, "") - - wrappedReader, err := bucket.GetRange(context.Background(), objPath, 0, 1) - testutil.Ok(t, err) - testutil.Assert(t, origReader != wrappedReader) - - // Calling Close() to the wrappedReader should close origReader. - testutil.Assert(t, !origReader.closeCalled) - testutil.Ok(t, wrappedReader.Close()) - testutil.Assert(t, origReader.closeCalled) - }) -} - func TestDownloadDir_CleanUp(t *testing.T) { b := unreliableBucket{ Bucket: NewInMemBucket(), @@ -417,19 +498,22 @@ func (b unreliableBucket) Get(ctx context.Context, name string) (io.ReadCloser, return b.Bucket.Get(ctx, name) } -// closeTrackerReader is a io.ReadCloser which keeps track whether Close() has been called. -type closeTrackerReader struct { +// mockReader implements io.ReadCloser and allows to mock the functions. +type mockReader struct { io.Reader - closeCalled bool + + close func() error } -func (r *closeTrackerReader) Close() error { - r.closeCalled = true +func (r *mockReader) Close() error { + if r.close != nil { + return r.close() + } return nil } -// testBucket implements Bucket and allows to customize the functions. -type testBucket struct { +// mockBucket implements Bucket and allows to mock the functions. +type mockBucket struct { Bucket upload func(ctx context.Context, name string, r io.Reader) error @@ -437,21 +521,21 @@ type testBucket struct { getRange func(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) } -func (b *testBucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *mockBucket) Upload(ctx context.Context, name string, r io.Reader) error { if b.upload != nil { return b.upload(ctx, name, r) } return errors.New("Upload has not been mocked") } -func (b *testBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { +func (b *mockBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { if b.get != nil { return b.get(ctx, name) } return nil, errors.New("Get has not been mocked") } -func (b *testBucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { +func (b *mockBucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { if b.getRange != nil { return b.getRange(ctx, name, off, length) } From cd121fe25e4c0a952f00ef07600fe9452924dc62 Mon Sep 17 00:00:00 2001 From: Joao Marcal Date: Mon, 5 Feb 2024 16:27:44 +0100 Subject: [PATCH 073/130] GCS: Adds HTTP Config similar to S3 (#86) * Add HTTP Config to GCS Signed-off-by: Joao Marcal * Fix tests Signed-off-by: Joao Marcal --------- Signed-off-by: Joao Marcal --- CHANGELOG.md | 1 + exthttp/transport.go | 10 +++++++ providers/gcs/gcs.go | 54 +++++++++++++++++++++++++++++++++---- providers/gcs/gcs_test.go | 57 +++++++++++++++++++++++++++++++++++++++ providers/s3/s3.go | 14 ++-------- 5 files changed, 119 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90b29a88..fbccc79d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#92](https://github.com/thanos-io/objstore/pull/92) GCS: Allow using a gRPC client. - [#94](https://github.com/thanos-io/objstore/pull/94) Allow timingReadCloser to be seeker - [#96](https://github.com/thanos-io/objstore/pull/96) Allow nopCloserWithObjectSize to be seeker +- [#86](https://github.com/thanos-io/objstore/pull/86) GCS: Add HTTP Config to GCS ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/exthttp/transport.go b/exthttp/transport.go index d9a807fd..92f86a11 100644 --- a/exthttp/transport.go +++ b/exthttp/transport.go @@ -11,6 +11,16 @@ import ( "github.com/prometheus/common/model" ) +var DefaultHTTPConfig = HTTPConfig{ + IdleConnTimeout: model.Duration(90 * time.Second), + ResponseHeaderTimeout: model.Duration(2 * time.Minute), + TLSHandshakeTimeout: model.Duration(10 * time.Second), + ExpectContinueTimeout: model.Duration(1 * time.Second), + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + MaxConnsPerHost: 0, +} + // HTTPConfig stores the http.Transport configuration for the cos and s3 minio client. type HTTPConfig struct { IdleConnTimeout model.Duration `yaml:"idle_conn_timeout"` diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 1d717b77..1b9c78c2 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -8,9 +8,11 @@ import ( "context" "fmt" "io" + "net/http" "runtime" "strings" "testing" + "time" "cloud.google.com/go/storage" "github.com/go-kit/log" @@ -19,17 +21,23 @@ import ( "golang.org/x/oauth2/google" "google.golang.org/api/iterator" "google.golang.org/api/option" + htransport "google.golang.org/api/transport/http" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "gopkg.in/yaml.v2" "github.com/thanos-io/objstore" + "github.com/thanos-io/objstore/exthttp" ) // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" +var DefaultConfig = Config{ + HTTPConfig: exthttp.DefaultHTTPConfig, +} + // Config stores the configuration for gcs bucket. type Config struct { Bucket string `yaml:"bucket"` @@ -39,7 +47,8 @@ type Config struct { // when direct path is not enabled. // See https://pkg.go.dev/cloud.google.com/go/storage#hdr-Experimental_gRPC_API for more details // on how to enable direct path. - GRPCConnPoolSize int `yaml:"grpc_conn_pool_size"` + GRPCConnPoolSize int `yaml:"grpc_conn_pool_size"` + HTTPConfig exthttp.HTTPConfig `yaml:"http_config"` } // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. @@ -51,14 +60,23 @@ type Bucket struct { closer io.Closer } +// parseConfig unmarshals a buffer into a Config with default values. +func parseConfig(conf []byte) (Config, error) { + config := DefaultConfig + if err := yaml.UnmarshalStrict(conf, &config); err != nil { + return Config{}, err + } + + return config, nil +} + // NewBucket returns a new Bucket against the given bucket handle. func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string) (*Bucket, error) { - var gc Config - if err := yaml.Unmarshal(conf, &gc); err != nil { + config, err := parseConfig(conf) + if err != nil { return nil, err } - - return NewBucketWithConfig(ctx, logger, gc, component) + return NewBucketWithConfig(ctx, logger, config, component) } // NewBucketWithConfig returns a new Bucket with gcs Config struct. @@ -76,12 +94,38 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp return nil, errors.Wrap(err, "failed to create credentials from JSON") } opts = append(opts, option.WithCredentials(credentials)) + } else { + opts = append(opts, option.WithoutAuthentication()) } opts = append(opts, option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())), ) + // Check if a roundtripper has been set in the config + // otherwise build the default transport. + var rt http.RoundTripper + if gc.HTTPConfig.Transport != nil { + rt = gc.HTTPConfig.Transport + } else { + var err error + rt, err = exthttp.DefaultTransport(gc.HTTPConfig) + if err != nil { + return nil, err + } + } + + gRT, err := htransport.NewTransport(context.Background(), rt, opts...) + if err != nil { + return nil, err + } + + httpCli := &http.Client{ + Transport: gRT, + Timeout: time.Duration(gc.HTTPConfig.IdleConnTimeout), + } + opts = append(opts, option.WithHTTPClient(httpCli)) + return newBucket(ctx, logger, gc, opts) } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 283de6b6..95e0b005 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -10,9 +10,11 @@ import ( "net/http/httptest" "os" "testing" + "time" "github.com/efficientgo/core/testutil" "github.com/go-kit/log" + "github.com/prometheus/common/model" ) func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) { @@ -44,3 +46,58 @@ func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) { testutil.NotOk(t, err) testutil.Equals(t, "storage: partial request not satisfied", err.Error()) } + +func TestParseConfig_HTTPConfig(t *testing.T) { + for _, tc := range []struct { + name string + input string + assertions func(cfg Config) + }{ + { + name: "DefaultHTTPConfig", + input: `bucket: abcd`, + assertions: func(cfg Config) { + testutil.Equals(t, cfg.HTTPConfig.IdleConnTimeout, model.Duration(90*time.Second)) + testutil.Equals(t, cfg.HTTPConfig.ResponseHeaderTimeout, model.Duration(2*time.Minute)) + testutil.Equals(t, cfg.HTTPConfig.InsecureSkipVerify, false) + }, + }, + { + name: "CustomHTTPConfig", + input: `bucket: abcd +http_config: + insecure_skip_verify: true + idle_conn_timeout: 50s + response_header_timeout: 1m`, + assertions: func(cfg Config) { + testutil.Equals(t, cfg.HTTPConfig.IdleConnTimeout, model.Duration(50*time.Second)) + testutil.Equals(t, cfg.HTTPConfig.ResponseHeaderTimeout, model.Duration(1*time.Minute)) + testutil.Equals(t, cfg.HTTPConfig.InsecureSkipVerify, true) + }, + }, + { + name: "CustomHTTPConfigWithTLS", + input: `bucket: abcd +http_config: + tls_config: + ca_file: /certs/ca.crt + cert_file: /certs/cert.crt + key_file: /certs/key.key + server_name: server + insecure_skip_verify: false`, + assertions: func(cfg Config) { + testutil.Equals(t, "/certs/ca.crt", cfg.HTTPConfig.TLSConfig.CAFile) + testutil.Equals(t, "/certs/cert.crt", cfg.HTTPConfig.TLSConfig.CertFile) + testutil.Equals(t, "/certs/key.key", cfg.HTTPConfig.TLSConfig.KeyFile) + testutil.Equals(t, "server", cfg.HTTPConfig.TLSConfig.ServerName) + testutil.Equals(t, false, cfg.HTTPConfig.TLSConfig.InsecureSkipVerify) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg, err := parseConfig([]byte(tc.input)) + testutil.Ok(t, err) + tc.assertions(cfg) + }) + } +} diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 507b550b..72130034 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -14,7 +14,6 @@ import ( "strconv" "strings" "testing" - "time" "github.com/efficientgo/core/logerrcapture" "github.com/go-kit/log" @@ -23,7 +22,6 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/pkg/errors" - "github.com/prometheus/common/model" "github.com/prometheus/common/version" "gopkg.in/yaml.v2" @@ -101,16 +99,8 @@ const ( ) var DefaultConfig = Config{ - PutUserMetadata: map[string]string{}, - HTTPConfig: exthttp.HTTPConfig{ - IdleConnTimeout: model.Duration(90 * time.Second), - ResponseHeaderTimeout: model.Duration(2 * time.Minute), - TLSHandshakeTimeout: model.Duration(10 * time.Second), - ExpectContinueTimeout: model.Duration(1 * time.Second), - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, - MaxConnsPerHost: 0, - }, + PutUserMetadata: map[string]string{}, + HTTPConfig: exthttp.DefaultHTTPConfig, PartSize: 1024 * 1024 * 64, // 64MB. BucketLookupType: AutoLookup, SendContentMd5: true, // Default to using MD5. From a8d75c522f64f288ae624118468bda27ac08c98a Mon Sep 17 00:00:00 2001 From: Bayan Taani <86984560+btaani@users.noreply.github.com> Date: Mon, 12 Feb 2024 14:31:25 +0100 Subject: [PATCH 074/130] Swift: Add HTTP_Config (#99) Swift: add HTTP_Config Signed-off-by: btaani --- CHANGELOG.md | 1 + providers/swift/swift.go | 72 ++++++++++++++++++++++------------- providers/swift/swift_test.go | 30 +++++++++++++++ 3 files changed, 77 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbccc79d..491cf396 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#94](https://github.com/thanos-io/objstore/pull/94) Allow timingReadCloser to be seeker - [#96](https://github.com/thanos-io/objstore/pull/96) Allow nopCloserWithObjectSize to be seeker - [#86](https://github.com/thanos-io/objstore/pull/86) GCS: Add HTTP Config to GCS +- [#99](https://github.com/thanos-io/objstore/pull/99) Swift: Add HTTP_Config ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/providers/swift/swift.go b/providers/swift/swift.go index 9bfa3cf8..a8c56c55 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "io" + "net/http" "os" "strconv" "strings" @@ -21,6 +22,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/thanos-io/objstore" + "github.com/thanos-io/objstore/exthttp" "gopkg.in/yaml.v2" ) @@ -37,33 +39,35 @@ var DefaultConfig = Config{ Retries: 3, ConnectTimeout: model.Duration(10 * time.Second), Timeout: model.Duration(5 * time.Minute), + HTTPConfig: exthttp.DefaultHTTPConfig, } type Config struct { - AuthVersion int `yaml:"auth_version"` - AuthUrl string `yaml:"auth_url"` - Username string `yaml:"username"` - UserDomainName string `yaml:"user_domain_name"` - UserDomainID string `yaml:"user_domain_id"` - UserId string `yaml:"user_id"` - Password string `yaml:"password"` - DomainId string `yaml:"domain_id"` - DomainName string `yaml:"domain_name"` - ApplicationCredentialID string `yaml:"application_credential_id"` - ApplicationCredentialName string `yaml:"application_credential_name"` - ApplicationCredentialSecret string `yaml:"application_credential_secret"` - ProjectID string `yaml:"project_id"` - ProjectName string `yaml:"project_name"` - ProjectDomainID string `yaml:"project_domain_id"` - ProjectDomainName string `yaml:"project_domain_name"` - RegionName string `yaml:"region_name"` - ContainerName string `yaml:"container_name"` - ChunkSize int64 `yaml:"large_object_chunk_size"` - SegmentContainerName string `yaml:"large_object_segments_container_name"` - Retries int `yaml:"retries"` - ConnectTimeout model.Duration `yaml:"connect_timeout"` - Timeout model.Duration `yaml:"timeout"` - UseDynamicLargeObjects bool `yaml:"use_dynamic_large_objects"` + AuthVersion int `yaml:"auth_version"` + AuthUrl string `yaml:"auth_url"` + Username string `yaml:"username"` + UserDomainName string `yaml:"user_domain_name"` + UserDomainID string `yaml:"user_domain_id"` + UserId string `yaml:"user_id"` + Password string `yaml:"password"` + DomainId string `yaml:"domain_id"` + DomainName string `yaml:"domain_name"` + ApplicationCredentialID string `yaml:"application_credential_id"` + ApplicationCredentialName string `yaml:"application_credential_name"` + ApplicationCredentialSecret string `yaml:"application_credential_secret"` + ProjectID string `yaml:"project_id"` + ProjectName string `yaml:"project_name"` + ProjectDomainID string `yaml:"project_domain_id"` + ProjectDomainName string `yaml:"project_domain_name"` + RegionName string `yaml:"region_name"` + ContainerName string `yaml:"container_name"` + ChunkSize int64 `yaml:"large_object_chunk_size"` + SegmentContainerName string `yaml:"large_object_segments_container_name"` + Retries int `yaml:"retries"` + ConnectTimeout model.Duration `yaml:"connect_timeout"` + Timeout model.Duration `yaml:"timeout"` + UseDynamicLargeObjects bool `yaml:"use_dynamic_large_objects"` + HTTPConfig exthttp.HTTPConfig `yaml:"http_config"` } func parseConfig(conf []byte) (*Config, error) { @@ -101,6 +105,7 @@ func configFromEnv() (*Config, error) { ConnectTimeout: model.Duration(c.ConnectTimeout), Timeout: model.Duration(c.Timeout), UseDynamicLargeObjects: false, + HTTPConfig: DefaultConfig.HTTPConfig, } if os.Getenv("SWIFT_CHUNK_SIZE") != "" { var err error @@ -115,7 +120,7 @@ func configFromEnv() (*Config, error) { return &config, nil } -func connectionFromConfig(sc *Config) *swift.Connection { +func connectionFromConfig(sc *Config, rt http.RoundTripper) *swift.Connection { connection := swift.Connection{ AuthVersion: sc.AuthVersion, AuthUrl: sc.AuthUrl, @@ -135,6 +140,7 @@ func connectionFromConfig(sc *Config) *swift.Connection { Retries: sc.Retries, ConnectTimeout: time.Duration(sc.ConnectTimeout), Timeout: time.Duration(sc.Timeout), + Transport: rt, } return &connection } @@ -173,7 +179,21 @@ func ensureContainer(connection *swift.Connection, name string, createIfNotExist } func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool) (*Container, error) { - connection := connectionFromConfig(sc) + + // Check if a roundtripper has been set in the config + // otherwise build the default transport. + var rt http.RoundTripper + if sc.HTTPConfig.Transport != nil { + rt = sc.HTTPConfig.Transport + } else { + var err error + rt, err = exthttp.DefaultTransport(sc.HTTPConfig) + if err != nil { + return nil, err + } + } + + connection := connectionFromConfig(sc, rt) if err := connection.Authenticate(); err != nil { return nil, errors.Wrap(err, "authentication") } diff --git a/providers/swift/swift_test.go b/providers/swift/swift_test.go index 9c32acbd..656e7756 100644 --- a/providers/swift/swift_test.go +++ b/providers/swift/swift_test.go @@ -5,8 +5,10 @@ package swift import ( "testing" + "time" "github.com/efficientgo/core/testutil" + "github.com/prometheus/common/model" ) func TestParseConfig(t *testing.T) { @@ -34,3 +36,31 @@ tenant_name: something`) // Must result in unmarshal error as there's no `tenant_name` in SwiftConfig. testutil.NotOk(t, err) } + +func TestParseConfig_HTTPConfig(t *testing.T) { + input := []byte(`auth_url: http://identity.something.com/v3 +username: thanos +user_domain_name: userDomain +project_name: thanosProject +project_domain_name: projectDomain +http_config: + tls_config: + ca_file: /certs/ca.crt + cert_file: /certs/cert.crt + key_file: /certs/key.key + server_name: server + insecure_skip_verify: false`) + cfg, err := parseConfig([]byte(input)) + + testutil.Ok(t, err) + + testutil.Equals(t, "http://identity.something.com/v3", cfg.AuthUrl) + testutil.Equals(t, "thanos", cfg.Username) + testutil.Equals(t, "userDomain", cfg.UserDomainName) + testutil.Equals(t, "thanosProject", cfg.ProjectName) + testutil.Equals(t, "projectDomain", cfg.ProjectDomainName) + testutil.Equals(t, model.Duration(90*time.Second), cfg.HTTPConfig.IdleConnTimeout) + testutil.Equals(t, model.Duration(2*time.Minute), cfg.HTTPConfig.ResponseHeaderTimeout) + testutil.Equals(t, false, cfg.HTTPConfig.InsecureSkipVerify) + +} From 7514e4eb66e61e1a5af899984ec11aa84ce4b7ec Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Mon, 26 Feb 2024 19:07:32 +0100 Subject: [PATCH 075/130] azure: bump sdk version Signed-off-by: Michael Hoffmann --- CHANGELOG.md | 3 ++- README.md | 32 ++++++++++++++++++++++++++++++++ go.mod | 22 +++++++++++----------- go.sum | 49 +++++++++++++++++++++++++------------------------ 4 files changed, 70 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 491cf396..a020541a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ## Unreleased ### Fixed +- [#102](https://github.com/thanos-io/objstore/pull/102) Azure: bump azblob sdk to get concurrency fixes. - [#33](https://github.com/thanos-io/objstore/pull/33) Tracing: Add `ContextWithTracer()` to inject the tracer into the context. - [#34](https://github.com/thanos-io/objstore/pull/34) Fix ignored options when creating shared credential Azure client. - [#62](https://github.com/thanos-io/objstore/pull/62) S3: Fix ignored context cancellation in `Iter` method. @@ -37,7 +38,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#76](https://github.com/thanos-io/objstore/pull/76) GCS: Query for object names only in `Iter` to possibly improve performance when listing objects. - [#85](https://github.com/thanos-io/objstore/pull/85) S3: Allow checksum algorithm to be configured - [#92](https://github.com/thanos-io/objstore/pull/92) GCS: Allow using a gRPC client. -- [#94](https://github.com/thanos-io/objstore/pull/94) Allow timingReadCloser to be seeker +- [#94](https://github.com/thanos-io/objstore/pull/94) Allow timingReadCloser to be seeker - [#96](https://github.com/thanos-io/objstore/pull/96) Allow nopCloserWithObjectSize to be seeker - [#86](https://github.com/thanos-io/objstore/pull/86) GCS: Add HTTP Config to GCS - [#99](https://github.com/thanos-io/objstore/pull/99) Swift: Add HTTP_Config diff --git a/README.md b/README.md index a42ab726..6ebdffc1 100644 --- a/README.md +++ b/README.md @@ -354,6 +354,22 @@ config: service_account: "" use_grpc: false grpc_conn_pool_size: 0 + http_config: + idle_conn_timeout: 0s + response_header_timeout: 0s + insecure_skip_verify: false + tls_handshake_timeout: 0s + expect_continue_timeout: 0s + max_idle_conns: 0 + max_idle_conns_per_host: 0 + max_conns_per_host: 0 + tls_config: + ca_file: "" + cert_file: "" + key_file: "" + server_name: "" + insecure_skip_verify: false + disable_compression: false prefix: "" ``` @@ -501,6 +517,22 @@ config: connect_timeout: 10s timeout: 5m use_dynamic_large_objects: false + http_config: + idle_conn_timeout: 1m30s + response_header_timeout: 2m + insecure_skip_verify: false + tls_handshake_timeout: 10s + expect_continue_timeout: 1s + max_idle_conns: 100 + max_idle_conns_per_host: 100 + max_conns_per_host: 0 + tls_config: + ca_file: "" + cert_file: "" + key_file: "" + server_name: "" + insecure_skip_verify: false + disable_compression: false prefix: "" ``` diff --git a/go.mod b/go.mod index b7b8cffc..aefc59c7 100644 --- a/go.mod +++ b/go.mod @@ -37,8 +37,8 @@ require ( cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.5 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.11.0 // indirect @@ -58,12 +58,12 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/gofrs/flock v0.8.1 // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.7 // indirect - github.com/google/uuid v1.4.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect @@ -79,15 +79,15 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.4.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect @@ -101,10 +101,10 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.15.0 // indirect + golang.org/x/crypto v0.19.0 // indirect ) diff --git a/go.sum b/go.sum index 5703ca6c..ec4abd33 100644 --- a/go.sum +++ b/go.sum @@ -41,16 +41,17 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w= cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= -github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 h1:IfFdxTUDiV58iZqPKgyWiz4X4fCxZeQ1pTQPImLYXpY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= @@ -141,8 +142,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= +github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -209,8 +210,8 @@ github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -282,8 +283,8 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/oracle/oci-go-sdk/v65 v65.41.1 h1:+lbosOyNiib3TGJDvLq1HwEAuFqkOjPJDIkyxM15WdQ= github.com/oracle/oci-go-sdk/v65 v65.41.1/go.mod h1:MXMLMzHnnd9wlpgadPkdlkZ9YrwQmCOmbX5kjVEJodw= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -339,7 +340,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= @@ -368,8 +369,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -435,8 +436,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -495,16 +496,16 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From c3ccc5d619577f9996944b4a2c0b9b90f75757f1 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Wed, 28 Feb 2024 10:34:15 +0000 Subject: [PATCH 076/130] GCS: Fix logic for falling back to Google defaults (#106) Signed-off-by: Saswata Mukherjee --- providers/gcs/gcs.go | 2 -- providers/gcs/gcs_test.go | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 1b9c78c2..c9ee1601 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -94,8 +94,6 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp return nil, errors.Wrap(err, "failed to create credentials from JSON") } opts = append(opts, option.WithCredentials(credentials)) - } else { - opts = append(opts, option.WithoutAuthentication()) } opts = append(opts, diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 95e0b005..15bd122d 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -15,6 +15,7 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/prometheus/common/model" + "google.golang.org/api/option" ) func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) { @@ -35,7 +36,8 @@ func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) { ServiceAccount: "", } - bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test") + // NewBucketWithConfig wraps newBucket and processes HTTP options. Can skip for test. + bkt, err := newBucket(context.Background(), log.NewNopLogger(), cfg, []option.ClientOption{}) testutil.Ok(t, err) reader, err := bkt.Get(context.Background(), "test") From af5a2a5335dfc7c27d2f0a3119cb5a371d5cdce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gy=C3=B6rgy=20Krajcsovits?= Date: Mon, 4 Mar 2024 11:40:09 +0100 Subject: [PATCH 077/130] metrics: add native histogram definitions to histograms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add exponential bucket native histograms for objstore_bucket_operation_transferred_bytes objstore_bucket_operation_duration_seconds The resolution, maximum buckets, reset time can be changed later without migration as PromQL is able to handle those. For transferred bytes, the exponential buckets align perfectly with the previous exponential bucket definitions. Signed-off-by: György Krajcsovits --- go.mod | 11 +- go.sum | 376 ++-------------------------------------------------- objstore.go | 8 ++ 3 files changed, 25 insertions(+), 370 deletions(-) diff --git a/go.mod b/go.mod index aefc59c7..a72cbfa9 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/oracle/oci-go-sdk/v65 v65.41.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.12.2 - github.com/prometheus/common v0.36.0 + github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/common v0.44.0 github.com/tencentyun/cos-go-sdk-v5 v0.7.40 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 @@ -71,7 +71,7 @@ require ( github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect @@ -80,8 +80,8 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect + github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect @@ -96,7 +96,6 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index ec4abd33..a4675567 100644 --- a/go.sum +++ b/go.sum @@ -1,46 +1,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w= cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= @@ -53,14 +21,9 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0/go.mod h1:SUZc9YRRHf github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible h1:9gWa46nstkJ9miBReJcN8Gq34cBFbzSpQZVVT9N09TM= @@ -91,18 +54,11 @@ github.com/baidubce/bce-sdk-go v0.9.111 h1:yGgtPpZYUZW4uoVorQ4xnuEgVeddACydlcJKW github.com/baidubce/bce-sdk-go v0.9.111/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -124,69 +80,39 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -194,18 +120,7 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -214,49 +129,28 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI= @@ -268,13 +162,10 @@ github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= @@ -285,54 +176,32 @@ github.com/oracle/oci-go-sdk/v65 v65.41.1 h1:+lbosOyNiib3TGJDvLq1HwEAuFqkOjPJDIk github.com/oracle/oci-go-sdk/v65 v65.41.1/go.mod h1:MXMLMzHnnd9wlpgadPkdlkZ9YrwQmCOmbX5kjVEJodw= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.36.0 h1:78hJTing+BLYLjhXE+Z2BubeEymH5Lr0/Mt8FKkxxYo= -github.com/prometheus/common v0.36.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -345,15 +214,7 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= @@ -362,142 +223,43 @@ go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZE go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -508,123 +270,34 @@ golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.150.0 h1:Z9k22qD289SZ8gCJrk4DrWXkNjtfvKAUo/l1ma8eBYE= google.golang.org/api v0.150.0/go.mod h1:ccy+MJ6nrYFgE3WgRx/AMXOxOmU8Q4hSa+jjibzhxcg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= @@ -632,17 +305,9 @@ google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= @@ -654,7 +319,6 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= @@ -663,31 +327,15 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/objstore.go b/objstore.go index b1d7d09a..c9130687 100644 --- a/objstore.go +++ b/objstore.go @@ -429,6 +429,10 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu Help: "Number of bytes transferred from/to bucket per operation.", ConstLabels: prometheus.Labels{"bucket": name}, Buckets: prometheus.ExponentialBuckets(2<<14, 2, 16), // 32KiB, 64KiB, ... 1GiB + // Use factor=2 for native histograms, which gives similar buckets as the original exponential buckets. + NativeHistogramBucketFactor: 2, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"operation"}), opsDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ @@ -436,6 +440,10 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu Help: "Duration of successful operations against the bucket per operation - iter operations include time spent on each callback.", ConstLabels: prometheus.Labels{"bucket": name}, Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, + // Use the recommended defaults for native histograms with 10% growth factor. + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"operation"}), lastSuccessfulUploadTime: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ From 36910f4839f1717a8fd86ec3a834fc0a97e9b5fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gy=C3=B6rgy=20Krajcsovits?= Date: Mon, 4 Mar 2024 11:42:20 +0100 Subject: [PATCH 078/130] Update changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: György Krajcsovits --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a020541a..5889578b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#96](https://github.com/thanos-io/objstore/pull/96) Allow nopCloserWithObjectSize to be seeker - [#86](https://github.com/thanos-io/objstore/pull/86) GCS: Add HTTP Config to GCS - [#99](https://github.com/thanos-io/objstore/pull/99) Swift: Add HTTP_Config +- [#108](https://github.com/thanos-io/objstore/pull/108) Metrics: Add native histogram definitions to histograms ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From e8336a5fd5f38f13281cb69ee7c3a4262450d4fb Mon Sep 17 00:00:00 2001 From: Joao Marcal Date: Sat, 9 Mar 2024 08:53:57 +0100 Subject: [PATCH 079/130] GCS: adds scope to fix bug introduced in HTTPConfig support (#109) GCS: adds scope to fix bug introduced in https://github.com/thanos-io/objstore/pull/86 Issue was reported in https://github.com/thanos-io/objstore/pull/106#issuecomment-1971490145 Signed-off-by: Joao Marcal --- providers/gcs/gcs.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index c9ee1601..75c61303 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -113,6 +113,10 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp } } + // GCS uses some defaults when "options.WithHTTPClient" is not used that are important when we call + // htransport.NewTransport namely the scopes that are then used for OAth authentication. So to build our own + // http client we need to se those defaults + opts = append(opts, option.WithScopes(storage.ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")) gRT, err := htransport.NewTransport(context.Background(), rt, opts...) if err != nil { return nil, err From d265efdffea7ee8ef9ce9fa6c7e166bfdae021bc Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Wed, 3 Apr 2024 15:29:33 +0200 Subject: [PATCH 080/130] Fix GCS grpc connections Signed-off-by: Pedro Tanaka --- providers/gcs/gcs.go | 18 ++++++++++++++---- providers/gcs/gcs_test.go | 14 ++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 75c61303..4e2ea6ba 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -100,6 +100,18 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())), ) + if !gc.UseGRPC { + var err error + opts, err = appendHttpOptions(gc, opts) + if err != nil { + return nil, err + } + } + + return newBucket(ctx, logger, gc, opts) +} + +func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOption, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. var rt http.RoundTripper @@ -119,16 +131,14 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp opts = append(opts, option.WithScopes(storage.ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")) gRT, err := htransport.NewTransport(context.Background(), rt, opts...) if err != nil { - return nil, err + return nil, nil } httpCli := &http.Client{ Transport: gRT, Timeout: time.Duration(gc.HTTPConfig.IdleConnTimeout), } - opts = append(opts, option.WithHTTPClient(httpCli)) - - return newBucket(ctx, logger, gc, opts) + return append(opts, option.WithHTTPClient(httpCli)), nil } func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option.ClientOption) (*Bucket, error) { diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 15bd122d..c2afcad6 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -49,6 +49,20 @@ func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) { testutil.Equals(t, "storage: partial request not satisfied", err.Error()) } +func TestNewBucketWithConfig_ShouldCreateGRPC(t *testing.T) { + cfg := Config{ + Bucket: "test-bucket", + ServiceAccount: "", + UseGRPC: true, + } + + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket") + testutil.Ok(t, err) + + // Check if the bucket is created. + testutil.Assert(t, bkt != nil, "expected bucket to be created") +} + func TestParseConfig_HTTPConfig(t *testing.T) { for _, tc := range []struct { name string From f0fdab7793181ac5223c91dc5cd65084d466aba1 Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Wed, 3 Apr 2024 15:40:29 +0200 Subject: [PATCH 081/130] Add changelog Signed-off-by: Pedro Tanaka --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5889578b..072aafc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ## Unreleased ### Fixed +- [#115](https://github.com/thanos-io/objstore/pull/115) GCS: Fix creation of bucket with GRPC connections. - [#102](https://github.com/thanos-io/objstore/pull/102) Azure: bump azblob sdk to get concurrency fixes. - [#33](https://github.com/thanos-io/objstore/pull/33) Tracing: Add `ContextWithTracer()` to inject the tracer into the context. - [#34](https://github.com/thanos-io/objstore/pull/34) Fix ignored options when creating shared credential Azure client. From 3fddfd756889545d77c513f9cc4bcc8a3a742884 Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Wed, 3 Apr 2024 15:50:19 +0200 Subject: [PATCH 082/130] Fixing test in CI Signed-off-by: Pedro Tanaka --- go.mod | 53 ++++++++++++-------- go.sum | 103 +++++++++++++++++++++++--------------- providers/gcs/gcs_test.go | 8 +++ 3 files changed, 102 insertions(+), 62 deletions(-) diff --git a/go.mod b/go.mod index a72cbfa9..a82d5ec2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/thanos-io/objstore go 1.20 require ( - cloud.google.com/go/storage v1.35.1 + cloud.google.com/go/storage v1.40.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go-v2 v1.16.0 github.com/aws/aws-sdk-go-v2/config v1.15.1 @@ -11,6 +11,7 @@ require ( github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4 github.com/efficientgo/e2e v0.13.1-0.20220922081603-45de9fc588a8 github.com/fatih/structtag v1.2.0 + github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474 github.com/go-kit/log v0.2.1 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible github.com/minio/minio-go/v7 v7.0.61 @@ -20,23 +21,24 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/prometheus/common v0.44.0 + github.com/stretchr/testify v1.8.4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40 - go.opentelemetry.io/otel v1.16.0 - go.opentelemetry.io/otel/trace v1.16.0 + go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.14.0 - golang.org/x/sync v0.5.0 - google.golang.org/api v0.150.0 - google.golang.org/grpc v1.59.0 + golang.org/x/oauth2 v0.18.0 + golang.org/x/sync v0.6.0 + google.golang.org/api v0.172.0 + google.golang.org/grpc v1.62.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.110.10 // indirect - cloud.google.com/go/compute v1.23.3 // indirect + cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go/compute v1.24.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.5 // indirect + cloud.google.com/go/iam v1.1.7 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect @@ -52,20 +54,25 @@ require ( github.com/aws/smithy-go v1.11.1 // indirect github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bluele/gcache v0.0.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.2 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect @@ -80,23 +87,27 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/sys v0.17.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.4.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + golang.org/x/time v0.5.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -105,5 +116,5 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.19.0 // indirect + golang.org/x/crypto v0.21.0 // indirect ) diff --git a/go.sum b/go.sum index a4675567..22fdf1ea 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= -cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= -cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= -cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= -cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= -cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w= -cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= +cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= +cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= +cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= +cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= @@ -56,6 +56,8 @@ github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -80,10 +82,19 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474 h1:TufioMBjkJ6/Oqmlye/ReuxHFS35HyLmypj/BNy/8GY= +github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474/go.mod h1:PQwxF4UU8wuL+srGxr3BOhIW5zXqgucwVlO/nPZLsxw= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= @@ -104,8 +115,10 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -129,8 +142,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= +github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -210,6 +223,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= @@ -217,17 +231,24 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7y github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= -go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -242,18 +263,18 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= -golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -266,8 +287,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -276,8 +297,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= -golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -288,9 +309,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.150.0 h1:Z9k22qD289SZ8gCJrk4DrWXkNjtfvKAUo/l1ma8eBYE= -google.golang.org/api v0.150.0/go.mod h1:ccy+MJ6nrYFgE3WgRx/AMXOxOmU8Q4hSa+jjibzhxcg= +google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= +google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= @@ -298,19 +318,19 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= -google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= -google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= -google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= +google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -322,8 +342,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -337,5 +357,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index c2afcad6..88c0dcd4 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -13,8 +13,10 @@ import ( "time" "github.com/efficientgo/core/testutil" + "github.com/fullstorydev/emulators/storage/gcsemu" "github.com/go-kit/log" "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" "google.golang.org/api/option" ) @@ -56,6 +58,12 @@ func TestNewBucketWithConfig_ShouldCreateGRPC(t *testing.T) { UseGRPC: true, } + svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) + err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) + require.NoError(t, err) + err = os.Setenv("GCS_EMULATOR_HOST", svr.Addr) + require.NoError(t, err) + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket") testutil.Ok(t, err) From bbb25c96af00845083829be828fd101e50d6f283 Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Wed, 3 Apr 2024 16:02:27 +0200 Subject: [PATCH 083/130] fixing small typo Signed-off-by: Pedro Tanaka --- providers/gcs/gcs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 4e2ea6ba..377c1c26 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -131,7 +131,7 @@ func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOp opts = append(opts, option.WithScopes(storage.ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")) gRT, err := htransport.NewTransport(context.Background(), rt, opts...) if err != nil { - return nil, nil + return nil, err } httpCli := &http.Client{ From 61ee31dd03581bcb86ff4921a338f5fbdb3f75a7 Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Wed, 3 Apr 2024 16:03:08 +0200 Subject: [PATCH 084/130] change changelog Signed-off-by: Pedro Tanaka --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 072aafc1..a5290479 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ## Unreleased ### Fixed -- [#115](https://github.com/thanos-io/objstore/pull/115) GCS: Fix creation of bucket with GRPC connections. +- [#115](https://github.com/thanos-io/objstore/pull/115) GCS: Fix creation of bucket with GRPC connections. Also update storage client to `v1.40.0`. - [#102](https://github.com/thanos-io/objstore/pull/102) Azure: bump azblob sdk to get concurrency fixes. - [#33](https://github.com/thanos-io/objstore/pull/33) Tracing: Add `ContextWithTracer()` to inject the tracer into the context. - [#34](https://github.com/thanos-io/objstore/pull/34) Fix ignored options when creating shared credential Azure client. From 074314a3826c0f33ca8420e076d836df0e402947 Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Wed, 3 Apr 2024 16:07:15 +0200 Subject: [PATCH 085/130] fixing deprecation Signed-off-by: Pedro Tanaka --- providers/gcs/gcs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 377c1c26..c4743630 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -24,6 +24,7 @@ import ( htransport "google.golang.org/api/transport/http" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/experimental" "google.golang.org/grpc/status" "gopkg.in/yaml.v2" @@ -148,7 +149,7 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option. ) if gc.UseGRPC { opts = append(opts, - option.WithGRPCDialOption(grpc.WithRecvBufferPool(grpc.NewSharedBufferPool())), + option.WithGRPCDialOption(experimental.WithRecvBufferPool(grpc.NewSharedBufferPool())), option.WithGRPCConnectionPool(gc.GRPCConnPoolSize), ) gcsClient, err = storage.NewGRPCClient(ctx, opts...) From 4517988ae0c2b1f3c3949ac96a006ee0984c64d6 Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Wed, 3 Apr 2024 16:16:37 +0200 Subject: [PATCH 086/130] trying once more to fix CI test Signed-off-by: Pedro Tanaka --- providers/gcs/gcs_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 88c0dcd4..2f2a4d0d 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -59,10 +59,13 @@ func TestNewBucketWithConfig_ShouldCreateGRPC(t *testing.T) { } svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) + testutil.Ok(t, err) err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) require.NoError(t, err) err = os.Setenv("GCS_EMULATOR_HOST", svr.Addr) require.NoError(t, err) + err = os.Setenv("STORAGE_EMULATOR_HOST_GRPC", svr.Addr) + require.NoError(t, err) bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket") testutil.Ok(t, err) From 2b48973a0aa53abee32eb81fc62e5defd8fad455 Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Wed, 3 Apr 2024 16:23:14 +0200 Subject: [PATCH 087/130] fixing unrelated linting breakage Signed-off-by: Pedro Tanaka --- client/factory_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/factory_test.go b/client/factory_test.go index dc9e46f0..4a9cf879 100644 --- a/client/factory_test.go +++ b/client/factory_test.go @@ -9,7 +9,7 @@ import ( "os" "github.com/go-kit/log" - "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" "github.com/thanos-io/objstore/tracing/opentelemetry" "github.com/thanos-io/objstore/tracing/opentracing" @@ -78,7 +78,7 @@ func ExampleTracingBucketUsingOpenTelemetry() { //nolint:govet } // Wrap it with tracing. - bucket = opentelemetry.WrapWithTraces(bucket, trace.NewNoopTracerProvider().Tracer("bucket")) + bucket = opentelemetry.WrapWithTraces(bucket, noop.NewTracerProvider().Tracer("bucket")) // Test it. exists, err := bucket.Exists(context.Background(), "example") From 645c63d7d37b44be65709859554cf8f19c297293 Mon Sep 17 00:00:00 2001 From: Pedro Tanaka Date: Wed, 3 Apr 2024 16:25:52 +0200 Subject: [PATCH 088/130] removing asssert lib Signed-off-by: Pedro Tanaka --- go.mod | 3 --- go.sum | 1 - providers/gcs/gcs_test.go | 7 +++---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index a82d5ec2..8887b088 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/prometheus/common v0.44.0 - github.com/stretchr/testify v1.8.4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40 go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 @@ -87,7 +86,6 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect @@ -107,7 +105,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/go.sum b/go.sum index 22fdf1ea..8e01ea18 100644 --- a/go.sum +++ b/go.sum @@ -223,7 +223,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 2f2a4d0d..c35b4f50 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -16,7 +16,6 @@ import ( "github.com/fullstorydev/emulators/storage/gcsemu" "github.com/go-kit/log" "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" "google.golang.org/api/option" ) @@ -61,11 +60,11 @@ func TestNewBucketWithConfig_ShouldCreateGRPC(t *testing.T) { svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) testutil.Ok(t, err) err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) - require.NoError(t, err) + testutil.Ok(t, err) err = os.Setenv("GCS_EMULATOR_HOST", svr.Addr) - require.NoError(t, err) + testutil.Ok(t, err) err = os.Setenv("STORAGE_EMULATOR_HOST_GRPC", svr.Addr) - require.NoError(t, err) + testutil.Ok(t, err) bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket") testutil.Ok(t, err) From e56e6f180a3b4a73fbafac3140db61a0603388de Mon Sep 17 00:00:00 2001 From: Charles Korn Date: Wed, 1 May 2024 11:50:07 +1000 Subject: [PATCH 089/130] Add tests for existing behaviour. Signed-off-by: Charles Korn --- objstore_test.go | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/objstore_test.go b/objstore_test.go index ababe62c..8221932f 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -412,7 +412,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { func TestTimingReader(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) - tr := newTimingReader(r, true, "", m.opsDuration, m.opsFailures, func(err error) bool { + tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) @@ -438,6 +438,44 @@ func TestTimingReader(t *testing.T) { _, isReaderAt := tr.(io.ReaderAt) testutil.Assert(t, isReaderAt) + + testutil.Equals(t, float64(0), promtest.ToFloat64(m.opsFailures.WithLabelValues(OpGet))) +} + +func TestTimingReader_ExpectedError(t *testing.T) { + readerErr := errors.New("something went wrong") + + m := WrapWithMetrics(NewInMemBucket(), nil, "") + r := dummyReader{readerErr} + tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.opsFetchedBytes, m.opsTransferredBytes) + + buf := make([]byte, 1) + _, err := io.ReadFull(tr, buf) + testutil.Equals(t, readerErr, err) + + testutil.Equals(t, float64(0), promtest.ToFloat64(m.opsFailures.WithLabelValues(OpGet))) +} + +func TestTimingReader_UnexpectedError(t *testing.T) { + readerErr := errors.New("something went wrong") + + m := WrapWithMetrics(NewInMemBucket(), nil, "") + r := dummyReader{readerErr} + tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) + + buf := make([]byte, 1) + _, err := io.ReadFull(tr, buf) + testutil.Equals(t, readerErr, err) + + testutil.Equals(t, float64(1), promtest.ToFloat64(m.opsFailures.WithLabelValues(OpGet))) +} + +type dummyReader struct { + err error +} + +func (r dummyReader) Read(_ []byte) (int, error) { + return 0, r.err } func TestTimingReader_ShouldCorrectlyWrapFile(t *testing.T) { From f1ab9d81b4979d2ab8bc2b2a1824b36c3c550c9d Mon Sep 17 00:00:00 2001 From: Charles Korn Date: Wed, 1 May 2024 11:51:58 +1000 Subject: [PATCH 090/130] Add failing test Signed-off-by: Charles Korn --- objstore_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/objstore_test.go b/objstore_test.go index 8221932f..f83073c4 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -470,6 +470,18 @@ func TestTimingReader_UnexpectedError(t *testing.T) { testutil.Equals(t, float64(1), promtest.ToFloat64(m.opsFailures.WithLabelValues(OpGet))) } +func TestTimingReader_ContextCancellation(t *testing.T) { + m := WrapWithMetrics(NewInMemBucket(), nil, "") + r := dummyReader{context.Canceled} + tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) + + buf := make([]byte, 1) + _, err := io.ReadFull(tr, buf) + testutil.Equals(t, context.Canceled, err) + + testutil.Equals(t, float64(0), promtest.ToFloat64(m.opsFailures.WithLabelValues(OpGet))) +} + type dummyReader struct { err error } From 88acb325249ee799563b434116e1144cfc7d3686 Mon Sep 17 00:00:00 2001 From: Charles Korn Date: Wed, 1 May 2024 11:55:23 +1000 Subject: [PATCH 091/130] Add fix for issue. Signed-off-by: Charles Korn --- objstore.go | 10 ++++++++-- objstore_test.go | 17 ++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/objstore.go b/objstore.go index c9130687..a83f7d2e 100644 --- a/objstore.go +++ b/objstore.go @@ -551,6 +551,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err return nil, err } return newTimingReader( + ctx, rc, true, op, @@ -574,6 +575,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in return nil, err } return newTimingReader( + ctx, rc, true, op, @@ -606,6 +608,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.ops.WithLabelValues(op).Inc() trc := newTimingReader( + ctx, r, false, op, @@ -663,6 +666,8 @@ func (b *metricBucket) Name() string { type timingReader struct { io.Reader + ctx context.Context + // closeReader holds whether the wrapper io.Reader should be closed when // Close() is called on the timingReader. closeReader bool @@ -682,7 +687,7 @@ type timingReader struct { transferredBytes *prometheus.HistogramVec } -func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { +func newTimingReader(ctx context.Context, r io.Reader, closeReader bool, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -690,6 +695,7 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H trc := timingReader{ Reader: r, + ctx: ctx, closeReader: closeReader, objSize: objSize, objSizeErr: objSizeErr, @@ -756,7 +762,7 @@ func (r *timingReader) Read(b []byte) (n int, err error) { r.readBytes += int64(n) // Report metric just once. if !r.alreadyGotErr && err != nil && err != io.EOF { - if !r.isFailureExpected(err) { + if !r.isFailureExpected(err) && r.ctx.Err() != context.Canceled { r.failed.WithLabelValues(r.op).Inc() } r.alreadyGotErr = true diff --git a/objstore_test.go b/objstore_test.go index f83073c4..35a5d88a 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -412,7 +412,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { func TestTimingReader(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) - tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { + tr := newTimingReader(context.Background(), r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) @@ -447,7 +447,7 @@ func TestTimingReader_ExpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.opsFetchedBytes, m.opsTransferredBytes) + tr := newTimingReader(context.Background(), r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.opsFetchedBytes, m.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -461,7 +461,7 @@ func TestTimingReader_UnexpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) + tr := newTimingReader(context.Background(), r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -471,13 +471,16 @@ func TestTimingReader_UnexpectedError(t *testing.T) { } func TestTimingReader_ContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + m := WrapWithMetrics(NewInMemBucket(), nil, "") - r := dummyReader{context.Canceled} - tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) + r := dummyReader{ctx.Err()} + tr := newTimingReader(ctx, r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) - testutil.Equals(t, context.Canceled, err) + testutil.Equals(t, ctx.Err(), err) testutil.Equals(t, float64(0), promtest.ToFloat64(m.opsFailures.WithLabelValues(OpGet))) } @@ -503,7 +506,7 @@ func TestTimingReader_ShouldCorrectlyWrapFile(t *testing.T) { }) m := WrapWithMetrics(NewInMemBucket(), nil, "") - r := newTimingReader(file, true, "", m.opsDuration, m.opsFailures, func(err error) bool { + r := newTimingReader(context.Background(), file, true, "", m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) From 374a7dc6f6a117675f53ed3b3edc8ff72ebe8a21 Mon Sep 17 00:00:00 2001 From: Charles Korn Date: Wed, 1 May 2024 12:00:59 +1000 Subject: [PATCH 092/130] Add changelog entry. Signed-off-by: Charles Korn --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5290479..0d0df778 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ## Unreleased ### Fixed +- [#117](https://github.com/thanos-io/objstore/pull/117) Metrics: Fix `objstore_bucket_operation_failures_total` incorrectly incremented if context is cancelled while reading object contents. - [#115](https://github.com/thanos-io/objstore/pull/115) GCS: Fix creation of bucket with GRPC connections. Also update storage client to `v1.40.0`. - [#102](https://github.com/thanos-io/objstore/pull/102) Azure: bump azblob sdk to get concurrency fixes. - [#33](https://github.com/thanos-io/objstore/pull/33) Tracing: Add `ContextWithTracer()` to inject the tracer into the context. From d44110ffb7e7c1155d3b01940d0a9adf724ff310 Mon Sep 17 00:00:00 2001 From: Charles Korn Date: Mon, 6 May 2024 17:21:46 +1000 Subject: [PATCH 093/130] Don't pass context to timingReader Signed-off-by: Charles Korn --- objstore.go | 10 ++-------- objstore_test.go | 10 +++++----- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/objstore.go b/objstore.go index a83f7d2e..31c167eb 100644 --- a/objstore.go +++ b/objstore.go @@ -551,7 +551,6 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err return nil, err } return newTimingReader( - ctx, rc, true, op, @@ -575,7 +574,6 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in return nil, err } return newTimingReader( - ctx, rc, true, op, @@ -608,7 +606,6 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.ops.WithLabelValues(op).Inc() trc := newTimingReader( - ctx, r, false, op, @@ -666,8 +663,6 @@ func (b *metricBucket) Name() string { type timingReader struct { io.Reader - ctx context.Context - // closeReader holds whether the wrapper io.Reader should be closed when // Close() is called on the timingReader. closeReader bool @@ -687,7 +682,7 @@ type timingReader struct { transferredBytes *prometheus.HistogramVec } -func newTimingReader(ctx context.Context, r io.Reader, closeReader bool, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { +func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -695,7 +690,6 @@ func newTimingReader(ctx context.Context, r io.Reader, closeReader bool, op stri trc := timingReader{ Reader: r, - ctx: ctx, closeReader: closeReader, objSize: objSize, objSizeErr: objSizeErr, @@ -762,7 +756,7 @@ func (r *timingReader) Read(b []byte) (n int, err error) { r.readBytes += int64(n) // Report metric just once. if !r.alreadyGotErr && err != nil && err != io.EOF { - if !r.isFailureExpected(err) && r.ctx.Err() != context.Canceled { + if !r.isFailureExpected(err) && !errors.Is(err, context.Canceled) { r.failed.WithLabelValues(r.op).Inc() } r.alreadyGotErr = true diff --git a/objstore_test.go b/objstore_test.go index 35a5d88a..7d56dfb3 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -412,7 +412,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { func TestTimingReader(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) - tr := newTimingReader(context.Background(), r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { + tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) @@ -447,7 +447,7 @@ func TestTimingReader_ExpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(context.Background(), r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.opsFetchedBytes, m.opsTransferredBytes) + tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.opsFetchedBytes, m.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -461,7 +461,7 @@ func TestTimingReader_UnexpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(context.Background(), r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) + tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -476,7 +476,7 @@ func TestTimingReader_ContextCancellation(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{ctx.Err()} - tr := newTimingReader(ctx, r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) + tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -506,7 +506,7 @@ func TestTimingReader_ShouldCorrectlyWrapFile(t *testing.T) { }) m := WrapWithMetrics(NewInMemBucket(), nil, "") - r := newTimingReader(context.Background(), file, true, "", m.opsDuration, m.opsFailures, func(err error) bool { + r := newTimingReader(file, true, "", m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) From 24f3dea9d4117d5db85365bfc7383c3016bb1402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20LOYET?= <822436+fatpat@users.noreply.github.com> Date: Fri, 9 Feb 2024 18:26:32 +0100 Subject: [PATCH 094/130] s3: add DisableMultipart option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jérôme LOYET <822436+fatpat@users.noreply.github.com> --- CHANGELOG.md | 1 + README.md | 1 + providers/s3/s3.go | 41 +++++++++++++++++++++++------------------ 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d0df778..97343dc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#86](https://github.com/thanos-io/objstore/pull/86) GCS: Add HTTP Config to GCS - [#99](https://github.com/thanos-io/objstore/pull/99) Swift: Add HTTP_Config - [#108](https://github.com/thanos-io/objstore/pull/108) Metrics: Add native histogram definitions to histograms +- [#100](https://github.com/thanos-io/objstore/pull/100) s3: add DisableMultipart option ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/README.md b/README.md index 6ebdffc1..949c2716 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,7 @@ config: list_objects_version: "" bucket_lookup_type: auto send_content_md5: true + disable_multipart: false part_size: 67108864 sse_config: type: "" diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 72130034..fd98bd85 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -101,6 +101,7 @@ const ( var DefaultConfig = Config{ PutUserMetadata: map[string]string{}, HTTPConfig: exthttp.DefaultHTTPConfig, + DisableMultipart: false, PartSize: 1024 * 1024 * 64, // 64MB. BucketLookupType: AutoLookup, SendContentMd5: true, // Default to using MD5. @@ -128,6 +129,7 @@ type Config struct { ListObjectsVersion string `yaml:"list_objects_version"` BucketLookupType BucketLookupType `yaml:"bucket_lookup_type"` SendContentMd5 bool `yaml:"send_content_md5"` + DisableMultipart bool `yaml:"disable_multipart"` // PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize. // NOTE we need to make sure this number does not produce more parts than 10 000. PartSize uint64 `yaml:"part_size"` @@ -150,15 +152,16 @@ type TraceConfig struct { // Bucket implements the store.Bucket interface against s3-compatible APIs. type Bucket struct { - logger log.Logger - name string - client *minio.Client - defaultSSE encrypt.ServerSide - putUserMetadata map[string]string - storageClass string - partSize uint64 - listObjectsV1 bool - sendContentMd5 bool + logger log.Logger + name string + client *minio.Client + defaultSSE encrypt.ServerSide + putUserMetadata map[string]string + storageClass string + disableMultipart bool + partSize uint64 + listObjectsV1 bool + sendContentMd5 bool } // parseConfig unmarshals a buffer into a Config with default values. @@ -319,15 +322,16 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B } bkt := &Bucket{ - logger: logger, - name: config.Bucket, - client: client, - defaultSSE: sse, - putUserMetadata: config.PutUserMetadata, - storageClass: storageClass, - partSize: config.PartSize, - listObjectsV1: config.ListObjectsVersion == "v1", - sendContentMd5: config.SendContentMd5, + logger: logger, + name: config.Bucket, + client: client, + defaultSSE: sse, + putUserMetadata: config.PutUserMetadata, + storageClass: storageClass, + disableMultipart: config.DisableMultipart, + partSize: config.PartSize, + listObjectsV1: config.ListObjectsVersion == "v1", + sendContentMd5: config.SendContentMd5, } return bkt, nil } @@ -500,6 +504,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { r, size, minio.PutObjectOptions{ + DisableMultipart: b.disableMultipart, PartSize: partSize, ServerSideEncryption: sse, UserMetadata: userMetadata, From 39f40b8d97f7f1bed8312fdc44258e213112f529 Mon Sep 17 00:00:00 2001 From: Vladimir Varankin Date: Thu, 13 Jun 2024 15:56:58 +0200 Subject: [PATCH 095/130] s3: add DisableDualstack to config (#112) * s3: add DisableDualstack to config Signed-off-by: Vladimir Varankin * ci: bump Go to 1.21 Signed-off-by: Vladimir Varankin --------- Signed-off-by: Vladimir Varankin Signed-off-by: Matej Gera <38492574+matej-g@users.noreply.github.com> Co-authored-by: Matej Gera <38492574+matej-g@users.noreply.github.com> --- .go-version | 2 +- CHANGELOG.md | 1 + README.md | 5 ++++- go.mod | 9 ++++----- go.sum | 27 +++++++++++++++++---------- providers/s3/s3.go | 6 ++++++ 6 files changed, 33 insertions(+), 17 deletions(-) diff --git a/.go-version b/.go-version index d8c94e3e..dbfae7a0 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.20.x +1.21.x diff --git a/CHANGELOG.md b/CHANGELOG.md index 97343dc0..3deec796 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#86](https://github.com/thanos-io/objstore/pull/86) GCS: Add HTTP Config to GCS - [#99](https://github.com/thanos-io/objstore/pull/99) Swift: Add HTTP_Config - [#108](https://github.com/thanos-io/objstore/pull/108) Metrics: Add native histogram definitions to histograms +- [#112](https://github.com/thanos-io/objstore/pull/112) S3: Add `DisableDualstack option. - [#100](https://github.com/thanos-io/objstore/pull/100) s3: add DisableMultipart option ### Changed diff --git a/README.md b/README.md index 949c2716..6d848e79 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ Thanos uses the [minio client](https://github.com/minio/minio-go) library to upl > NOTE: S3 client was designed for AWS S3, but it can be configured against other S3-compatible object storages e.g Ceph -The S# object storage yaml configuration definition: +The S3 object storage yaml configuration definition: ```yaml mdox-exec="go run scripts/cfggen/main.go --name=s3.Config" type: S3 @@ -153,6 +153,7 @@ config: bucket: "" endpoint: "" region: "" + disable_dualstack: false aws_sdk_auth: false access_key: "" insecure: false @@ -200,6 +201,8 @@ The field `prefix` can be used to transparently use prefixes in your S3 bucket. The AWS region to endpoint mapping can be found in this [link](https://docs.aws.amazon.com/general/latest/gr/s3.html). +By default, the library prefers using [dual-stack endpoints](https://docs.aws.amazon.com/AmazonS3/latest/userguide/dual-stack-endpoints.html). You can explicitly disable this behaviour by setting `disable_dualstack: true`. + Make sure you use a correct signature version. Currently AWS requires signature v4, so it needs `signature_version2: false`. If you don't specify it, you will get an `Access Denied` error. On the other hand, several S3 compatible APIs use `signature_version2: true`. You can configure the timeout settings for the HTTP client by setting the `http_config.idle_conn_timeout` and `http_config.response_header_timeout` keys. As a rule of thumb, if you are seeing errors like `timeout awaiting response headers` in your logs, you may want to increase the value of `http_config.response_header_timeout`. diff --git a/go.mod b/go.mod index 8887b088..e5c575bc 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/thanos-io/objstore -go 1.20 +go 1.21 require ( cloud.google.com/go/storage v1.40.0 @@ -14,7 +14,7 @@ require ( github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474 github.com/go-kit/log v0.2.1 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible - github.com/minio/minio-go/v7 v7.0.61 + github.com/minio/minio-go/v7 v7.0.69 github.com/ncw/swift v1.0.53 github.com/opentracing/opentracing-go v1.2.0 github.com/oracle/oci-go-sdk/v65 v65.41.1 @@ -74,8 +74,8 @@ require ( github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/md5-simd v1.1.2 // indirect @@ -89,7 +89,6 @@ require ( github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect diff --git a/go.sum b/go.sum index 8e01ea18..1ea067bf 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMiv github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 h1:IfFdxTUDiV58iZqPKgyWiz4X4fCxZeQ1pTQPImLYXpY= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= @@ -70,6 +71,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4 h1:rydBwnBoywKQMjWF0z8SriYtQ+uUcaFsxuijMjJr5PI= @@ -129,11 +131,13 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -152,12 +156,13 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -166,8 +171,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI= -github.com/minio/minio-go/v7 v7.0.61/go.mod h1:BTu8FcrEw+HidY0zd/0eny43QnVNkXRPXrLXFuQBHXg= +github.com/minio/minio-go/v7 v7.0.69 h1:l8AnsQFyY1xiwa/DaQskY4NXSLA2yrGsW5iD9nRPVS0= +github.com/minio/minio-go/v7 v7.0.69/go.mod h1:XAvOPJQ5Xlzk5o3o/ArO2NMbhSGkimC+bpW/ngRKDmQ= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= @@ -203,12 +208,11 @@ github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -218,11 +222,11 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= @@ -239,6 +243,7 @@ go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -281,7 +286,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -308,6 +312,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -347,6 +352,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -357,5 +363,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/providers/s3/s3.go b/providers/s3/s3.go index fd98bd85..dad89e66 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -117,6 +117,7 @@ type Config struct { Bucket string `yaml:"bucket"` Endpoint string `yaml:"endpoint"` Region string `yaml:"region"` + DisableDualstack bool `yaml:"disable_dualstack"` AWSSDKAuth bool `yaml:"aws_sdk_auth"` AccessKey string `yaml:"access_key"` Insecure bool `yaml:"insecure"` @@ -302,6 +303,11 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B } } + if config.DisableDualstack { + // The value in the config is inverted for backward compatibility + client.SetS3EnableDualstack(false) + } + if config.TraceConfig.Enable { logWriter := log.NewStdlibAdapter(level.Debug(logger), log.MessageKey("s3TraceMsg")) client.TraceOn(logWriter) From 124528d695c2f8690618d2149aac2566b9442cc3 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Mon, 17 Jun 2024 10:33:02 +0200 Subject: [PATCH 096/130] azure: Add new storage_create_container configuration property (#116) * azure: Add new storage_create_container configuration property In some cases account does not have permissions to read container properties or create one but has all permissions to do CRUD operations inside the container, e.g. SAS tokens. To solve such use case the new configuration property was added for the Azure object storage config which creates a new container explicitly by settings `storage_create_container` to `true`. To keep backward compatibility with existing object storage configurations the default value for it is always `true`. Signed-off-by: Andrey Pleskach * Add changelog Signed-off-by: Andrey Pleskach --------- Signed-off-by: Andrey Pleskach --- CHANGELOG.md | 1 + providers/azure/azure.go | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3deec796..9fa1b8a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#108](https://github.com/thanos-io/objstore/pull/108) Metrics: Add native histogram definitions to histograms - [#112](https://github.com/thanos-io/objstore/pull/112) S3: Add `DisableDualstack option. - [#100](https://github.com/thanos-io/objstore/pull/100) s3: add DisableMultipart option +- [#116](https://github.com/thanos-io/objstore/pull/116) Azure: Add new storage_create_container configuration property ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/providers/azure/azure.go b/providers/azure/azure.go index a72f3052..63bd2c1a 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -29,7 +29,8 @@ import ( // DefaultConfig for Azure objstore client. var DefaultConfig = Config{ - Endpoint: "blob.core.windows.net", + Endpoint: "blob.core.windows.net", + StorageCreateContainer: true, HTTPConfig: exthttp.HTTPConfig{ IdleConnTimeout: model.Duration(90 * time.Second), ResponseHeaderTimeout: model.Duration(2 * time.Minute), @@ -47,6 +48,7 @@ type Config struct { StorageAccountName string `yaml:"storage_account"` StorageAccountKey string `yaml:"storage_account_key"` StorageConnectionString string `yaml:"storage_connection_string"` + StorageCreateContainer bool `yaml:"storage_create_container"` ContainerName string `yaml:"container"` Endpoint string `yaml:"endpoint"` UserAssignedID string `yaml:"user_assigned_id"` @@ -167,19 +169,20 @@ func NewBucketWithConfig(logger log.Logger, conf Config, component string) (*Buc } // Check if storage account container already exists, and create one if it does not. - ctx := context.Background() - _, err = containerClient.GetProperties(ctx, &container.GetPropertiesOptions{}) - if err != nil { - if !bloberror.HasCode(err, bloberror.ContainerNotFound) { - return nil, err - } - _, err := containerClient.Create(ctx, nil) + if conf.StorageCreateContainer { + ctx := context.Background() + _, err = containerClient.GetProperties(ctx, &container.GetPropertiesOptions{}) if err != nil { - return nil, errors.Wrapf(err, "error creating Azure blob container: %s", conf.ContainerName) + if !bloberror.HasCode(err, bloberror.ContainerNotFound) { + return nil, err + } + _, err := containerClient.Create(ctx, nil) + if err != nil { + return nil, errors.Wrapf(err, "error creating Azure blob container: %s", conf.ContainerName) + } + level.Info(logger).Log("msg", "Azure blob container successfully created", "address", conf.ContainerName) } - level.Info(logger).Log("msg", "Azure blob container successfully created", "address", conf.ContainerName) } - bkt := &Bucket{ logger: logger, containerClient: containerClient, From 210469930d1d8164eeed12d0306edee79f77d254 Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Sat, 22 Jun 2024 11:45:05 +0200 Subject: [PATCH 097/130] *: upgrade minio-go version to `7.0.72` Signed-off-by: Michael Hoffmann --- CHANGELOG.md | 1 + go.mod | 21 +++++++++------------ go.sum | 44 ++++++++++++++++++-------------------------- 3 files changed, 28 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fa1b8a4..09123946 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,4 +57,5 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#70](https://github.com/thanos-io/objstore/pull/70) GCS: Update cloud.google.com/go/storage version to `v1.27.0`. - [#71](https://github.com/thanos-io/objstore/pull/71) Replace method `IsCustomerManagedKeyError` for a more generic `IsAccessDeniedErr` on the bucket interface. - [#89](https://github.com/thanos-io/objstore/pull/89) GCS: Upgrade cloud.google.com/go/storage version to `v1.35.1`. +- [#123](https://github.com/thanos-io/objstore/pull/123) *: Upgrade minio-go version to `v7.0.71`. ### Removed diff --git a/go.mod b/go.mod index e5c575bc..c76fa357 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474 github.com/go-kit/log v0.2.1 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible - github.com/minio/minio-go/v7 v7.0.69 + github.com/minio/minio-go/v7 v7.0.72 github.com/ncw/swift v1.0.53 github.com/opentracing/opentracing-go v1.2.0 github.com/oracle/oci-go-sdk/v65 v65.41.1 @@ -26,7 +26,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/atomic v1.9.0 golang.org/x/oauth2 v0.18.0 - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.7.0 google.golang.org/api v0.172.0 google.golang.org/grpc v1.62.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 @@ -62,6 +62,7 @@ require ( github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.3 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect @@ -73,16 +74,12 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/jpillora/backoff v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect @@ -94,9 +91,9 @@ require ( go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect @@ -112,5 +109,5 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.24.0 // indirect ) diff --git a/go.sum b/go.sum index 1ea067bf..f111af51 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,8 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= @@ -135,7 +137,6 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= @@ -154,13 +155,11 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= -github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -171,17 +170,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.69 h1:l8AnsQFyY1xiwa/DaQskY4NXSLA2yrGsW5iD9nRPVS0= -github.com/minio/minio-go/v7 v7.0.69/go.mod h1:XAvOPJQ5Xlzk5o3o/ArO2NMbhSGkimC+bpW/ngRKDmQ= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/minio/minio-go/v7 v7.0.72 h1:ZSbxs2BfJensLyHdVOgHv+pfmvxYraaUy07ER04dWnA= +github.com/minio/minio-go/v7 v7.0.72/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= @@ -251,8 +243,8 @@ go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -267,8 +259,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= @@ -277,8 +269,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -290,16 +282,16 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 3a628f97156f386b82f2fbbd7c419aabf8237a81 Mon Sep 17 00:00:00 2001 From: Kaviraj Kanagaraj Date: Fri, 19 Jul 2024 10:35:09 +0200 Subject: [PATCH 098/130] feat(gcs): Add support for `chunkSize` for writer This PR add support for custom `ChunkSize` when uploading object via GCS writer. https://pkg.go.dev/google.golang.org/cloud/storage#Writer Signed-off-by: Kaviraj Kanagaraj --- providers/gcs/gcs.go | 27 ++++++++++++++++++++------- providers/gcs/gcs_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index c4743630..a697e597 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -50,13 +50,19 @@ type Config struct { // on how to enable direct path. GRPCConnPoolSize int `yaml:"grpc_conn_pool_size"` HTTPConfig exthttp.HTTPConfig `yaml:"http_config"` + + // ChunkSizeBytes controls the maximum number of bytes of the object that the + // Writer will attempt to send to the server in a single request + // Used as storage.Writer.ChunkSize of https://pkg.go.dev/google.golang.org/cloud/storage#Writer + ChunkSizeBytes int `yaml:"chunk_size_bytes"` } // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. type Bucket struct { - logger log.Logger - bkt *storage.BucketHandle - name string + logger log.Logger + bkt *storage.BucketHandle + name string + chunkSize int closer io.Closer } @@ -160,10 +166,11 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option. return nil, err } bkt := &Bucket{ - logger: logger, - bkt: gcsClient.Bucket(gc.Bucket), - closer: gcsClient, - name: gc.Bucket, + logger: logger, + bkt: gcsClient.Bucket(gc.Bucket), + closer: gcsClient, + name: gc.Bucket, + chunkSize: gc.ChunkSizeBytes, } return bkt, nil } @@ -260,6 +267,12 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { w := b.bkt.Object(name).NewWriter(ctx) + // if `chunkSize` is 0, we don't set any custom value for writer's ChunkSize. + // It uses whatever the default value https://pkg.go.dev/google.golang.org/cloud/storage#Writer + if b.chunkSize > 0 { + w.ChunkSize = b.chunkSize + } + if _, err := io.Copy(w, r); err != nil { return err } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index c35b4f50..4c827c4b 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -73,6 +73,36 @@ func TestNewBucketWithConfig_ShouldCreateGRPC(t *testing.T) { testutil.Assert(t, bkt != nil, "expected bucket to be created") } +func TestParseConfig_ChunkSize(t *testing.T) { + for _, tc := range []struct { + name string + input string + assertions func(cfg Config) + }{ + { + name: "DefaultConfig", + input: `bucket: abcd`, + assertions: func(cfg Config) { + testutil.Equals(t, cfg.ChunkSizeBytes, 0) + }, + }, + { + name: "DefaultConfig", + input: `bucket: abcd +chunk_size_bytes: 1024`, + assertions: func(cfg Config) { + testutil.Equals(t, cfg.ChunkSizeBytes, 1024) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg, err := parseConfig([]byte(tc.input)) + testutil.Ok(t, err) + tc.assertions(cfg) + }) + } +} + func TestParseConfig_HTTPConfig(t *testing.T) { for _, tc := range []struct { name string From 8bfc5f66b89504150050ab53385557a9c0e78277 Mon Sep 17 00:00:00 2001 From: Kaviraj Kanagaraj Date: Fri, 19 Jul 2024 10:39:10 +0200 Subject: [PATCH 099/130] Add changelog Signed-off-by: Kaviraj Kanagaraj --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09123946..b5d7d12c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#112](https://github.com/thanos-io/objstore/pull/112) S3: Add `DisableDualstack option. - [#100](https://github.com/thanos-io/objstore/pull/100) s3: add DisableMultipart option - [#116](https://github.com/thanos-io/objstore/pull/116) Azure: Add new storage_create_container configuration property +- [#128](https://github.com/thanos-io/objstore/pull/128) GCS: Add support for `ChunkSize` for writer. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From 315f46869556311bba1542245ed1f3ca319d5ea1 Mon Sep 17 00:00:00 2001 From: Kaviraj Kanagaraj Date: Fri, 19 Jul 2024 10:43:51 +0200 Subject: [PATCH 100/130] change test name Signed-off-by: Kaviraj Kanagaraj --- providers/gcs/gcs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 4c827c4b..d03c5dd9 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -87,7 +87,7 @@ func TestParseConfig_ChunkSize(t *testing.T) { }, }, { - name: "DefaultConfig", + name: "CustomConfig", input: `bucket: abcd chunk_size_bytes: 1024`, assertions: func(cfg Config) { From 5e40a450bfcb96b05c1f09165080a33bad2868e1 Mon Sep 17 00:00:00 2001 From: Kaviraj Kanagaraj Date: Fri, 26 Jul 2024 18:10:31 +0200 Subject: [PATCH 101/130] feat: Decouple metrics from bucket Currently if a single process uses multiple object client, we get metrics duplicate registration failures. With this PR, 1. New `BucketMetrics` API return just a `Metrics` that can be created once and used for multiple clients. 2. New `Wrap` API that takes `metrics` and returns instrumented bucket just like `WrapWithMetrics` API. This PR doesn't change any existing API `WrapWithMetrics` Signed-off-by: Kaviraj Kanagaraj --- objstore.go | 164 ++++++++++++++++++++++++++----------------- objstore_test.go | 124 ++++++++++++++++---------------- providers/oci/oci.go | 1 + 3 files changed, 162 insertions(+), 127 deletions(-) diff --git a/objstore.go b/objstore.go index 31c167eb..8e9b2fd9 100644 --- a/objstore.go +++ b/objstore.go @@ -400,11 +400,8 @@ type IsOpFailureExpectedFunc func(error) bool var _ InstrumentedBucket = &metricBucket{} -// WrapWithMetrics takes a bucket and registers metrics with the given registry for -// operations run against the bucket. -func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBucket { - bkt := &metricBucket{ - bkt: b, +func BucketMetrics(reg prometheus.Registerer, name string) *Metrics { + return &Metrics{ isOpFailureExpected: func(err error) bool { return false }, ops: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Name: "objstore_bucket_operations_total", @@ -430,8 +427,8 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu ConstLabels: prometheus.Labels{"bucket": name}, Buckets: prometheus.ExponentialBuckets(2<<14, 2, 16), // 32KiB, 64KiB, ... 1GiB // Use factor=2 for native histograms, which gives similar buckets as the original exponential buckets. - NativeHistogramBucketFactor: 2, - NativeHistogramMaxBucketNumber: 100, + NativeHistogramBucketFactor: 2, + NativeHistogramMaxBucketNumber: 100, NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"operation"}), @@ -441,8 +438,8 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu ConstLabels: prometheus.Labels{"bucket": name}, Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, // Use the recommended defaults for native histograms with 10% growth factor. - NativeHistogramBucketFactor: 1.1, - NativeHistogramMaxBucketNumber: 100, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"operation"}), @@ -452,6 +449,27 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu ConstLabels: prometheus.Labels{"bucket": name}, }), } +} + +// WrapWithMetrics takes a bucket and registers metrics with the given registry for +// operations run against the bucket. +func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBucket { + metrics := BucketMetrics(reg, name) + return wrapWithMetrics(b, metrics) +} + +// WrapWith takes a `bucket` and `metrics` that returns instrumented bucket. +// Similar to WrapWithMetrics, but `metrics` can be passed separately as an argument. +func WrapWith(b Bucket, metrics *Metrics) *metricBucket { + return wrapWithMetrics(b, metrics) +} + +func wrapWithMetrics(b Bucket, metrics *Metrics) *metricBucket { + bkt := &metricBucket{ + bkt: b, + metrics: metrics, + } + for _, op := range []string{ OpIter, OpGet, @@ -461,10 +479,10 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu OpDelete, OpAttributes, } { - bkt.ops.WithLabelValues(op) - bkt.opsFailures.WithLabelValues(op) - bkt.opsDuration.WithLabelValues(op) - bkt.opsFetchedBytes.WithLabelValues(op) + bkt.metrics.ops.WithLabelValues(op) + bkt.metrics.opsFailures.WithLabelValues(op) + bkt.metrics.opsDuration.WithLabelValues(op) + bkt.metrics.opsFetchedBytes.WithLabelValues(op) } // fetched bytes only relevant for get, getrange and upload @@ -473,14 +491,12 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu OpGetRange, OpUpload, } { - bkt.opsTransferredBytes.WithLabelValues(op) + bkt.metrics.opsTransferredBytes.WithLabelValues(op) } return bkt } -type metricBucket struct { - bkt Bucket - +type Metrics struct { ops *prometheus.CounterVec opsFailures *prometheus.CounterVec isOpFailureExpected IsOpFailureExpectedFunc @@ -491,16 +507,34 @@ type metricBucket struct { lastSuccessfulUploadTime prometheus.Gauge } +func (m *Metrics) mustRegister(reg prometheus.Registerer) { + reg.MustRegister( + m.ops, + m.opsFailures, + m.opsFetchedBytes, + m.opsTransferredBytes, + m.opsDuration, + m.lastSuccessfulUploadTime, + ) +} + +type metricBucket struct { + bkt Bucket + metrics *Metrics +} + func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { return &metricBucket{ - bkt: b.bkt, - ops: b.ops, - opsFailures: b.opsFailures, - opsFetchedBytes: b.opsFetchedBytes, - opsTransferredBytes: b.opsTransferredBytes, - isOpFailureExpected: fn, - opsDuration: b.opsDuration, - lastSuccessfulUploadTime: b.lastSuccessfulUploadTime, + bkt: b.bkt, + metrics: &Metrics{ + ops: b.metrics.ops, + opsFailures: b.metrics.opsFailures, + opsFetchedBytes: b.metrics.opsFetchedBytes, + opsTransferredBytes: b.metrics.opsTransferredBytes, + isOpFailureExpected: fn, + opsDuration: b.metrics.opsDuration, + lastSuccessfulUploadTime: b.metrics.lastSuccessfulUploadTime, + }, } } @@ -510,43 +544,43 @@ func (b *metricBucket) ReaderWithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket func (b *metricBucket) Iter(ctx context.Context, dir string, f func(name string) error, options ...IterOption) error { const op = OpIter - b.ops.WithLabelValues(op).Inc() + b.metrics.ops.WithLabelValues(op).Inc() start := time.Now() err := b.bkt.Iter(ctx, dir, f, options...) if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { - b.opsFailures.WithLabelValues(op).Inc() + if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.metrics.opsFailures.WithLabelValues(op).Inc() } } - b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return err } func (b *metricBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) { const op = OpAttributes - b.ops.WithLabelValues(op).Inc() + b.metrics.ops.WithLabelValues(op).Inc() start := time.Now() attrs, err := b.bkt.Attributes(ctx, name) if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { - b.opsFailures.WithLabelValues(op).Inc() + if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.metrics.opsFailures.WithLabelValues(op).Inc() } return attrs, err } - b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return attrs, nil } func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { const op = OpGet - b.ops.WithLabelValues(op).Inc() + b.metrics.ops.WithLabelValues(op).Inc() rc, err := b.bkt.Get(ctx, name) if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { - b.opsFailures.WithLabelValues(op).Inc() + if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.metrics.opsFailures.WithLabelValues(op).Inc() } return nil, err } @@ -554,22 +588,22 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err rc, true, op, - b.opsDuration, - b.opsFailures, - b.isOpFailureExpected, - b.opsFetchedBytes, - b.opsTransferredBytes, + b.metrics.opsDuration, + b.metrics.opsFailures, + b.metrics.isOpFailureExpected, + b.metrics.opsFetchedBytes, + b.metrics.opsTransferredBytes, ), nil } func (b *metricBucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { const op = OpGetRange - b.ops.WithLabelValues(op).Inc() + b.metrics.ops.WithLabelValues(op).Inc() rc, err := b.bkt.GetRange(ctx, name, off, length) if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { - b.opsFailures.WithLabelValues(op).Inc() + if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.metrics.opsFailures.WithLabelValues(op).Inc() } return nil, err } @@ -577,69 +611,69 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in rc, true, op, - b.opsDuration, - b.opsFailures, - b.isOpFailureExpected, - b.opsFetchedBytes, - b.opsTransferredBytes, + b.metrics.opsDuration, + b.metrics.opsFailures, + b.metrics.isOpFailureExpected, + b.metrics.opsFetchedBytes, + b.metrics.opsTransferredBytes, ), nil } func (b *metricBucket) Exists(ctx context.Context, name string) (bool, error) { const op = OpExists - b.ops.WithLabelValues(op).Inc() + b.metrics.ops.WithLabelValues(op).Inc() start := time.Now() ok, err := b.bkt.Exists(ctx, name) if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { - b.opsFailures.WithLabelValues(op).Inc() + if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.metrics.opsFailures.WithLabelValues(op).Inc() } return false, err } - b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return ok, nil } func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) error { const op = OpUpload - b.ops.WithLabelValues(op).Inc() + b.metrics.ops.WithLabelValues(op).Inc() trc := newTimingReader( r, false, op, - b.opsDuration, - b.opsFailures, - b.isOpFailureExpected, + b.metrics.opsDuration, + b.metrics.opsFailures, + b.metrics.isOpFailureExpected, nil, - b.opsTransferredBytes, + b.metrics.opsTransferredBytes, ) defer trc.Close() err := b.bkt.Upload(ctx, name, trc) if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { - b.opsFailures.WithLabelValues(op).Inc() + if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.metrics.opsFailures.WithLabelValues(op).Inc() } return err } - b.lastSuccessfulUploadTime.SetToCurrentTime() + b.metrics.lastSuccessfulUploadTime.SetToCurrentTime() return nil } func (b *metricBucket) Delete(ctx context.Context, name string) error { const op = OpDelete - b.ops.WithLabelValues(op).Inc() + b.metrics.ops.WithLabelValues(op).Inc() start := time.Now() if err := b.bkt.Delete(ctx, name); err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { - b.opsFailures.WithLabelValues(op).Inc() + if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.metrics.opsFailures.WithLabelValues(op).Inc() } return err } - b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return nil } diff --git a/objstore_test.go b/objstore_test.go index 7d56dfb3..b62858fb 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -23,54 +23,54 @@ import ( func TestMetricBucket_Close(t *testing.T) { bkt := WrapWithMetrics(NewInMemBucket(), nil, "abc") // Expected initialized metrics. - testutil.Equals(t, 7, promtest.CollectAndCount(bkt.ops)) - testutil.Equals(t, 7, promtest.CollectAndCount(bkt.opsFailures)) - testutil.Equals(t, 7, promtest.CollectAndCount(bkt.opsDuration)) + testutil.Equals(t, 7, promtest.CollectAndCount(bkt.metrics.ops)) + testutil.Equals(t, 7, promtest.CollectAndCount(bkt.metrics.opsFailures)) + testutil.Equals(t, 7, promtest.CollectAndCount(bkt.metrics.opsDuration)) AcceptanceTest(t, bkt.WithExpectedErrs(bkt.IsObjNotFoundErr)) - testutil.Equals(t, float64(9), promtest.ToFloat64(bkt.ops.WithLabelValues(OpIter))) - testutil.Equals(t, float64(2), promtest.ToFloat64(bkt.ops.WithLabelValues(OpAttributes))) - testutil.Equals(t, float64(3), promtest.ToFloat64(bkt.ops.WithLabelValues(OpGet))) - testutil.Equals(t, float64(3), promtest.ToFloat64(bkt.ops.WithLabelValues(OpGetRange))) - testutil.Equals(t, float64(2), promtest.ToFloat64(bkt.ops.WithLabelValues(OpExists))) - testutil.Equals(t, float64(9), promtest.ToFloat64(bkt.ops.WithLabelValues(OpUpload))) - testutil.Equals(t, float64(3), promtest.ToFloat64(bkt.ops.WithLabelValues(OpDelete))) - testutil.Equals(t, 7, promtest.CollectAndCount(bkt.ops)) - testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpIter))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpAttributes))) - testutil.Equals(t, float64(1), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpGet))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpGetRange))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpExists))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpUpload))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpDelete))) - testutil.Equals(t, 7, promtest.CollectAndCount(bkt.opsFailures)) - testutil.Equals(t, 7, promtest.CollectAndCount(bkt.opsDuration)) - lastUpload := promtest.ToFloat64(bkt.lastSuccessfulUploadTime) + testutil.Equals(t, float64(9), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpIter))) + testutil.Equals(t, float64(2), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpAttributes))) + testutil.Equals(t, float64(3), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpGet))) + testutil.Equals(t, float64(3), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpGetRange))) + testutil.Equals(t, float64(2), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpExists))) + testutil.Equals(t, float64(9), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpUpload))) + testutil.Equals(t, float64(3), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpDelete))) + testutil.Equals(t, 7, promtest.CollectAndCount(bkt.metrics.ops)) + testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpIter))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpAttributes))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpGet))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpGetRange))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpExists))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpUpload))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpDelete))) + testutil.Equals(t, 7, promtest.CollectAndCount(bkt.metrics.opsFailures)) + testutil.Equals(t, 7, promtest.CollectAndCount(bkt.metrics.opsDuration)) + lastUpload := promtest.ToFloat64(bkt.metrics.lastSuccessfulUploadTime) testutil.Assert(t, lastUpload > 0, "last upload not greater than 0, val: %f", lastUpload) // Clear bucket, but don't clear metrics to ensure we use same. bkt.bkt = NewInMemBucket() AcceptanceTest(t, bkt) - testutil.Equals(t, float64(18), promtest.ToFloat64(bkt.ops.WithLabelValues(OpIter))) - testutil.Equals(t, float64(4), promtest.ToFloat64(bkt.ops.WithLabelValues(OpAttributes))) - testutil.Equals(t, float64(6), promtest.ToFloat64(bkt.ops.WithLabelValues(OpGet))) - testutil.Equals(t, float64(6), promtest.ToFloat64(bkt.ops.WithLabelValues(OpGetRange))) - testutil.Equals(t, float64(4), promtest.ToFloat64(bkt.ops.WithLabelValues(OpExists))) - testutil.Equals(t, float64(18), promtest.ToFloat64(bkt.ops.WithLabelValues(OpUpload))) - testutil.Equals(t, float64(6), promtest.ToFloat64(bkt.ops.WithLabelValues(OpDelete))) - testutil.Equals(t, 7, promtest.CollectAndCount(bkt.ops)) - testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpIter))) + testutil.Equals(t, float64(18), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpIter))) + testutil.Equals(t, float64(4), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpAttributes))) + testutil.Equals(t, float64(6), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpGet))) + testutil.Equals(t, float64(6), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpGetRange))) + testutil.Equals(t, float64(4), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpExists))) + testutil.Equals(t, float64(18), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpUpload))) + testutil.Equals(t, float64(6), promtest.ToFloat64(bkt.metrics.ops.WithLabelValues(OpDelete))) + testutil.Equals(t, 7, promtest.CollectAndCount(bkt.metrics.ops)) + testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpIter))) // Not expected not found error here. - testutil.Equals(t, float64(1), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpAttributes))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpAttributes))) // Not expected not found errors, this should increment failure metric on get for not found as well, so +2. - testutil.Equals(t, float64(3), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpGet))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpGetRange))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpExists))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpUpload))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.opsFailures.WithLabelValues(OpDelete))) - testutil.Equals(t, 7, promtest.CollectAndCount(bkt.opsFailures)) - testutil.Equals(t, 7, promtest.CollectAndCount(bkt.opsDuration)) - testutil.Assert(t, promtest.ToFloat64(bkt.lastSuccessfulUploadTime) > lastUpload) + testutil.Equals(t, float64(3), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpGet))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpGetRange))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpExists))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpUpload))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bkt.metrics.opsFailures.WithLabelValues(OpDelete))) + testutil.Equals(t, 7, promtest.CollectAndCount(bkt.metrics.opsFailures)) + testutil.Equals(t, 7, promtest.CollectAndCount(bkt.metrics.opsDuration)) + testutil.Assert(t, promtest.ToFloat64(bkt.metrics.lastSuccessfulUploadTime) > lastUpload) } // TestMetricBucket_MultipleClients tests that the metrics from two different buckets clients are not conflicting with each other. @@ -164,8 +164,8 @@ func TestMetricBucket_ReaderClose(t *testing.T) { testutil.Ok(t, reader.Close()) testutil.Assert(t, closeCalled) - testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpUpload))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpUpload))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.metrics.ops.WithLabelValues(OpUpload))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bucket.metrics.opsFailures.WithLabelValues(OpUpload))) }) t.Run("Get() should return a wrapper io.ReadCloser that correctly Close the wrapped one", func(t *testing.T) { @@ -194,8 +194,8 @@ func TestMetricBucket_ReaderClose(t *testing.T) { testutil.Ok(t, wrappedReader.Close()) testutil.Assert(t, closeCalled) - testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpGet))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpGet))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.metrics.ops.WithLabelValues(OpGet))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bucket.metrics.opsFailures.WithLabelValues(OpGet))) }) t.Run("GetRange() should return a wrapper io.ReadCloser that correctly Close the wrapped one", func(t *testing.T) { @@ -224,8 +224,8 @@ func TestMetricBucket_ReaderClose(t *testing.T) { testutil.Ok(t, wrappedReader.Close()) testutil.Assert(t, closeCalled) - testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpGetRange))) - testutil.Equals(t, float64(0), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpGetRange))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.metrics.ops.WithLabelValues(OpGetRange))) + testutil.Equals(t, float64(0), promtest.ToFloat64(bucket.metrics.opsFailures.WithLabelValues(OpGetRange))) }) } @@ -246,8 +246,8 @@ func TestMetricBucket_ReaderCloseError(t *testing.T) { testutil.NotOk(t, bucket.Upload(context.Background(), "test", origReader)) - testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpUpload))) - testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpUpload))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.metrics.ops.WithLabelValues(OpUpload))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.metrics.opsFailures.WithLabelValues(OpUpload))) }) t.Run("Get() should track failure if reader Close() returns error", func(t *testing.T) { @@ -262,8 +262,8 @@ func TestMetricBucket_ReaderCloseError(t *testing.T) { testutil.NotOk(t, reader.Close()) testutil.NotOk(t, reader.Close()) // Called twice to ensure metrics are not tracked twice. - testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpGet))) - testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpGet))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.metrics.ops.WithLabelValues(OpGet))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.metrics.opsFailures.WithLabelValues(OpGet))) }) t.Run("GetRange() should track failure if reader Close() returns error", func(t *testing.T) { @@ -278,8 +278,8 @@ func TestMetricBucket_ReaderCloseError(t *testing.T) { testutil.NotOk(t, reader.Close()) testutil.NotOk(t, reader.Close()) // Called twice to ensure metrics are not tracked twice. - testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.ops.WithLabelValues(OpGetRange))) - testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.opsFailures.WithLabelValues(OpGetRange))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.metrics.ops.WithLabelValues(OpGetRange))) + testutil.Equals(t, float64(1), promtest.ToFloat64(bucket.metrics.opsFailures.WithLabelValues(OpGetRange))) }) } @@ -412,9 +412,9 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { func TestTimingReader(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) - tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { + tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false - }, m.opsFetchedBytes, m.opsTransferredBytes) + }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) size, err := TryToGetSize(tr) @@ -439,7 +439,7 @@ func TestTimingReader(t *testing.T) { _, isReaderAt := tr.(io.ReaderAt) testutil.Assert(t, isReaderAt) - testutil.Equals(t, float64(0), promtest.ToFloat64(m.opsFailures.WithLabelValues(OpGet))) + testutil.Equals(t, float64(0), promtest.ToFloat64(m.metrics.opsFailures.WithLabelValues(OpGet))) } func TestTimingReader_ExpectedError(t *testing.T) { @@ -447,13 +447,13 @@ func TestTimingReader_ExpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.opsFetchedBytes, m.opsTransferredBytes) + tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) testutil.Equals(t, readerErr, err) - testutil.Equals(t, float64(0), promtest.ToFloat64(m.opsFailures.WithLabelValues(OpGet))) + testutil.Equals(t, float64(0), promtest.ToFloat64(m.metrics.opsFailures.WithLabelValues(OpGet))) } func TestTimingReader_UnexpectedError(t *testing.T) { @@ -461,13 +461,13 @@ func TestTimingReader_UnexpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) + tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) testutil.Equals(t, readerErr, err) - testutil.Equals(t, float64(1), promtest.ToFloat64(m.opsFailures.WithLabelValues(OpGet))) + testutil.Equals(t, float64(1), promtest.ToFloat64(m.metrics.opsFailures.WithLabelValues(OpGet))) } func TestTimingReader_ContextCancellation(t *testing.T) { @@ -476,13 +476,13 @@ func TestTimingReader_ContextCancellation(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{ctx.Err()} - tr := newTimingReader(r, true, OpGet, m.opsDuration, m.opsFailures, func(err error) bool { return false }, m.opsFetchedBytes, m.opsTransferredBytes) + tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) testutil.Equals(t, ctx.Err(), err) - testutil.Equals(t, float64(0), promtest.ToFloat64(m.opsFailures.WithLabelValues(OpGet))) + testutil.Equals(t, float64(0), promtest.ToFloat64(m.metrics.opsFailures.WithLabelValues(OpGet))) } type dummyReader struct { @@ -506,9 +506,9 @@ func TestTimingReader_ShouldCorrectlyWrapFile(t *testing.T) { }) m := WrapWithMetrics(NewInMemBucket(), nil, "") - r := newTimingReader(file, true, "", m.opsDuration, m.opsFailures, func(err error) bool { + r := newTimingReader(file, true, "", m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false - }, m.opsFetchedBytes, m.opsTransferredBytes) + }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) // It must both implement io.Seeker and io.ReaderAt. _, isSeeker := r.(io.Seeker) diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 2db35461..6c49fd3f 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -381,6 +381,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { } ctx := context.Background() + fmt.Println("Debug!!! bkt", bkt) bkt.name = objstore.CreateTemporaryTestBucketName(t) if err := bkt.createBucket(ctx, config.Compartment); err != nil { t.Errorf("failed to create temporary Oracle Cloud Infrastructure bucket '%s' for testing", bkt.name) From c99911b01ee5d7882bc02c7efc2c7fa5c70ecdf0 Mon Sep 17 00:00:00 2001 From: Kaviraj Kanagaraj Date: Mon, 29 Jul 2024 14:45:21 +0200 Subject: [PATCH 102/130] Add changelog Signed-off-by: Kaviraj Kanagaraj --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5d7d12c..6aae677a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#100](https://github.com/thanos-io/objstore/pull/100) s3: add DisableMultipart option - [#116](https://github.com/thanos-io/objstore/pull/116) Azure: Add new storage_create_container configuration property - [#128](https://github.com/thanos-io/objstore/pull/128) GCS: Add support for `ChunkSize` for writer. +- [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From 1c93b10484399f3d20fb18e09ffce058a60ddc9a Mon Sep 17 00:00:00 2001 From: Kaviraj Kanagaraj Date: Mon, 29 Jul 2024 15:14:47 +0200 Subject: [PATCH 103/130] remove `fmt.Println` debug accidentially added. Signed-off-by: Kaviraj Kanagaraj --- providers/oci/oci.go | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 6c49fd3f..2db35461 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -381,7 +381,6 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { } ctx := context.Background() - fmt.Println("Debug!!! bkt", bkt) bkt.name = objstore.CreateTemporaryTestBucketName(t) if err := bkt.createBucket(ctx, config.Compartment); err != nil { t.Errorf("failed to create temporary Oracle Cloud Infrastructure bucket '%s' for testing", bkt.name) From 699d378590ed94cfb0d8ea45c3c0755aad1c2afd Mon Sep 17 00:00:00 2001 From: Kaviraj Kanagaraj Date: Tue, 30 Jul 2024 08:55:41 +0200 Subject: [PATCH 104/130] remove unused `mustRegister` method Signed-off-by: Kaviraj Kanagaraj --- objstore.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/objstore.go b/objstore.go index 8e9b2fd9..87ec9e98 100644 --- a/objstore.go +++ b/objstore.go @@ -507,17 +507,6 @@ type Metrics struct { lastSuccessfulUploadTime prometheus.Gauge } -func (m *Metrics) mustRegister(reg prometheus.Registerer) { - reg.MustRegister( - m.ops, - m.opsFailures, - m.opsFetchedBytes, - m.opsTransferredBytes, - m.opsDuration, - m.lastSuccessfulUploadTime, - ) -} - type metricBucket struct { bkt Bucket metrics *Metrics From f06322381f09b1e031fc464eacf79b7b5106d9ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 28 Aug 2024 17:47:49 +0300 Subject: [PATCH 105/130] go.mod: update gRPC dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update gRPC dependency to a version that includes memory pooling. Remove old methods in `gcs` as pooling is always enabled now. Signed-off-by: Giedrius Statkevičius --- .github/workflows/docs.yaml | 2 +- go.mod | 16 +++++------ go.sum | 55 ++++++++++--------------------------- providers/gcs/gcs.go | 3 -- 4 files changed, 22 insertions(+), 54 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 6be30e4d..98f16bba 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -20,7 +20,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.20.x + go-version: 1.21.x - uses: actions/cache@v1 with: diff --git a/go.mod b/go.mod index c76fa357..4ec29639 100644 --- a/go.mod +++ b/go.mod @@ -25,18 +25,17 @@ require ( go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.18.0 + golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.7.0 google.golang.org/api v0.172.0 - google.golang.org/grpc v1.62.1 + google.golang.org/grpc v1.66.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( cloud.google.com/go v0.112.1 // indirect - cloud.google.com/go/compute v1.24.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.7 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect @@ -54,7 +53,7 @@ require ( github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bluele/gcache v0.0.2 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -95,11 +94,10 @@ require ( golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index f111af51..f2ecc7c5 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= -cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= -cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= @@ -60,8 +58,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -117,8 +115,6 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= @@ -130,7 +126,6 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -223,7 +218,6 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= @@ -242,54 +236,40 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -299,9 +279,6 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= @@ -309,24 +286,22 @@ google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -336,10 +311,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index a697e597..87e31f66 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -22,9 +22,7 @@ import ( "google.golang.org/api/iterator" "google.golang.org/api/option" htransport "google.golang.org/api/transport/http" - "google.golang.org/grpc" "google.golang.org/grpc/codes" - "google.golang.org/grpc/experimental" "google.golang.org/grpc/status" "gopkg.in/yaml.v2" @@ -155,7 +153,6 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option. ) if gc.UseGRPC { opts = append(opts, - option.WithGRPCDialOption(experimental.WithRecvBufferPool(grpc.NewSharedBufferPool())), option.WithGRPCConnectionPool(gc.GRPCConnPoolSize), ) gcsClient, err = storage.NewGRPCClient(ctx, opts...) From ef1de0fb0857cf8bf9d64fce5fa7efcedc70a1a8 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Tue, 11 Jun 2024 08:30:48 +0200 Subject: [PATCH 106/130] Add support for WriterTo in TimingReader This commit adds support for the WriterTo interface in the TimingReader so that it can be used efficiently in io.Copy. Signed-off-by: Filip Petkovski --- objstore.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/objstore.go b/objstore.go index 87ec9e98..924b2948 100644 --- a/objstore.go +++ b/objstore.go @@ -728,7 +728,7 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H _, isSeeker := r.(io.Seeker) _, isReaderAt := r.(io.ReaderAt) - + _, isWriterTo := r.(io.WriterTo) if isSeeker && isReaderAt { // The assumption is that in most cases when io.ReaderAt() is implemented then // io.Seeker is implemented too (e.g. os.File). @@ -737,6 +737,9 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H if isSeeker { return &timingReaderSeeker{timingReader: trc} } + if isWriterTo { + return &timingReaderWriterTo{timingReader: trc} + } return &trc } @@ -802,3 +805,11 @@ type timingReaderSeekerReaderAt struct { func (rsc *timingReaderSeekerReaderAt) ReadAt(p []byte, off int64) (int, error) { return (rsc.Reader).(io.ReaderAt).ReadAt(p, off) } + +type timingReaderWriterTo struct { + timingReader +} + +func (t *timingReaderWriterTo) WriteTo(w io.Writer) (n int64, err error) { + return (t.Reader).(io.WriterTo).WriteTo(w) +} From 46e7fe0322a083d4a520f2c80a8562794ef8380c Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Tue, 11 Jun 2024 09:44:21 +0200 Subject: [PATCH 107/130] Fix metrics Signed-off-by: Filip Petkovski --- objstore.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/objstore.go b/objstore.go index 924b2948..a9bffca7 100644 --- a/objstore.go +++ b/objstore.go @@ -728,7 +728,6 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H _, isSeeker := r.(io.Seeker) _, isReaderAt := r.(io.ReaderAt) - _, isWriterTo := r.(io.WriterTo) if isSeeker && isReaderAt { // The assumption is that in most cases when io.ReaderAt() is implemented then // io.Seeker is implemented too (e.g. os.File). @@ -737,7 +736,7 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H if isSeeker { return &timingReaderSeeker{timingReader: trc} } - if isWriterTo { + if _, isWriterTo := r.(io.WriterTo); isWriterTo { return &timingReaderWriterTo{timingReader: trc} } @@ -775,11 +774,16 @@ func (r *timingReader) Close() error { func (r *timingReader) Read(b []byte) (n int, err error) { n, err = r.Reader.Read(b) + r.updateMetrics(n, err) + return n, err +} + +func (r *timingReader) updateMetrics(n int, err error) { if r.fetchedBytes != nil { r.fetchedBytes.WithLabelValues(r.op).Add(float64(n)) } - r.readBytes += int64(n) + // Report metric just once. if !r.alreadyGotErr && err != nil && err != io.EOF { if !r.isFailureExpected(err) && !errors.Is(err, context.Canceled) { @@ -787,7 +791,6 @@ func (r *timingReader) Read(b []byte) (n int, err error) { } r.alreadyGotErr = true } - return n, err } type timingReaderSeeker struct { @@ -811,5 +814,7 @@ type timingReaderWriterTo struct { } func (t *timingReaderWriterTo) WriteTo(w io.Writer) (n int64, err error) { - return (t.Reader).(io.WriterTo).WriteTo(w) + n, err = (t.Reader).(io.WriterTo).WriteTo(w) + t.timingReader.updateMetrics(int(n), err) + return n, err } From 1a62b9e910bfa2477710e42df0374ad36f50bfdc Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Thu, 12 Sep 2024 10:06:32 +0200 Subject: [PATCH 108/130] fix: timing reader ignores initial get_range Some objstore providers like s3 do one initial read that already talks to object storage, this read might fail due to timeout, which is invisible in the current instrumentation since only subsequent reads are instrumented. This PR fixes this. Signed-off-by: Michael Hoffmann --- objstore.go | 15 +++++++++++++-- objstore_test.go | 11 ++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/objstore.go b/objstore.go index a9bffca7..bfee7503 100644 --- a/objstore.go +++ b/objstore.go @@ -566,14 +566,18 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err const op = OpGet b.metrics.ops.WithLabelValues(op).Inc() + start := time.Now() + rc, err := b.bkt.Get(ctx, name) if err != nil { if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } + b.metrics.opsDuration.WithLabelValues(op).Observe(float64(time.Since(start))) return nil, err } return newTimingReader( + start, rc, true, op, @@ -589,14 +593,18 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in const op = OpGetRange b.metrics.ops.WithLabelValues(op).Inc() + start := time.Now() + rc, err := b.bkt.GetRange(ctx, name, off, length) if err != nil { if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } + b.metrics.opsDuration.WithLabelValues(op).Observe(float64(time.Since(start))) return nil, err } return newTimingReader( + start, rc, true, op, @@ -628,7 +636,10 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err const op = OpUpload b.metrics.ops.WithLabelValues(op).Inc() + start := time.Now() + trc := newTimingReader( + start, r, false, op, @@ -705,7 +716,7 @@ type timingReader struct { transferredBytes *prometheus.HistogramVec } -func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { +func newTimingReader(start time.Time, r io.Reader, closeReader bool, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -716,7 +727,7 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H closeReader: closeReader, objSize: objSize, objSizeErr: objSizeErr, - start: time.Now(), + start: start, op: op, duration: dur, failed: failed, diff --git a/objstore_test.go b/objstore_test.go index b62858fb..51486d23 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/efficientgo/core/testutil" "github.com/go-kit/log" @@ -412,7 +413,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { func TestTimingReader(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) @@ -447,7 +448,7 @@ func TestTimingReader_ExpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -461,7 +462,7 @@ func TestTimingReader_UnexpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -476,7 +477,7 @@ func TestTimingReader_ContextCancellation(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{ctx.Err()} - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -506,7 +507,7 @@ func TestTimingReader_ShouldCorrectlyWrapFile(t *testing.T) { }) m := WrapWithMetrics(NewInMemBucket(), nil, "") - r := newTimingReader(file, true, "", m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { + r := newTimingReader(time.Now(), file, true, "", m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) From f735308ff69b039c35a24d342e13e402143bf5f6 Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Thu, 12 Sep 2024 20:16:07 +0200 Subject: [PATCH 109/130] s3: stat object instead of empty read Minio Client issues a HEAD request against the object which for some object storage providers might be easier then a GET with Range header set to zero bytes. Signed-off-by: Michael Hoffmann --- providers/s3/s3.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/providers/s3/s3.go b/providers/s3/s3.go index dad89e66..afd0982d 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -15,7 +15,6 @@ import ( "strings" "testing" - "github.com/efficientgo/core/logerrcapture" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/minio/minio-go/v7" @@ -438,21 +437,12 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } } - r, err := b.client.GetObject(ctx, b.name, name, *opts) - if err != nil { - return nil, err - } - // NotFoundObject error is revealed only after first Read. This does the initial GetRequest. Prefetch this here - // for convenience. - if _, err := r.Read(nil); err != nil { - defer logerrcapture.Do(b.logger, r.Close, "s3 get range obj close") - - // First GET Object request error. + // StatObject to see if the object exists and we have permissions to read it + if _, err := b.client.StatObject(ctx, b.name, name, *opts); err != nil { return nil, err } - - return r, nil + return b.client.GetObject(ctx, b.name, name, *opts) } // Get returns a reader for the given object name. From ba86bbae5a4a23a33f47717d53ad19460432c33a Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Fri, 13 Sep 2024 11:14:14 -0400 Subject: [PATCH 110/130] Update the GCS client to v1.43.0 The commit updates the GCS client to the latest released version. Signed-off-by: Filip Petkovski --- CHANGELOG.md | 1 + go.mod | 20 ++++++++++--------- go.sum | 56 ++++++++++++++++++++++++++++------------------------ 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aae677a..12375cbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan We use *breaking :warning:* to mark changes that are not backward compatible (relates only to v0.y.z releases.) ## Unreleased +- [#38](https://github.com/thanos-io/objstore/pull/38) GCS: Upgrade cloud.google.com/go/storage version to `v1.43.0`. ### Fixed - [#117](https://github.com/thanos-io/objstore/pull/117) Metrics: Fix `objstore_bucket_operation_failures_total` incorrectly incremented if context is cancelled while reading object contents. diff --git a/go.mod b/go.mod index 4ec29639..cb95d387 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/thanos-io/objstore go 1.21 require ( - cloud.google.com/go/storage v1.40.0 + cloud.google.com/go/storage v1.43.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go-v2 v1.16.0 github.com/aws/aws-sdk-go-v2/config v1.15.1 @@ -27,16 +27,18 @@ require ( go.uber.org/atomic v1.9.0 golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.7.0 - google.golang.org/api v0.172.0 + google.golang.org/api v0.187.0 google.golang.org/grpc v1.66.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.6.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect - cloud.google.com/go/iam v1.1.7 // indirect + cloud.google.com/go/iam v1.1.8 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect @@ -71,7 +73,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect @@ -94,10 +96,10 @@ require ( golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/protobuf v1.34.1 // indirect + google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index f2ecc7c5..0b5b860b 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,18 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= -cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= +cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= -cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= -cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= -cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= @@ -132,8 +138,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -142,8 +148,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= -github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -212,8 +218,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= @@ -228,8 +234,8 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= -go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -280,21 +286,19 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= -google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= +google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= +google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d h1:PksQg4dV6Sem3/HkBX+Ltq8T0ke0PKIRBNBatoDTVls= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -311,8 +315,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 03fd703d46f5cdfc2c31bcf02b09a93fad656615 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Fri, 13 Sep 2024 21:42:18 +0530 Subject: [PATCH 111/130] add roundtripper param in NewBucket Signed-off-by: milinddethe15 --- client/factory.go | 19 +++++++++-------- client/factory_test.go | 6 +++--- go.mod | 3 +++ go.sum | 2 ++ providers/azure/azure.go | 12 +++++++---- providers/azure/azure_test.go | 27 +++++++++++++++++++++++- providers/azure/helpers.go | 17 +++++++++++---- providers/cos/cos.go | 27 +++++++++++++++++------- providers/cos/cos_test.go | 31 ++++++++++++++++++++++++++++ providers/gcs/gcs.go | 12 ++++++----- providers/gcs/gcs_test.go | 25 +++++++++++++++++++++- providers/obs/obs.go | 28 ++++++++++++++++--------- providers/oci/oci.go | 26 ++++++++++++++--------- providers/oss/oss.go | 22 +++++++++++++------- providers/s3/s3.go | 20 ++++++++++-------- providers/s3/s3_e2e_test.go | 1 + providers/s3/s3_test.go | 39 ++++++++++++++++++++++++++++------- providers/swift/swift.go | 22 +++++++++++--------- providers/swift/swift_test.go | 24 +++++++++++++++++++++ 19 files changed, 275 insertions(+), 88 deletions(-) diff --git a/client/factory.go b/client/factory.go index 9e1adec9..cfecd67c 100644 --- a/client/factory.go +++ b/client/factory.go @@ -6,6 +6,7 @@ package client import ( "context" "fmt" + "net/http" "strings" "github.com/thanos-io/objstore" @@ -49,7 +50,7 @@ type BucketConfig struct { // NewBucket initializes and returns new object storage clients. // NOTE: confContentYaml can contain secrets. -func NewBucket(logger log.Logger, confContentYaml []byte, component string) (objstore.Bucket, error) { +func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt http.RoundTripper) (objstore.Bucket, error) { level.Info(logger).Log("msg", "loading bucket configuration") bucketConf := &BucketConfig{} if err := yaml.UnmarshalStrict(confContentYaml, bucketConf); err != nil { @@ -64,25 +65,25 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string) (obj var bucket objstore.Bucket switch strings.ToUpper(string(bucketConf.Type)) { case string(GCS): - bucket, err = gcs.NewBucket(context.Background(), logger, config, component) + bucket, err = gcs.NewBucket(context.Background(), logger, config, component, rt) case string(S3): - bucket, err = s3.NewBucket(logger, config, component) + bucket, err = s3.NewBucket(logger, config, component, rt) case string(AZURE): - bucket, err = azure.NewBucket(logger, config, component) + bucket, err = azure.NewBucket(logger, config, component, rt) case string(SWIFT): - bucket, err = swift.NewContainer(logger, config) + bucket, err = swift.NewContainer(logger, config, rt) case string(COS): - bucket, err = cos.NewBucket(logger, config, component) + bucket, err = cos.NewBucket(logger, config, component, rt) case string(ALIYUNOSS): - bucket, err = oss.NewBucket(logger, config, component) + bucket, err = oss.NewBucket(logger, config, component, rt) case string(FILESYSTEM): bucket, err = filesystem.NewBucketFromConfig(config) case string(BOS): bucket, err = bos.NewBucket(logger, config, component) case string(OCI): - bucket, err = oci.NewBucket(logger, config) + bucket, err = oci.NewBucket(logger, config, rt) case string(OBS): - bucket, err = obs.NewBucket(logger, config) + bucket, err = obs.NewBucket(logger, config, rt) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/client/factory_test.go b/client/factory_test.go index 4a9cf879..cd008add 100644 --- a/client/factory_test.go +++ b/client/factory_test.go @@ -23,7 +23,7 @@ func ExampleBucket() { } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } @@ -46,7 +46,7 @@ func ExampleTracingBucketUsingOpenTracing() { //nolint:govet } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } @@ -72,7 +72,7 @@ func ExampleTracingBucketUsingOpenTelemetry() { //nolint:govet } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } diff --git a/go.mod b/go.mod index 4ec29639..5a98916f 100644 --- a/go.mod +++ b/go.mod @@ -82,6 +82,7 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect @@ -99,6 +100,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -107,5 +109,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect + github.com/stretchr/testify v1.9.0 golang.org/x/crypto v0.24.0 // indirect ) diff --git a/go.sum b/go.sum index f2ecc7c5..f015ecc4 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 63bd2c1a..c2f3adb5 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -6,6 +6,7 @@ package azure import ( "context" "io" + "net/http" "os" "strings" "testing" @@ -145,7 +146,7 @@ type Bucket struct { } // NewBucket returns a new Bucket using the provided Azure config. -func NewBucket(logger log.Logger, azureConfig []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, azureConfig []byte, component string, rt http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new Azure bucket connection", "component", component) conf, err := parseConfig(azureConfig) if err != nil { @@ -154,11 +155,14 @@ func NewBucket(logger log.Logger, azureConfig []byte, component string) (*Bucket if conf.MSIResource != "" { level.Warn(logger).Log("msg", "The field msi_resource has been deprecated and should no longer be set") } - return NewBucketWithConfig(logger, conf, component) + return NewBucketWithConfig(logger, conf, component, rt) } // NewBucketWithConfig returns a new Bucket using the provided Azure config struct. -func NewBucketWithConfig(logger log.Logger, conf Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, conf Config, component string, rt http.RoundTripper) (*Bucket, error) { + if rt != nil { + conf.HTTPConfig.Transport = rt + } if err := conf.validate(); err != nil { return nil, err } @@ -355,7 +359,7 @@ func NewTestBucket(t testing.TB, component string) (objstore.Bucket, func(), err if err != nil { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), bc, component) + bkt, err := NewBucket(log.NewNopLogger(), bc, component, nil) if err != nil { t.Errorf("Cannot create Azure storage container:") return nil, nil, err diff --git a/providers/azure/azure_test.go b/providers/azure/azure_test.go index c49695ab..cc9c1317 100644 --- a/providers/azure/azure_test.go +++ b/providers/azure/azure_test.go @@ -4,10 +4,13 @@ package azure import ( + "net/http" "testing" "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/thanos-io/objstore/exthttp" ) @@ -20,7 +23,7 @@ type TestCase struct { } var validConfig = []byte(`storage_account: "myStorageAccount" -storage_account_key: "abc123" +storage_account_key: "bXlTdXBlclNlY3JldEtleTEyMyFAIw==" container: "MyContainer" endpoint: "blob.core.windows.net" reader_config: @@ -222,3 +225,25 @@ http_config: testutil.Ok(t, err) testutil.Equals(t, true, transport.TLSClientConfig.InsecureSkipVerify) } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + cfg, err := parseConfig(validConfig) + testutil.Ok(t, err) + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + _, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) + + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 846394a0..5ced695b 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -20,9 +20,17 @@ import ( const DirDelim = "/" func getContainerClient(conf Config) (*container.Client, error) { - dt, err := exthttp.DefaultTransport(conf.HTTPConfig) - if err != nil { - return nil, err + // Check if a roundtripper has been set in the config + // otherwise build the default transport. + var rt http.RoundTripper + if conf.HTTPConfig.Transport != nil { + rt = conf.HTTPConfig.Transport + } else { + var err error + rt, err = exthttp.DefaultTransport(conf.HTTPConfig) + if err != nil { + return nil, err + } } opt := &container.ClientOptions{ ClientOptions: azcore.ClientOptions{ @@ -35,7 +43,7 @@ func getContainerClient(conf Config) (*container.Client, error) { Telemetry: policy.TelemetryOptions{ ApplicationID: "Thanos", }, - Transport: &http.Client{Transport: dt}, + Transport: &http.Client{Transport: rt}, }, } @@ -65,6 +73,7 @@ func getContainerClient(conf Config) (*container.Client, error) { // Otherwise use a token credential var cred azcore.TokenCredential + var err error // Use Managed Identity Credential if a user assigned ID is set if conf.UserAssignedID != "" { diff --git a/providers/cos/cos.go b/providers/cos/cos.go index e518cae2..ce61ca87 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -95,7 +95,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided cos configuration. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { if logger == nil { logger = log.NewNopLogger() } @@ -104,12 +104,11 @@ func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, rt) } // NewBucketWithConfig returns a new Bucket using the provided cos config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate cos configuration") } @@ -128,7 +127,21 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B } } b := &cos.BaseURL{BucketURL: bucketURL} - tpt, _ := exthttp.DefaultTransport(config.HTTPConfig) + if rt != nil { + config.HTTPConfig.Transport = rt + } + // Check if a roundtripper has been set in the config + // otherwise build the default transport. + var tpt http.RoundTripper + if config.HTTPConfig.Transport != nil { + tpt = config.HTTPConfig.Transport + } else { + var err error + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, err + } + } client := cos.NewClient(b, &http.Client{ Transport: &cos.AuthorizationTransport{ SecretID: config.SecretId, @@ -485,7 +498,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } @@ -506,7 +519,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/cos/cos_test.go b/providers/cos/cos_test.go index 6d4316f2..c0f713b1 100644 --- a/providers/cos/cos_test.go +++ b/providers/cos/cos_test.go @@ -4,10 +4,14 @@ package cos import ( + "context" + "net/http" "testing" "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/thanos-io/objstore/exthttp" @@ -137,3 +141,30 @@ func TestConfig_validate(t *testing.T) { }) } } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + config := Config{ + Bucket: "bucket", + AppId: "123", + Region: "ap-beijing", + SecretId: "sid", + SecretKey: "skey", + } + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "Test") + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 87e31f66..e07ca05b 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -76,20 +76,22 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(ctx, logger, config, component) + return NewBucketWithConfig(ctx, logger, config, component, rt) } // NewBucketWithConfig returns a new Bucket with gcs Config struct. -func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string) (*Bucket, error) { +func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, rt http.RoundTripper) (*Bucket, error) { if gc.Bucket == "" { return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") } - + if rt != nil { + gc.HTTPConfig.Transport = rt + } var opts []option.ClientOption // If ServiceAccount is provided, use them in GCS client, otherwise fallback to Google default logic. @@ -312,7 +314,7 @@ func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error return nil, nil, err } - b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index d03c5dd9..76e2b99b 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -66,7 +66,7 @@ func TestNewBucketWithConfig_ShouldCreateGRPC(t *testing.T) { err = os.Setenv("STORAGE_EMULATOR_HOST_GRPC", svr.Addr) testutil.Ok(t, err) - bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket") + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", nil) testutil.Ok(t, err) // Check if the bucket is created. @@ -157,3 +157,26 @@ http_config: }) } } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +// func TestNewBucketWithErrorRoundTripper(t *testing.T) { +// cfg := Config{ +// Bucket: "test-bucket", +// ServiceAccount: "", +// } +// // Create an error RoundTripper +// rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + +// // Create the bucket with the custom RoundTripper +// _, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) +// testutil.NotOk(t, err) // Expect an error when using the error RoundTripper +// testutil.Assert(t, err.Error() == "RoundTripper error" || errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +// } diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 7bd9666b..dcd342d5 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -7,6 +7,7 @@ import ( "context" "io" "math" + "net/http" "os" "strings" "testing" @@ -74,13 +75,12 @@ type Bucket struct { name string } -func NewBucket(logger log.Logger, conf []byte) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - - return NewBucketWithConfig(logger, config) + return NewBucketWithConfig(logger, config, rt) } func parseConfig(conf []byte) (Config, error) { @@ -92,17 +92,25 @@ func parseConfig(conf []byte) (Config, error) { return config, nil } -func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, rt http.RoundTripper) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate obs config err") } - - rt, err := exthttp.DefaultTransport(config.HTTPConfig) - if err != nil { - return nil, errors.Wrap(err, "get http transport err") + if rt != nil { + config.HTTPConfig.Transport = rt + } + var tpt *http.Transport + if config.HTTPConfig.Transport != nil { + tpt = config.HTTPConfig.Transport.(*http.Transport) + } else { + var err error + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, errors.Wrap(err, "get http transport err") + } } - client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt)) + client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(tpt)) if err != nil { return nil, errors.Wrap(err, "initialize obs client err") } @@ -369,7 +377,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool, location if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc) + b, err := NewBucket(log.NewNopLogger(), bc, nil) if err != nil { return nil, nil, err } diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 2db35461..e8f8e0c5 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -58,13 +58,14 @@ type HTTPConfig struct { ResponseHeaderTimeout model.Duration `yaml:"response_header_timeout"` InsecureSkipVerify bool `yaml:"insecure_skip_verify"` - TLSHandshakeTimeout model.Duration `yaml:"tls_handshake_timeout"` - ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"` - MaxIdleConns int `yaml:"max_idle_conns"` - MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` - MaxConnsPerHost int `yaml:"max_conns_per_host"` - DisableCompression bool `yaml:"disable_compression"` - ClientTimeout time.Duration `yaml:"client_timeout"` + TLSHandshakeTimeout model.Duration `yaml:"tls_handshake_timeout"` + ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"` + MaxIdleConns int `yaml:"max_idle_conns"` + MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` + MaxConnsPerHost int `yaml:"max_conns_per_host"` + DisableCompression bool `yaml:"disable_compression"` + ClientTimeout time.Duration `yaml:"client_timeout"` + Transport http.RoundTripper `yaml:"-"` } // Config stores the configuration for oci bucket. @@ -288,7 +289,7 @@ func (b *Bucket) deleteBucket(ctx context.Context) (err error) { } // NewBucket returns a new Bucket using the provided oci config values. -func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { +func NewBucket(logger log.Logger, ociConfig []byte, rt http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new oci bucket connection") var config = DefaultConfig var configurationProvider common.ConfigurationProvider @@ -335,8 +336,13 @@ func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { return nil, errors.Wrapf(err, "unable to create ObjectStorage client with the given oci configurations") } + if rt != nil { + config.HTTPConfig.Transport = rt + } else { + config.HTTPConfig.Transport = CustomTransport(config) + } httpClient := http.Client{ - Transport: CustomTransport(config), + Transport: config.HTTPConfig.Transport, Timeout: config.HTTPConfig.ClientTimeout, } client.HTTPClient = &httpClient @@ -375,7 +381,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), ociConfig) + bkt, err := NewBucket(log.NewNopLogger(), ociConfig, nil) if err != nil { return nil, nil, err } diff --git a/providers/oss/oss.go b/providers/oss/oss.go index e7e3a648..572778d1 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -158,22 +158,30 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt } // NewBucket returns a new Bucket using the provided oss config values. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { var config Config if err := yaml.Unmarshal(conf, &config); err != nil { return nil, errors.Wrap(err, "parse aliyun oss config file failed") } - - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, rt) } // NewBucketWithConfig returns a new Bucket using the provided oss config struct. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { if err := validate(config); err != nil { return nil, err } - - client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) + var client *alioss.Client + var err error + if rt != nil { + // custom RoundTripper + clientOption := func(client *alioss.Client) { + client.HTTPClient = &http.Client{Transport: rt} + } + client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret, clientOption) + } else { + client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) + } if err != nil { return nil, errors.Wrap(err, "create aliyun oss client failed") } @@ -274,7 +282,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool) (objstore return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-aliyun-oss-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-aliyun-oss-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/s3/s3.go b/providers/s3/s3.go index dad89e66..5fc5f188 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -176,13 +176,13 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, rt) } type overrideSignerType struct { @@ -202,7 +202,7 @@ func (s *overrideSignerType) Retrieve() (credentials.Value, error) { } // NewBucketWithConfig returns a new Bucket using the provided s3 config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { var chain []credentials.Provider // TODO(bwplotka): Don't do flags as they won't scale, use actual params like v2, v4 instead @@ -242,15 +242,17 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B }), } } - + if rt != nil { + config.HTTPConfig.Transport = rt + } // Check if a roundtripper has been set in the config // otherwise build the default transport. - var rt http.RoundTripper + var tpt http.RoundTripper if config.HTTPConfig.Transport != nil { - rt = config.HTTPConfig.Transport + tpt = config.HTTPConfig.Transport } else { var err error - rt, err = exthttp.DefaultTransport(config.HTTPConfig) + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) if err != nil { return nil, err } @@ -260,7 +262,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B Creds: credentials.NewChainCredentials(chain), Secure: !config.Insecure, Region: config.Region, - Transport: rt, + Transport: tpt, BucketLookup: config.BucketLookupType.MinioType(), }) if err != nil { @@ -611,7 +613,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/s3/s3_e2e_test.go b/providers/s3/s3_e2e_test.go index 4b75a014..ac9ec261 100644 --- a/providers/s3/s3_e2e_test.go +++ b/providers/s3/s3_e2e_test.go @@ -37,6 +37,7 @@ func BenchmarkUpload(b *testing.B) { log.NewNopLogger(), e2ethanos.NewS3Config(bucket, m.Endpoint("https"), m.Dir()), "test-feed", + nil, ) testutil.Ok(b, err) diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index cdab39c3..049b330b 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "io" "net/http" "net/http/httptest" @@ -324,7 +325,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { // Default config should return no SSE config. cfg := DefaultConfig cfg.Endpoint = endpoint - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err := bkt.getServerSideEncryption(context.Background()) @@ -335,7 +336,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { cfg = DefaultConfig cfg.Endpoint = endpoint cfg.SSEConfig = SSEConfig{Type: SSES3} - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -351,7 +352,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { Type: SSEKMS, KMSKeyID: "key", } - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -375,7 +376,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { KMSKeyID: "key", KMSEncryptionContext: map[string]string{"foo": "bar"}, } - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -396,7 +397,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { override, err := encrypt.NewSSEKMS("test", nil) testutil.Ok(t, err) - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.WithValue(context.Background(), sseConfigKey, override)) @@ -423,7 +424,7 @@ func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) { cfg.AccessKey = "test" cfg.SecretKey = "test" - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) reader, err := bkt.Get(context.Background(), "test") @@ -448,7 +449,7 @@ func TestParseConfig_CustomStorageClass(t *testing.T) { cfg.Endpoint = endpoint storageClass := "STANDARD_IA" cfg.PutUserMetadata[testCase.storageClassKey] = storageClass - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) testutil.Equals(t, storageClass, bkt.storageClass) }) @@ -458,7 +459,29 @@ func TestParseConfig_CustomStorageClass(t *testing.T) { func TestParseConfig_DefaultStorageClassIsZero(t *testing.T) { cfg := DefaultConfig cfg.Endpoint = endpoint - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) testutil.Equals(t, "", bkt.storageClass) } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + cfg := DefaultConfig + cfg.Endpoint = endpoint + cfg.Bucket = "test" + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "test") + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/swift/swift.go b/providers/swift/swift.go index a8c56c55..f287f551 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -154,12 +154,12 @@ type Container struct { segmentsContainer string } -func NewContainer(logger log.Logger, conf []byte) (*Container, error) { +func NewContainer(logger log.Logger, conf []byte, rt http.RoundTripper) (*Container, error) { sc, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parse config") } - return NewContainerFromConfig(logger, sc, false) + return NewContainerFromConfig(logger, sc, false, rt) } func ensureContainer(connection *swift.Connection, name string, createIfNotExist bool) error { @@ -178,22 +178,24 @@ func ensureContainer(connection *swift.Connection, name string, createIfNotExist return nil } -func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool) (*Container, error) { - +func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, rt http.RoundTripper) (*Container, error) { + if rt != nil { + sc.HTTPConfig.Transport = rt + } // Check if a roundtripper has been set in the config // otherwise build the default transport. - var rt http.RoundTripper + var tpt http.RoundTripper if sc.HTTPConfig.Transport != nil { - rt = sc.HTTPConfig.Transport + tpt = sc.HTTPConfig.Transport } else { var err error - rt, err = exthttp.DefaultTransport(sc.HTTPConfig) + tpt, err = exthttp.DefaultTransport(sc.HTTPConfig) if err != nil { return nil, err } } - connection := connectionFromConfig(sc, rt) + connection := connectionFromConfig(sc, tpt) if err := connection.Authenticate(); err != nil { return nil, errors.Wrap(err, "authentication") } @@ -378,7 +380,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) { "needs to be manually cleared. This means that it is only useful to run one test in a time. This is due " + "to safety (accidentally pointing prod container for test) as well as swift not being fully strong consistent.") } - c, err := NewContainerFromConfig(log.NewNopLogger(), config, false) + c, err := NewContainerFromConfig(log.NewNopLogger(), config, false, nil) if err != nil { return nil, nil, errors.Wrap(err, "initializing new container") } @@ -392,7 +394,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) { } config.ContainerName = objstore.CreateTemporaryTestBucketName(t) config.SegmentContainerName = config.ContainerName - c, err := NewContainerFromConfig(log.NewNopLogger(), config, true) + c, err := NewContainerFromConfig(log.NewNopLogger(), config, true, nil) if err != nil { return nil, nil, errors.Wrap(err, "initializing new container") } diff --git a/providers/swift/swift_test.go b/providers/swift/swift_test.go index 656e7756..a203378c 100644 --- a/providers/swift/swift_test.go +++ b/providers/swift/swift_test.go @@ -4,10 +4,13 @@ package swift import ( + "net/http" "testing" "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/prometheus/common/model" ) @@ -64,3 +67,24 @@ http_config: testutil.Equals(t, false, cfg.HTTPConfig.InsecureSkipVerify) } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + logger := log.NewNopLogger() + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + config := DefaultConfig + config.AuthUrl = "http://identity.something.com/v3" + _, err := NewContainerFromConfig(logger, &config, false, rt) + + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} From 528b2c24299388f74c1a2c56f699fa39b8c95a23 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Sun, 15 Sep 2024 05:26:56 +0530 Subject: [PATCH 112/130] add tests for custom rt Signed-off-by: milinddethe15 --- go.mod | 4 +-- go.sum | 5 ++-- providers/cos/cos_test.go | 2 +- providers/gcs/gcs_test.go | 34 ++++++++++++++--------- providers/obs/obs_test.go | 33 +++++++++++++++++++++++ providers/oci/oci_test.go | 57 +++++++++++++++++++++++++++++++++++++++ providers/oss/oss_test.go | 39 +++++++++++++++++++++++++++ 7 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 providers/obs/obs_test.go create mode 100644 providers/oci/oci_test.go create mode 100644 providers/oss/oss_test.go diff --git a/go.mod b/go.mod index 5a98916f..1d10fb58 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,6 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect @@ -100,7 +99,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -109,6 +107,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/crypto v0.24.0 // indirect ) diff --git a/go.sum b/go.sum index f015ecc4..61b88553 100644 --- a/go.sum +++ b/go.sum @@ -204,16 +204,15 @@ github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= diff --git a/providers/cos/cos_test.go b/providers/cos/cos_test.go index c0f713b1..d67e2dfd 100644 --- a/providers/cos/cos_test.go +++ b/providers/cos/cos_test.go @@ -155,7 +155,7 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { config := Config{ Bucket: "bucket", AppId: "123", - Region: "ap-beijing", + Region: "test", SecretId: "sid", SecretKey: "skey", } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 76e2b99b..120e3450 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -15,6 +15,7 @@ import ( "github.com/efficientgo/core/testutil" "github.com/fullstorydev/emulators/storage/gcsemu" "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/prometheus/common/model" "google.golang.org/api/option" ) @@ -167,16 +168,23 @@ func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { return nil, ert.Err } -// func TestNewBucketWithErrorRoundTripper(t *testing.T) { -// cfg := Config{ -// Bucket: "test-bucket", -// ServiceAccount: "", -// } -// // Create an error RoundTripper -// rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} - -// // Create the bucket with the custom RoundTripper -// _, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) -// testutil.NotOk(t, err) // Expect an error when using the error RoundTripper -// testutil.Assert(t, err.Error() == "RoundTripper error" || errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) -// } +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + // Create an error RoundTripper + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + cfg := Config{ + Bucket: "test-bucket", + ServiceAccount: "", + UseGRPC: false, + } + svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) + testutil.Ok(t, err) + err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) + testutil.Ok(t, err) + err = os.Setenv("GCS_EMULATOR_HOST", svr.Addr) + testutil.Ok(t, err) + defer svr.Close() + + _, err = NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) + testutil.NotOk(t, err) // Expect an error when using the error RoundTripper + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/obs/obs_test.go b/providers/obs/obs_test.go new file mode 100644 index 00000000..26563b3c --- /dev/null +++ b/providers/obs/obs_test.go @@ -0,0 +1,33 @@ +package obs + +import ( + "net/http" + "testing" + + "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" +) + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error + Transport *http.Transport +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + config := DefaultConfig + config.Endpoint = "https://test.com" + config.SecretKey = "test123" + config.AccessKey = "test123" + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + _, err := NewBucketWithConfig(log.NewNopLogger(), config, rt) + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/oci/oci_test.go b/providers/oci/oci_test.go new file mode 100644 index 00000000..87288cee --- /dev/null +++ b/providers/oci/oci_test.go @@ -0,0 +1,57 @@ +package oci + +import ( + "net/http" + "testing" + + "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + + mockPrivateKey := `-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDCFENGw33yGihy92pDjZQhl0C36rPJj+CvfSC8+q28hxA161QF +NUd13wuCTUcq0Qd2qsBe/2hFyc2DCJJg0h1L78+6Z4UMR7EOcpfdUE9Hf3m/hs+F +UR45uBJeDK1HSFHD8bHKD6kv8FPGfJTotc+2xjJwoYi+1hqp1fIekaxsyQIDAQAB +AoGBAJR8ZkCUvx5kzv+utdl7T5MnordT1TvoXXJGXK7ZZ+UuvMNUCdN2QPc4sBiA +QWvLw1cSKt5DsKZ8UETpYPy8pPYnnDEz2dDYiaew9+xEpubyeW2oH4Zx71wqBtOK +kqwrXa/pzdpiucRRjk6vE6YY7EBBs/g7uanVpGibOVAEsqH1AkEA7DkjVH28WDUg +f1nqvfn2Kj6CT7nIcE3jGJsZZ7zlZmBmHFDONMLUrXR/Zm3pR5m0tCmBqa5RK95u +412jt1dPIwJBANJT3v8pnkth48bQo/fKel6uEYyboRtA5/uHuHkZ6FQF7OUkGogc +mSJluOdc5t6hI1VsLn0QZEjQZMEOWr+wKSMCQQCC4kXJEsHAve77oP6HtG/IiEn7 +kpyUXRNvFsDE0czpJJBvL/aRFUJxuRK91jhjC68sA7NsKMGg5OXb5I5Jj36xAkEA +gIT7aFOYBFwGgQAQkWNKLvySgKbAZRTeLBacpHMuQdl1DfdntvAyqpAZ0lY0RKmW +G6aFKaqQfOXKCyWoUiVknQJAXrlgySFci/2ueKlIE1QqIiLSZ8V8OlpFLRnb1pzI +7U1yQXnTAEFYM560yJlzUpOb1V4cScGd365tiSMvxLOvTA== +-----END RSA PRIVATE KEY-----` + + config := DefaultConfig + config.Provider = "raw" + config.Tenancy = "test" + config.User = "test" + config.Region = "test" + config.Fingerprint = "123" + config.PrivateKey = mockPrivateKey + config.Passphrase = "123" + ociConfig, err := yaml.Marshal(config) + testutil.Ok(t, err) + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + _, err = NewBucket(log.NewNopLogger(), ociConfig, rt) + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/oss/oss_test.go b/providers/oss/oss_test.go new file mode 100644 index 00000000..23f174d5 --- /dev/null +++ b/providers/oss/oss_test.go @@ -0,0 +1,39 @@ +package oss + +import ( + "context" + "net/http" + "testing" + + "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" +) + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + + var config Config + config.Endpoint = "http://test.com/" + config.AccessKeyID = "123" + config.AccessKeySecret = "123" + config.Bucket = "test" + config.AccessKeySecret = "123" + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) + // We expect an error from the RoundTripper + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "test") + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} From f2f0217f183f5d4a1db10a86cbb48919a61b5646 Mon Sep 17 00:00:00 2001 From: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> Date: Sun, 15 Sep 2024 00:26:25 +0530 Subject: [PATCH 113/130] fix lint Signed-off-by: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> --- providers/s3/s3_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index 049b330b..e34e19bb 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -7,7 +7,6 @@ import ( "context" "encoding/base64" "encoding/json" - "errors" "io" "net/http" "net/http/httptest" @@ -17,7 +16,8 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/minio/minio-go/v7/pkg/encrypt" - + "github.com/pkg/errors" + "github.com/thanos-io/objstore/exthttp" ) From 85504b96f6bbadeefda99d28cb00b07a423f3d09 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Tue, 1 Oct 2024 11:22:44 +0530 Subject: [PATCH 114/130] refactor code Signed-off-by: milinddethe15 --- client/factory.go | 2 +- errutil/rt_error.go | 12 ++++++++++++ providers/azure/azure_test.go | 13 ++----------- providers/azure/helpers.go | 11 ++++------- providers/bos/bos.go | 1 + providers/cos/cos.go | 19 ++++++------------- providers/cos/cos_test.go | 13 ++----------- providers/gcs/gcs.go | 10 ++++------ providers/gcs/gcs_test.go | 13 +++---------- providers/obs/obs.go | 28 ++++++++++------------------ providers/obs/obs_test.go | 33 --------------------------------- providers/oci/oci.go | 3 +-- providers/oci/oci_test.go | 13 ++----------- providers/oss/oss.go | 5 +---- providers/oss/oss_test.go | 13 ++----------- providers/s3/s3.go | 10 ++++------ providers/s3/s3_test.go | 14 +++----------- providers/swift/swift.go | 10 ++++------ providers/swift/swift_test.go | 13 ++----------- 19 files changed, 64 insertions(+), 172 deletions(-) create mode 100644 errutil/rt_error.go delete mode 100644 providers/obs/obs_test.go diff --git a/client/factory.go b/client/factory.go index cfecd67c..bd345024 100644 --- a/client/factory.go +++ b/client/factory.go @@ -83,7 +83,7 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt h case string(OCI): bucket, err = oci.NewBucket(logger, config, rt) case string(OBS): - bucket, err = obs.NewBucket(logger, config, rt) + bucket, err = obs.NewBucket(logger, config) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/errutil/rt_error.go b/errutil/rt_error.go new file mode 100644 index 00000000..ad1309e1 --- /dev/null +++ b/errutil/rt_error.go @@ -0,0 +1,12 @@ +package errutil + +import "net/http" + +// ErrorRoundTripper is a custom RoundTripper that always returns an error. +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} diff --git a/providers/azure/azure_test.go b/providers/azure/azure_test.go index cc9c1317..85533eaa 100644 --- a/providers/azure/azure_test.go +++ b/providers/azure/azure_test.go @@ -4,7 +4,6 @@ package azure import ( - "net/http" "testing" "time" @@ -12,6 +11,7 @@ import ( "github.com/go-kit/log" "github.com/pkg/errors" + "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" ) @@ -226,20 +226,11 @@ http_config: testutil.Equals(t, true, transport.TLSClientConfig.InsecureSkipVerify) } -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { cfg, err := parseConfig(validConfig) testutil.Ok(t, err) - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} _, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 5ced695b..1f9dde08 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -23,14 +23,12 @@ func getContainerClient(conf Config) (*container.Client, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. var rt http.RoundTripper + rt, err := exthttp.DefaultTransport(conf.HTTPConfig) + if err != nil { + return nil, err + } if conf.HTTPConfig.Transport != nil { rt = conf.HTTPConfig.Transport - } else { - var err error - rt, err = exthttp.DefaultTransport(conf.HTTPConfig) - if err != nil { - return nil, err - } } opt := &container.ClientOptions{ ClientOptions: azcore.ClientOptions{ @@ -73,7 +71,6 @@ func getContainerClient(conf Config) (*container.Client, error) { // Otherwise use a token credential var cred azcore.TokenCredential - var err error // Use Managed Identity Credential if a user assigned ID is set if conf.UserAssignedID != "" { diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 72e1b1e0..f88b7a21 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -66,6 +66,7 @@ func parseConfig(conf []byte) (Config, error) { // NewBucket new bos bucket. func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { + //TODO: Add support for custom roundtripper. if logger == nil { logger = log.NewNopLogger() } diff --git a/providers/cos/cos.go b/providers/cos/cos.go index ce61ca87..a8b853e3 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -127,20 +127,13 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt } } b := &cos.BaseURL{BucketURL: bucketURL} - if rt != nil { - config.HTTPConfig.Transport = rt - } - // Check if a roundtripper has been set in the config - // otherwise build the default transport. var tpt http.RoundTripper - if config.HTTPConfig.Transport != nil { - tpt = config.HTTPConfig.Transport - } else { - var err error - tpt, err = exthttp.DefaultTransport(config.HTTPConfig) - if err != nil { - return nil, err - } + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, err + } + if rt != nil { + tpt = rt } client := cos.NewClient(b, &http.Client{ Transport: &cos.AuthorizationTransport{ diff --git a/providers/cos/cos_test.go b/providers/cos/cos_test.go index d67e2dfd..f682aee9 100644 --- a/providers/cos/cos_test.go +++ b/providers/cos/cos_test.go @@ -5,7 +5,6 @@ package cos import ( "context" - "net/http" "testing" "time" @@ -14,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" ) @@ -142,15 +142,6 @@ func TestConfig_validate(t *testing.T) { } } -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { config := Config{ Bucket: "bucket", @@ -159,7 +150,7 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { SecretId: "sid", SecretKey: "skey", } - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) testutil.Ok(t, err) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index e07ca05b..4617e0b7 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -122,14 +122,12 @@ func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOp // Check if a roundtripper has been set in the config // otherwise build the default transport. var rt http.RoundTripper + rt, err := exthttp.DefaultTransport(gc.HTTPConfig) + if err != nil { + return nil, err + } if gc.HTTPConfig.Transport != nil { rt = gc.HTTPConfig.Transport - } else { - var err error - rt, err = exthttp.DefaultTransport(gc.HTTPConfig) - if err != nil { - return nil, err - } } // GCS uses some defaults when "options.WithHTTPClient" is not used that are important when we call diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 120e3450..66ea41cf 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -17,6 +17,7 @@ import ( "github.com/go-kit/log" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/thanos-io/objstore/errutil" "google.golang.org/api/option" ) @@ -159,18 +160,10 @@ http_config: } } -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { // Create an error RoundTripper - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} + cfg := Config{ Bucket: "test-bucket", ServiceAccount: "", diff --git a/providers/obs/obs.go b/providers/obs/obs.go index dcd342d5..eb6ca7ca 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -7,7 +7,6 @@ import ( "context" "io" "math" - "net/http" "os" "strings" "testing" @@ -75,12 +74,13 @@ type Bucket struct { name string } -func NewBucket(logger log.Logger, conf []byte, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte) (*Bucket, error) { + //TODO: Add support for custom roundtripper. config, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - return NewBucketWithConfig(logger, config, rt) + return NewBucketWithConfig(logger, config) } func parseConfig(conf []byte) (Config, error) { @@ -92,25 +92,17 @@ func parseConfig(conf []byte) (Config, error) { return config, nil } -func NewBucketWithConfig(logger log.Logger, config Config, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate obs config err") } - if rt != nil { - config.HTTPConfig.Transport = rt - } - var tpt *http.Transport - if config.HTTPConfig.Transport != nil { - tpt = config.HTTPConfig.Transport.(*http.Transport) - } else { - var err error - tpt, err = exthttp.DefaultTransport(config.HTTPConfig) - if err != nil { - return nil, errors.Wrap(err, "get http transport err") - } + + rt, err := exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, errors.Wrap(err, "get http transport err") } - client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(tpt)) + client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt)) if err != nil { return nil, errors.Wrap(err, "initialize obs client err") } @@ -377,7 +369,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool, location if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, nil) + b, err := NewBucket(log.NewNopLogger(), bc) if err != nil { return nil, nil, err } diff --git a/providers/obs/obs_test.go b/providers/obs/obs_test.go deleted file mode 100644 index 26563b3c..00000000 --- a/providers/obs/obs_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package obs - -import ( - "net/http" - "testing" - - "github.com/efficientgo/core/testutil" - "github.com/go-kit/log" - "github.com/pkg/errors" -) - -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error - Transport *http.Transport -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - -func TestNewBucketWithErrorRoundTripper(t *testing.T) { - config := DefaultConfig - config.Endpoint = "https://test.com" - config.SecretKey = "test123" - config.AccessKey = "test123" - - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} - _, err := NewBucketWithConfig(log.NewNopLogger(), config, rt) - // We expect an error from the RoundTripper - testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) -} diff --git a/providers/oci/oci.go b/providers/oci/oci.go index e8f8e0c5..e2f9e98b 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -336,10 +336,9 @@ func NewBucket(logger log.Logger, ociConfig []byte, rt http.RoundTripper) (*Buck return nil, errors.Wrapf(err, "unable to create ObjectStorage client with the given oci configurations") } + config.HTTPConfig.Transport = CustomTransport(config) if rt != nil { config.HTTPConfig.Transport = rt - } else { - config.HTTPConfig.Transport = CustomTransport(config) } httpClient := http.Client{ Transport: config.HTTPConfig.Transport, diff --git a/providers/oci/oci_test.go b/providers/oci/oci_test.go index 87288cee..7ea292fd 100644 --- a/providers/oci/oci_test.go +++ b/providers/oci/oci_test.go @@ -1,24 +1,15 @@ package oci import ( - "net/http" "testing" "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/pkg/errors" + "github.com/thanos-io/objstore/errutil" "gopkg.in/yaml.v2" ) -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { mockPrivateKey := `-----BEGIN RSA PRIVATE KEY----- @@ -48,7 +39,7 @@ G6aFKaqQfOXKCyWoUiVknQJAXrlgySFci/2ueKlIE1QqIiLSZ8V8OlpFLRnb1pzI ociConfig, err := yaml.Marshal(config) testutil.Ok(t, err) - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} _, err = NewBucket(log.NewNopLogger(), ociConfig, rt) // We expect an error from the RoundTripper diff --git a/providers/oss/oss.go b/providers/oss/oss.go index 572778d1..0bf5bd4b 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -171,16 +171,13 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt if err := validate(config); err != nil { return nil, err } - var client *alioss.Client - var err error + client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) if rt != nil { // custom RoundTripper clientOption := func(client *alioss.Client) { client.HTTPClient = &http.Client{Transport: rt} } client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret, clientOption) - } else { - client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) } if err != nil { return nil, errors.Wrap(err, "create aliyun oss client failed") diff --git a/providers/oss/oss_test.go b/providers/oss/oss_test.go index 23f174d5..f3380e71 100644 --- a/providers/oss/oss_test.go +++ b/providers/oss/oss_test.go @@ -2,23 +2,14 @@ package oss import ( "context" - "net/http" "testing" "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/pkg/errors" + "github.com/thanos-io/objstore/errutil" ) -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { var config Config @@ -28,7 +19,7 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { config.Bucket = "test" config.AccessKeySecret = "123" - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) // We expect an error from the RoundTripper diff --git a/providers/s3/s3.go b/providers/s3/s3.go index c85c0970..3d9ba213 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -247,14 +247,12 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt // Check if a roundtripper has been set in the config // otherwise build the default transport. var tpt http.RoundTripper + tpt, err := exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, err + } if config.HTTPConfig.Transport != nil { tpt = config.HTTPConfig.Transport - } else { - var err error - tpt, err = exthttp.DefaultTransport(config.HTTPConfig) - if err != nil { - return nil, err - } } client, err := minio.New(config.Endpoint, &minio.Options{ diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index e34e19bb..2a44f0e0 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -17,7 +17,8 @@ import ( "github.com/go-kit/log" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/pkg/errors" - + + "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" ) @@ -464,20 +465,11 @@ func TestParseConfig_DefaultStorageClassIsZero(t *testing.T) { testutil.Equals(t, "", bkt.storageClass) } -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { cfg := DefaultConfig cfg.Endpoint = endpoint cfg.Bucket = "test" - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) testutil.Ok(t, err) _, err = bkt.Get(context.Background(), "test") diff --git a/providers/swift/swift.go b/providers/swift/swift.go index f287f551..682c494c 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -185,14 +185,12 @@ func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, // Check if a roundtripper has been set in the config // otherwise build the default transport. var tpt http.RoundTripper + tpt, err := exthttp.DefaultTransport(sc.HTTPConfig) + if err != nil { + return nil, err + } if sc.HTTPConfig.Transport != nil { tpt = sc.HTTPConfig.Transport - } else { - var err error - tpt, err = exthttp.DefaultTransport(sc.HTTPConfig) - if err != nil { - return nil, err - } } connection := connectionFromConfig(sc, tpt) diff --git a/providers/swift/swift_test.go b/providers/swift/swift_test.go index a203378c..629dfd77 100644 --- a/providers/swift/swift_test.go +++ b/providers/swift/swift_test.go @@ -4,7 +4,6 @@ package swift import ( - "net/http" "testing" "time" @@ -12,6 +11,7 @@ import ( "github.com/go-kit/log" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/thanos-io/objstore/errutil" ) func TestParseConfig(t *testing.T) { @@ -68,18 +68,9 @@ http_config: } -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { logger := log.NewNopLogger() - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} config := DefaultConfig config.AuthUrl = "http://identity.something.com/v3" _, err := NewContainerFromConfig(logger, &config, false, rt) From 6eb7db97d6d7b1ba60f2d68ea271c8f3737d619d Mon Sep 17 00:00:00 2001 From: Suraj Patil <31805557+patilsuraj767@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:57:06 +0530 Subject: [PATCH 115/130] S3: exporting minio client (#143) Signed-off-by: Suraj Patil --- providers/s3/s3.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/providers/s3/s3.go b/providers/s3/s3.go index afd0982d..0cc36f69 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -154,7 +154,7 @@ type TraceConfig struct { type Bucket struct { logger log.Logger name string - client *minio.Client + Client *minio.Client defaultSSE encrypt.ServerSide putUserMetadata map[string]string storageClass string @@ -329,7 +329,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B bkt := &Bucket{ logger: logger, name: config.Bucket, - client: client, + Client: client, defaultSSE: sse, putUserMetadata: config.PutUserMetadata, storageClass: storageClass, @@ -400,7 +400,7 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt UseV1: b.listObjectsV1, } - for object := range b.client.ListObjects(ctx, b.name, opts) { + for object := range b.Client.ListObjects(ctx, b.name, opts) { // Catch the error when failed to list objects. if object.Err != nil { return object.Err @@ -439,10 +439,10 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( } // StatObject to see if the object exists and we have permissions to read it - if _, err := b.client.StatObject(ctx, b.name, name, *opts); err != nil { + if _, err := b.Client.StatObject(ctx, b.name, name, *opts); err != nil { return nil, err } - return b.client.GetObject(ctx, b.name, name, *opts) + return b.Client.GetObject(ctx, b.name, name, *opts) } // Get returns a reader for the given object name. @@ -457,7 +457,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( // Exists checks if the given object exists. func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { - _, err := b.client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) + _, err := b.Client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) if err != nil { if b.IsObjNotFoundErr(err) { return false, nil @@ -493,7 +493,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { userMetadata[k] = v } - if _, err := b.client.PutObject( + if _, err := b.Client.PutObject( ctx, b.name, name, @@ -520,7 +520,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { // Attributes returns information about the specified object. func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error) { - objInfo, err := b.client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) + objInfo, err := b.Client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) if err != nil { return objstore.ObjectAttributes{}, err } @@ -533,7 +533,7 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt // Delete removes the object with the given name. func (b *Bucket) Delete(ctx context.Context, name string) error { - return b.client.RemoveObject(ctx, b.name, name, minio.RemoveObjectOptions{}) + return b.Client.RemoveObject(ctx, b.name, name, minio.RemoveObjectOptions{}) } // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. @@ -622,7 +622,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke bktToCreate = objstore.CreateTemporaryTestBucketName(t) } - if err := b.client.MakeBucket(ctx, bktToCreate, minio.MakeBucketOptions{Region: location}); err != nil { + if err := b.Client.MakeBucket(ctx, bktToCreate, minio.MakeBucketOptions{Region: location}); err != nil { return nil, nil, err } b.name = bktToCreate @@ -630,7 +630,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke return b, func() { objstore.EmptyBucket(t, ctx, b) - if err := b.client.RemoveBucket(ctx, bktToCreate); err != nil { + if err := b.Client.RemoveBucket(ctx, bktToCreate); err != nil { t.Logf("deleting bucket %s failed: %s", bktToCreate, err) } }, nil From 075bf61a90bc7383aa90af2c5fda458af5e9fd30 Mon Sep 17 00:00:00 2001 From: Suraj Patil <31805557+patilsuraj767@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:47:38 +0530 Subject: [PATCH 116/130] Revert #143 (#144) Signed-off-by: Suraj Patil --- providers/s3/s3.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 0cc36f69..afd0982d 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -154,7 +154,7 @@ type TraceConfig struct { type Bucket struct { logger log.Logger name string - Client *minio.Client + client *minio.Client defaultSSE encrypt.ServerSide putUserMetadata map[string]string storageClass string @@ -329,7 +329,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B bkt := &Bucket{ logger: logger, name: config.Bucket, - Client: client, + client: client, defaultSSE: sse, putUserMetadata: config.PutUserMetadata, storageClass: storageClass, @@ -400,7 +400,7 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt UseV1: b.listObjectsV1, } - for object := range b.Client.ListObjects(ctx, b.name, opts) { + for object := range b.client.ListObjects(ctx, b.name, opts) { // Catch the error when failed to list objects. if object.Err != nil { return object.Err @@ -439,10 +439,10 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( } // StatObject to see if the object exists and we have permissions to read it - if _, err := b.Client.StatObject(ctx, b.name, name, *opts); err != nil { + if _, err := b.client.StatObject(ctx, b.name, name, *opts); err != nil { return nil, err } - return b.Client.GetObject(ctx, b.name, name, *opts) + return b.client.GetObject(ctx, b.name, name, *opts) } // Get returns a reader for the given object name. @@ -457,7 +457,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( // Exists checks if the given object exists. func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { - _, err := b.Client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) + _, err := b.client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) if err != nil { if b.IsObjNotFoundErr(err) { return false, nil @@ -493,7 +493,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { userMetadata[k] = v } - if _, err := b.Client.PutObject( + if _, err := b.client.PutObject( ctx, b.name, name, @@ -520,7 +520,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { // Attributes returns information about the specified object. func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error) { - objInfo, err := b.Client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) + objInfo, err := b.client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) if err != nil { return objstore.ObjectAttributes{}, err } @@ -533,7 +533,7 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt // Delete removes the object with the given name. func (b *Bucket) Delete(ctx context.Context, name string) error { - return b.Client.RemoveObject(ctx, b.name, name, minio.RemoveObjectOptions{}) + return b.client.RemoveObject(ctx, b.name, name, minio.RemoveObjectOptions{}) } // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. @@ -622,7 +622,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke bktToCreate = objstore.CreateTemporaryTestBucketName(t) } - if err := b.Client.MakeBucket(ctx, bktToCreate, minio.MakeBucketOptions{Region: location}); err != nil { + if err := b.client.MakeBucket(ctx, bktToCreate, minio.MakeBucketOptions{Region: location}); err != nil { return nil, nil, err } b.name = bktToCreate @@ -630,7 +630,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke return b, func() { objstore.EmptyBucket(t, ctx, b) - if err := b.Client.RemoveBucket(ctx, bktToCreate); err != nil { + if err := b.client.RemoveBucket(ctx, bktToCreate); err != nil { t.Logf("deleting bucket %s failed: %s", bktToCreate, err) } }, nil From b469d7451d6d9b7b3a0df9dde1f48e02dc712079 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Wed, 2 Oct 2024 00:36:38 +0530 Subject: [PATCH 117/130] gcs error-rt test Signed-off-by: milinddethe15 --- go.mod | 2 +- providers/gcs/fakecred.json | 9 +++++++++ providers/gcs/gcs_test.go | 18 +++++++++--------- 3 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 providers/gcs/fakecred.json diff --git a/go.mod b/go.mod index af1cc40e..db189c20 100644 --- a/go.mod +++ b/go.mod @@ -88,6 +88,7 @@ require ( github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect @@ -109,6 +110,5 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - github.com/stretchr/testify v1.9.0 // indirect golang.org/x/crypto v0.24.0 // indirect ) diff --git a/providers/gcs/fakecred.json b/providers/gcs/fakecred.json new file mode 100644 index 00000000..fe130026 --- /dev/null +++ b/providers/gcs/fakecred.json @@ -0,0 +1,9 @@ +{ + "type": "service_account", + "private_key_id": "abc", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDY3E8o1NEFcjMM\nHW/5ZfFJw29/8NEqpViNjQIx95Xx5KDtJ+nWn9+OW0uqsSqKlKGhAdAo+Q6bjx2c\nuXVsXTu7XrZUY5Kltvj94DvUa1wjNXs606r/RxWTJ58bfdC+gLLxBfGnB6CwK0YQ\nxnfpjNbkUfVVzO0MQD7UP0Hl5ZcY0Puvxd/yHuONQn/rIAieTHH1pqgW+zrH/y3c\n59IGThC9PPtugI9ea8RSnVj3PWz1bX2UkCDpy9IRh9LzJLaYYX9RUd7++dULUlat\nAaXBh1U6emUDzhrIsgApjDVtimOPbmQWmX1S60mqQikRpVYZ8u+NDD+LNw+/Eovn\nxCj2Y3z1AgMBAAECggEAWDBzoqO1IvVXjBA2lqId10T6hXmN3j1ifyH+aAqK+FVl\nGjyWjDj0xWQcJ9ync7bQ6fSeTeNGzP0M6kzDU1+w6FgyZqwdmXWI2VmEizRjwk+/\n/uLQUcL7I55Dxn7KUoZs/rZPmQDxmGLoue60Gg6z3yLzVcKiDc7cnhzhdBgDc8vd\nQorNAlqGPRnm3EqKQ6VQp6fyQmCAxrr45kspRXNLddat3AMsuqImDkqGKBmF3Q1y\nxWGe81LphUiRqvqbyUlh6cdSZ8pLBpc9m0c3qWPKs9paqBIvgUPlvOZMqec6x4S6\nChbdkkTRLnbsRr0Yg/nDeEPlkhRBhasXpxpMUBgPywKBgQDs2axNkFjbU94uXvd5\nznUhDVxPFBuxyUHtsJNqW4p/ujLNimGet5E/YthCnQeC2P3Ym7c3fiz68amM6hiA\nOnW7HYPZ+jKFnefpAtjyOOs46AkftEg07T9XjwWNPt8+8l0DYawPoJgbM5iE0L2O\nx8TU1Vs4mXc+ql9F90GzI0x3VwKBgQDqZOOqWw3hTnNT07Ixqnmd3dugV9S7eW6o\nU9OoUgJB4rYTpG+yFqNqbRT8bkx37iKBMEReppqonOqGm4wtuRR6LSLlgcIU9Iwx\nyfH12UWqVmFSHsgZFqM/cK3wGev38h1WBIOx3/djKn7BdlKVh8kWyx6uC8bmV+E6\nOoK0vJD6kwKBgHAySOnROBZlqzkiKW8c+uU2VATtzJSydrWm0J4wUPJifNBa/hVW\ndcqmAzXC9xznt5AVa3wxHBOfyKaE+ig8CSsjNyNZ3vbmr0X04FoV1m91k2TeXNod\njMTobkPThaNm4eLJMN2SQJuaHGTGERWC0l3T18t+/zrDMDCPiSLX1NAvAoGBAN1T\nVLJYdjvIMxf1bm59VYcepbK7HLHFkRq6xMJMZbtG0ryraZjUzYvB4q4VjHk2UDiC\nlhx13tXWDZH7MJtABzjyg+AI7XWSEQs2cBXACos0M4Myc6lU+eL+iA+OuoUOhmrh\nqmT8YYGu76/IBWUSqWuvcpHPpwl7871i4Ga/I3qnAoGBANNkKAcMoeAbJQK7a/Rn\nwPEJB+dPgNDIaboAsh1nZhVhN5cvdvCWuEYgOGCPQLYQF0zmTLcM+sVxOYgfy8mV\nfbNgPgsP5xmu6dw2COBKdtozw0HrWSRjACd1N4yGu75+wPCcX/gQarcjRcXXZeEa\nNtBLSfcqPULqD+h7br9lEJio\n-----END PRIVATE KEY-----\n", + "client_email": "123-abc@developer.gserviceaccount.com", + "client_id": "123-abc.apps.googleusercontent.com", + "auth_uri": "http://127.0.0.1:43649/", + "token_uri": "http://127.0.0.1:43649/" + } \ No newline at end of file diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 66ea41cf..f627ff04 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -161,23 +161,23 @@ http_config: } func TestNewBucketWithErrorRoundTripper(t *testing.T) { - // Create an error RoundTripper + // Create an error RoundTripper. rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - cfg := Config{ Bucket: "test-bucket", ServiceAccount: "", UseGRPC: false, } - svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) - testutil.Ok(t, err) - err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) + svr, err := gcsemu.NewServer("127.0.0.1:43649", gcsemu.Options{}) testutil.Ok(t, err) + defer svr.Close() err = os.Setenv("GCS_EMULATOR_HOST", svr.Addr) testutil.Ok(t, err) - defer svr.Close() - - _, err = NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) - testutil.NotOk(t, err) // Expect an error when using the error RoundTripper + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "fakecred.json") + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) + testutil.Ok(t, err) + testutil.Assert(t, bkt.name == "test-bucket") + _, err = bkt.Get(context.Background(), "test-bucket") + testutil.NotOk(t, err) testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) } From baca2f75dab03a338759c9ddb3b1b0ab7a92b0a4 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Wed, 2 Oct 2024 15:39:49 +0530 Subject: [PATCH 118/130] added error-rt gcs test Signed-off-by: milinddethe15 --- providers/gcs/fakecred.json | 9 --------- providers/gcs/gcs.go | 7 +++++-- providers/gcs/gcs_test.go | 8 ++++---- 3 files changed, 9 insertions(+), 15 deletions(-) delete mode 100644 providers/gcs/fakecred.json diff --git a/providers/gcs/fakecred.json b/providers/gcs/fakecred.json deleted file mode 100644 index fe130026..00000000 --- a/providers/gcs/fakecred.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": "service_account", - "private_key_id": "abc", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDY3E8o1NEFcjMM\nHW/5ZfFJw29/8NEqpViNjQIx95Xx5KDtJ+nWn9+OW0uqsSqKlKGhAdAo+Q6bjx2c\nuXVsXTu7XrZUY5Kltvj94DvUa1wjNXs606r/RxWTJ58bfdC+gLLxBfGnB6CwK0YQ\nxnfpjNbkUfVVzO0MQD7UP0Hl5ZcY0Puvxd/yHuONQn/rIAieTHH1pqgW+zrH/y3c\n59IGThC9PPtugI9ea8RSnVj3PWz1bX2UkCDpy9IRh9LzJLaYYX9RUd7++dULUlat\nAaXBh1U6emUDzhrIsgApjDVtimOPbmQWmX1S60mqQikRpVYZ8u+NDD+LNw+/Eovn\nxCj2Y3z1AgMBAAECggEAWDBzoqO1IvVXjBA2lqId10T6hXmN3j1ifyH+aAqK+FVl\nGjyWjDj0xWQcJ9ync7bQ6fSeTeNGzP0M6kzDU1+w6FgyZqwdmXWI2VmEizRjwk+/\n/uLQUcL7I55Dxn7KUoZs/rZPmQDxmGLoue60Gg6z3yLzVcKiDc7cnhzhdBgDc8vd\nQorNAlqGPRnm3EqKQ6VQp6fyQmCAxrr45kspRXNLddat3AMsuqImDkqGKBmF3Q1y\nxWGe81LphUiRqvqbyUlh6cdSZ8pLBpc9m0c3qWPKs9paqBIvgUPlvOZMqec6x4S6\nChbdkkTRLnbsRr0Yg/nDeEPlkhRBhasXpxpMUBgPywKBgQDs2axNkFjbU94uXvd5\nznUhDVxPFBuxyUHtsJNqW4p/ujLNimGet5E/YthCnQeC2P3Ym7c3fiz68amM6hiA\nOnW7HYPZ+jKFnefpAtjyOOs46AkftEg07T9XjwWNPt8+8l0DYawPoJgbM5iE0L2O\nx8TU1Vs4mXc+ql9F90GzI0x3VwKBgQDqZOOqWw3hTnNT07Ixqnmd3dugV9S7eW6o\nU9OoUgJB4rYTpG+yFqNqbRT8bkx37iKBMEReppqonOqGm4wtuRR6LSLlgcIU9Iwx\nyfH12UWqVmFSHsgZFqM/cK3wGev38h1WBIOx3/djKn7BdlKVh8kWyx6uC8bmV+E6\nOoK0vJD6kwKBgHAySOnROBZlqzkiKW8c+uU2VATtzJSydrWm0J4wUPJifNBa/hVW\ndcqmAzXC9xznt5AVa3wxHBOfyKaE+ig8CSsjNyNZ3vbmr0X04FoV1m91k2TeXNod\njMTobkPThaNm4eLJMN2SQJuaHGTGERWC0l3T18t+/zrDMDCPiSLX1NAvAoGBAN1T\nVLJYdjvIMxf1bm59VYcepbK7HLHFkRq6xMJMZbtG0ryraZjUzYvB4q4VjHk2UDiC\nlhx13tXWDZH7MJtABzjyg+AI7XWSEQs2cBXACos0M4Myc6lU+eL+iA+OuoUOhmrh\nqmT8YYGu76/IBWUSqWuvcpHPpwl7871i4Ga/I3qnAoGBANNkKAcMoeAbJQK7a/Rn\nwPEJB+dPgNDIaboAsh1nZhVhN5cvdvCWuEYgOGCPQLYQF0zmTLcM+sVxOYgfy8mV\nfbNgPgsP5xmu6dw2COBKdtozw0HrWSRjACd1N4yGu75+wPCcX/gQarcjRcXXZeEa\nNtBLSfcqPULqD+h7br9lEJio\n-----END PRIVATE KEY-----\n", - "client_email": "123-abc@developer.gserviceaccount.com", - "client_id": "123-abc.apps.googleusercontent.com", - "auth_uri": "http://127.0.0.1:43649/", - "token_uri": "http://127.0.0.1:43649/" - } \ No newline at end of file diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 4617e0b7..b5dae200 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -52,7 +52,8 @@ type Config struct { // ChunkSizeBytes controls the maximum number of bytes of the object that the // Writer will attempt to send to the server in a single request // Used as storage.Writer.ChunkSize of https://pkg.go.dev/google.golang.org/cloud/storage#Writer - ChunkSizeBytes int `yaml:"chunk_size_bytes"` + ChunkSizeBytes int `yaml:"chunk_size_bytes"` + noAuth bool `yaml:"no_auth"` } // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. @@ -102,7 +103,9 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp } opts = append(opts, option.WithCredentials(credentials)) } - + if gc.noAuth { + opts = append(opts, option.WithoutAuthentication()) + } opts = append(opts, option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())), ) diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index f627ff04..a06b3a16 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -167,16 +167,16 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { Bucket: "test-bucket", ServiceAccount: "", UseGRPC: false, + noAuth: true, } - svr, err := gcsemu.NewServer("127.0.0.1:43649", gcsemu.Options{}) + svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) testutil.Ok(t, err) defer svr.Close() - err = os.Setenv("GCS_EMULATOR_HOST", svr.Addr) + err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) testutil.Ok(t, err) - os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "fakecred.json") + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) testutil.Ok(t, err) - testutil.Assert(t, bkt.name == "test-bucket") _, err = bkt.Get(context.Background(), "test-bucket") testutil.NotOk(t, err) testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) From 7f505a232e86119497b4224583f6fabe14b29d9c Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Wed, 2 Oct 2024 17:17:34 +0530 Subject: [PATCH 119/130] refactor code Signed-off-by: milinddethe15 --- providers/azure/helpers.go | 2 -- providers/bos/bos.go | 2 +- providers/gcs/gcs_test.go | 1 - providers/obs/obs.go | 2 +- providers/oci/oci_test.go | 3 +-- providers/oss/oss.go | 1 - providers/oss/oss_test.go | 14 ++++++-------- 7 files changed, 9 insertions(+), 16 deletions(-) diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 1f9dde08..2915fbbb 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -20,8 +20,6 @@ import ( const DirDelim = "/" func getContainerClient(conf Config) (*container.Client, error) { - // Check if a roundtripper has been set in the config - // otherwise build the default transport. var rt http.RoundTripper rt, err := exthttp.DefaultTransport(conf.HTTPConfig) if err != nil { diff --git a/providers/bos/bos.go b/providers/bos/bos.go index f88b7a21..74f9688e 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -66,7 +66,7 @@ func parseConfig(conf []byte) (Config, error) { // NewBucket new bos bucket. func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { - //TODO: Add support for custom roundtripper. + // TODO(https://github.com/thanos-io/objstore/pull/140): Add support for custom roundtripper. if logger == nil { logger = log.NewNopLogger() } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index a06b3a16..39b55041 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -161,7 +161,6 @@ http_config: } func TestNewBucketWithErrorRoundTripper(t *testing.T) { - // Create an error RoundTripper. rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} cfg := Config{ Bucket: "test-bucket", diff --git a/providers/obs/obs.go b/providers/obs/obs.go index eb6ca7ca..7de70633 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -75,7 +75,7 @@ type Bucket struct { } func NewBucket(logger log.Logger, conf []byte) (*Bucket, error) { - //TODO: Add support for custom roundtripper. + // TODO(https://github.com/thanos-io/objstore/pull/140): Add support for custom roundtripper. config, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") diff --git a/providers/oci/oci_test.go b/providers/oci/oci_test.go index 7ea292fd..0fc81ccf 100644 --- a/providers/oci/oci_test.go +++ b/providers/oci/oci_test.go @@ -11,8 +11,7 @@ import ( ) func TestNewBucketWithErrorRoundTripper(t *testing.T) { - - mockPrivateKey := `-----BEGIN RSA PRIVATE KEY----- + const mockPrivateKey = `-----BEGIN RSA PRIVATE KEY----- MIICXgIBAAKBgQDCFENGw33yGihy92pDjZQhl0C36rPJj+CvfSC8+q28hxA161QF NUd13wuCTUcq0Qd2qsBe/2hFyc2DCJJg0h1L78+6Z4UMR7EOcpfdUE9Hf3m/hs+F UR45uBJeDK1HSFHD8bHKD6kv8FPGfJTotc+2xjJwoYi+1hqp1fIekaxsyQIDAQAB diff --git a/providers/oss/oss.go b/providers/oss/oss.go index 0bf5bd4b..0a4cc76e 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -173,7 +173,6 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt } client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) if rt != nil { - // custom RoundTripper clientOption := func(client *alioss.Client) { client.HTTPClient = &http.Client{Transport: rt} } diff --git a/providers/oss/oss_test.go b/providers/oss/oss_test.go index f3380e71..99ad68a3 100644 --- a/providers/oss/oss_test.go +++ b/providers/oss/oss_test.go @@ -11,14 +11,12 @@ import ( ) func TestNewBucketWithErrorRoundTripper(t *testing.T) { - - var config Config - config.Endpoint = "http://test.com/" - config.AccessKeyID = "123" - config.AccessKeySecret = "123" - config.Bucket = "test" - config.AccessKeySecret = "123" - + config := Config{ + Endpoint: "http://test.com/", + AccessKeyID: "123", + AccessKeySecret: "123", + Bucket: "test", + } rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) From 25c6a369f264d21c50b122137d94363d1708a0a7 Mon Sep 17 00:00:00 2001 From: Ake van der Meer Date: Thu, 10 Oct 2024 17:58:36 +0200 Subject: [PATCH 120/130] Revert "s3: stat object instead of empty read" This reverts commit f735308ff69b039c35a24d342e13e402143bf5f6. Signed-off-by: Ake van der Meer --- providers/s3/s3.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 3d9ba213..2f0447a7 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -15,6 +15,7 @@ import ( "strings" "testing" + "github.com/efficientgo/core/logerrcapture" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/minio/minio-go/v7" @@ -437,12 +438,21 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } } + r, err := b.client.GetObject(ctx, b.name, name, *opts) + if err != nil { + return nil, err + } - // StatObject to see if the object exists and we have permissions to read it - if _, err := b.client.StatObject(ctx, b.name, name, *opts); err != nil { + // NotFoundObject error is revealed only after first Read. This does the initial GetRequest. Prefetch this here + // for convenience. + if _, err := r.Read(nil); err != nil { + defer logerrcapture.Do(b.logger, r.Close, "s3 get range obj close") + + // First GET Object request error. return nil, err } - return b.client.GetObject(ctx, b.name, name, *opts) + + return r, nil } // Get returns a reader for the given object name. From f15a3f89666ab99d76712d66672bbc897bdf01ab Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 15 Oct 2024 11:32:43 +0530 Subject: [PATCH 121/130] Include content length in the response of Get and GetRange Signed-off-by: Ashwanth Goli --- objstore.go | 10 ++++++++++ providers/azure/azure.go | 2 +- providers/bos/bos.go | 2 +- providers/cos/cos.go | 12 +----------- providers/filesystem/filesystem.go | 13 ++++++++++--- providers/gcs/gcs.go | 14 ++++++++++++-- providers/obs/obs.go | 2 +- providers/oci/oci.go | 4 ++-- providers/s3/s3.go | 5 ++++- 9 files changed, 42 insertions(+), 22 deletions(-) diff --git a/objstore.go b/objstore.go index bfee7503..dcb65e1d 100644 --- a/objstore.go +++ b/objstore.go @@ -829,3 +829,13 @@ func (t *timingReaderWriterTo) WriteTo(w io.Writer) (n int64, err error) { t.timingReader.updateMetrics(int(n), err) return n, err } + +type ObjectSizerReadCloser struct { + io.ReadCloser + Size int64 +} + +// ObjectSize implement ObjectSizer. +func (o ObjectSizerReadCloser) ObjectSize() (int64, error) { + return o.Size, nil +} diff --git a/providers/azure/azure.go b/providers/azure/azure.go index c2f3adb5..957ccdd6 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -273,7 +273,7 @@ func (b *Bucket) getBlobReader(ctx context.Context, name string, httpRange blob. return nil, errors.Wrapf(err, "cannot download blob, address: %s", blobClient.URL()) } retryOpts := azblob.RetryReaderOptions{MaxRetries: int32(b.readerMaxRetries)} - return resp.NewRetryReader(ctx, &retryOpts), nil + return objstore.ObjectSizerReadCloser{ReadCloser: resp.NewRetryReader(ctx, &retryOpts), Size: *resp.ContentLength}, nil } // Get returns a reader for the given object name. diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 74f9688e..6a111fe2 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -308,7 +308,7 @@ func (b *Bucket) getRange(_ context.Context, bucketName, objectKey string, off, return nil, err } - return obj.Body, nil + return objstore.ObjectSizerReadCloser{ReadCloser: obj.Body, Size: obj.ContentLength}, err } func configFromEnv() Config { diff --git a/providers/cos/cos.go b/providers/cos/cos.go index a8b853e3..2c1c07fe 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -320,20 +320,10 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } // Add size info into reader to pass it to Upload function. - r := objectSizerReadCloser{ReadCloser: resp.Body, size: resp.ContentLength} + r := objstore.ObjectSizerReadCloser{ReadCloser: resp.Body, Size: resp.ContentLength} return r, nil } -type objectSizerReadCloser struct { - io.ReadCloser - size int64 -} - -// ObjectSize implement objstore.ObjectSizer. -func (o objectSizerReadCloser) ObjectSize() (int64, error) { - return o.size, nil -} - // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return b.getRange(ctx, name, 0, -1) diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 21c70485..be49c5f3 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -150,8 +150,12 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return nil, errors.New("object name is empty") } - file := filepath.Join(b.rootDir, name) - if _, err := os.Stat(file); err != nil { + var ( + file = filepath.Join(b.rootDir, name) + stat os.FileInfo + err error + ) + if stat, err = os.Stat(file); err != nil { return nil, errors.Wrapf(err, "stat %s", file) } @@ -171,7 +175,10 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return f, nil } - return &rangeReaderCloser{Reader: io.LimitReader(f, length), f: f}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: &rangeReaderCloser{Reader: io.LimitReader(f, length), + f: f, + }, Size: stat.Size()}, nil } // Exists checks if the given directory exists in memory. diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index b5dae200..89582774 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -226,12 +226,22 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { - return b.bkt.Object(name).NewReader(ctx) + r, err := b.bkt.Object(name).NewReader(ctx) + if err != nil { + return r, err + } + + return objstore.ObjectSizerReadCloser{ReadCloser: r, Size: r.Attrs.Size}, nil } // GetRange returns a new range reader for the given object name and range. func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { - return b.bkt.Object(name).NewRangeReader(ctx, off, length) + r, err := b.bkt.Object(name).NewRangeReader(ctx, off, length) + if err != nil { + return r, err + } + + return objstore.ObjectSizerReadCloser{ReadCloser: r, Size: r.Attrs.Size}, nil } // Attributes returns information about the specified object. diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 7de70633..ec513f90 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -299,7 +299,7 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io if err != nil { return nil, errors.Wrap(err, "failed to get object") } - return output.Body, nil + return objstore.ObjectSizerReadCloser{ReadCloser: output.Body, Size: output.ContentLength}, nil } // Exists checks if the given object exists in the bucket. diff --git a/providers/oci/oci.go b/providers/oci/oci.go index e2f9e98b..b5c3bed1 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -134,7 +134,7 @@ func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { if err != nil { return nil, err } - return response.Content, nil + return objstore.ObjectSizerReadCloser{ReadCloser: response.Content, Size: *response.ContentLength}, nil } // GetRange returns a new range reader for the given object name and range. @@ -164,7 +164,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, offset, length int64 if err != nil { return nil, err } - return response.Content, nil + return objstore.ObjectSizerReadCloser{ReadCloser: response.Content, Size: *response.ContentLength}, nil } // Upload the contents of the reader as an object into the bucket. diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 2f0447a7..ebed16ee 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -452,7 +452,10 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } - return r, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: 0, + }, nil } // Get returns a reader for the given object name. From 76887a7c572dfab8e65561781724045d14f37bc1 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 8 Oct 2024 19:42:09 +0530 Subject: [PATCH 122/130] add for oss Signed-off-by: Ashwanth Goli --- providers/oss/oss.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/providers/oss/oss.go b/providers/oss/oss.go index 0a4cc76e..1f59d7f3 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -16,6 +16,7 @@ import ( "testing" "time" + "github.com/aliyun/aliyun-oss-go-sdk/oss" alioss "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/go-kit/log" "github.com/pkg/errors" @@ -342,12 +343,17 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io opts = append(opts, opt) } - resp, err := b.bucket.GetObject(name, opts...) + resp, err := b.bucket.DoGetObject(&oss.GetObjectRequest{ObjectKey: name}, opts) if err != nil { return nil, err } - return resp, nil + size, err := clientutil.ParseContentLength(resp.Response.Headers) + if err == nil { + return objstore.ObjectSizerReadCloser{ReadCloser: resp.Response, Size: size}, nil + } + + return resp.Response, nil } // Get returns a reader for the given object name. From 7fdd46598d15d4eaa33cf057e345241b9dd205f2 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 8 Oct 2024 19:49:01 +0530 Subject: [PATCH 123/130] add changelog Signed-off-by: Ashwanth Goli --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12375cbf..4686a470 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ## Unreleased - [#38](https://github.com/thanos-io/objstore/pull/38) GCS: Upgrade cloud.google.com/go/storage version to `v1.43.0`. +- [#145](https://github.com/thanos-io/objstore/pull/145) Include content length in the response of Get and GetRange. ### Fixed - [#117](https://github.com/thanos-io/objstore/pull/117) Metrics: Fix `objstore_bucket_operation_failures_total` incorrectly incremented if context is cancelled while reading object contents. From 37b701c09aaea0648bfaf6fb493008ab33657d17 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Wed, 9 Oct 2024 18:38:45 +0530 Subject: [PATCH 124/130] update getRange for fs Signed-off-by: Ashwanth Goli --- providers/filesystem/filesystem.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index be49c5f3..423abb90 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -164,21 +164,29 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return nil, err } + var newOffset int64 if off > 0 { - _, err := f.Seek(off, 0) + newOffset, err = f.Seek(off, 0) if err != nil { return nil, errors.Wrapf(err, "seek %v", off) } } + size := stat.Size() - newOffset if length == -1 { - return f, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: f, + Size: size, + }, nil } return objstore.ObjectSizerReadCloser{ - ReadCloser: &rangeReaderCloser{Reader: io.LimitReader(f, length), - f: f, - }, Size: stat.Size()}, nil + ReadCloser: &rangeReaderCloser{ + Reader: io.LimitReader(f, length), + f: f, + }, + Size: min(length, size), + }, nil } // Exists checks if the given directory exists in memory. From ddca4de27326b4fb5958eb8ec097edc71763c545 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 15 Oct 2024 11:33:50 +0530 Subject: [PATCH 125/130] add acceptance test + refactor Signed-off-by: Ashwanth Goli --- objstore.go | 4 ++-- providers/azure/azure.go | 8 +++++++- providers/bos/bos.go | 7 ++++++- providers/cos/cos.go | 8 ++++++-- providers/filesystem/filesystem.go | 8 ++++++-- providers/gcs/gcs.go | 15 +++++++++++++-- providers/obs/obs.go | 7 ++++++- providers/oci/oci.go | 13 +++++++++++-- providers/oss/oss.go | 7 ++++++- providers/s3/s3.go | 9 ++++++++- providers/swift/swift.go | 6 +++++- testing.go | 20 ++++++++++++++++++++ 12 files changed, 96 insertions(+), 16 deletions(-) diff --git a/objstore.go b/objstore.go index dcb65e1d..116538ca 100644 --- a/objstore.go +++ b/objstore.go @@ -832,10 +832,10 @@ func (t *timingReaderWriterTo) WriteTo(w io.Writer) (n int64, err error) { type ObjectSizerReadCloser struct { io.ReadCloser - Size int64 + Size func() (int64, error) } // ObjectSize implement ObjectSizer. func (o ObjectSizerReadCloser) ObjectSize() (int64, error) { - return o.Size, nil + return o.Size() } diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 957ccdd6..9a4e8518 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -273,7 +273,13 @@ func (b *Bucket) getBlobReader(ctx context.Context, name string, httpRange blob. return nil, errors.Wrapf(err, "cannot download blob, address: %s", blobClient.URL()) } retryOpts := azblob.RetryReaderOptions{MaxRetries: int32(b.readerMaxRetries)} - return objstore.ObjectSizerReadCloser{ReadCloser: resp.NewRetryReader(ctx, &retryOpts), Size: *resp.ContentLength}, nil + + return objstore.ObjectSizerReadCloser{ + ReadCloser: resp.NewRetryReader(ctx, &retryOpts), + Size: func() (int64, error) { + return *resp.ContentLength, nil + }, + }, nil } // Get returns a reader for the given object name. diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 6a111fe2..1f81e920 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -308,7 +308,12 @@ func (b *Bucket) getRange(_ context.Context, bucketName, objectKey string, off, return nil, err } - return objstore.ObjectSizerReadCloser{ReadCloser: obj.Body, Size: obj.ContentLength}, err + return objstore.ObjectSizerReadCloser{ + ReadCloser: obj.Body, + Size: func() (int64, error) { + return obj.ContentLength, nil + }, + }, err } func configFromEnv() Config { diff --git a/providers/cos/cos.go b/providers/cos/cos.go index 2c1c07fe..f88a8e76 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -320,8 +320,12 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } // Add size info into reader to pass it to Upload function. - r := objstore.ObjectSizerReadCloser{ReadCloser: resp.Body, Size: resp.ContentLength} - return r, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: resp.Body, + Size: func() (int64, error) { + return resp.ContentLength, nil + }, + }, nil } // Get returns a reader for the given object name. diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 423abb90..2ed42ee8 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -176,7 +176,9 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( if length == -1 { return objstore.ObjectSizerReadCloser{ ReadCloser: f, - Size: size, + Size: func() (int64, error) { + return size, nil + }, }, nil } @@ -185,7 +187,9 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( Reader: io.LimitReader(f, length), f: f, }, - Size: min(length, size), + Size: func() (int64, error) { + return min(length, size), nil + }, }, nil } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 89582774..e022b14f 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -231,7 +231,12 @@ func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return r, err } - return objstore.ObjectSizerReadCloser{ReadCloser: r, Size: r.Attrs.Size}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: func() (int64, error) { + return r.Attrs.Size, nil + }, + }, nil } // GetRange returns a new range reader for the given object name and range. @@ -241,7 +246,13 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return r, err } - return objstore.ObjectSizerReadCloser{ReadCloser: r, Size: r.Attrs.Size}, nil + sz := r.Remain() + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: func() (int64, error) { + return sz, nil + }, + }, nil } // Attributes returns information about the specified object. diff --git a/providers/obs/obs.go b/providers/obs/obs.go index ec513f90..cb450365 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -299,7 +299,12 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io if err != nil { return nil, errors.Wrap(err, "failed to get object") } - return objstore.ObjectSizerReadCloser{ReadCloser: output.Body, Size: output.ContentLength}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: output.Body, + Size: func() (int64, error) { + return output.ContentLength, nil + }, + }, nil } // Exists checks if the given object exists in the bucket. diff --git a/providers/oci/oci.go b/providers/oci/oci.go index b5c3bed1..3bdf80f3 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -134,7 +134,12 @@ func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { if err != nil { return nil, err } - return objstore.ObjectSizerReadCloser{ReadCloser: response.Content, Size: *response.ContentLength}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: response.Content, + Size: func() (int64, error) { + return *response.ContentLength, nil + }, + }, nil } // GetRange returns a new range reader for the given object name and range. @@ -164,7 +169,11 @@ func (b *Bucket) GetRange(ctx context.Context, name string, offset, length int64 if err != nil { return nil, err } - return objstore.ObjectSizerReadCloser{ReadCloser: response.Content, Size: *response.ContentLength}, nil + return objstore.ObjectSizerReadCloser{ReadCloser: response.Content, + Size: func() (int64, error) { + return *response.ContentLength, nil + }, + }, nil } // Upload the contents of the reader as an object into the bucket. diff --git a/providers/oss/oss.go b/providers/oss/oss.go index 1f59d7f3..d6e1bbf5 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -350,7 +350,12 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io size, err := clientutil.ParseContentLength(resp.Response.Headers) if err == nil { - return objstore.ObjectSizerReadCloser{ReadCloser: resp.Response, Size: size}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: resp.Response, + Size: func() (int64, error) { + return size, nil + }, + }, nil } return resp.Response, nil diff --git a/providers/s3/s3.go b/providers/s3/s3.go index ebed16ee..eac8191a 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -454,7 +454,14 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return objstore.ObjectSizerReadCloser{ ReadCloser: r, - Size: 0, + Size: func() (int64, error) { + stat, err := r.Stat() + if err != nil { + return 0, err + } + + return stat.Size, nil + }, }, nil } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index 682c494c..44fa6ed0 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -262,7 +262,11 @@ func (c *Container) get(name string, headers swift.Headers, checkHash bool) (io. if err != nil { return nil, errors.Wrap(err, "open object") } - return file, err + + return objstore.ObjectSizerReadCloser{ + ReadCloser: file, + Size: file.Length, + }, nil } // Get returns a reader for the given object name. diff --git a/testing.go b/testing.go index b8e3744c..28cbd658 100644 --- a/testing.go +++ b/testing.go @@ -106,6 +106,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rc1, err := bkt.Get(ctx, "id1/obj_1.some") testutil.Ok(t, err) defer func() { testutil.Ok(t, rc1.Close()) }() + + sz, err := TryToGetSize(rc1) + testutil.Ok(t, err) + testutil.Equals(t, int64(11), sz, "expected size to be equal to 11") + content, err := io.ReadAll(rc1) testutil.Ok(t, err) testutil.Equals(t, "@test-data@", string(content)) @@ -118,6 +123,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rc2, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, 3) testutil.Ok(t, err) defer func() { testutil.Ok(t, rc2.Close()) }() + + sz, err = TryToGetSize(rc2) + testutil.Ok(t, err) + testutil.Equals(t, int64(3), sz, "expected size to be equal to 3") + content, err = io.ReadAll(rc2) testutil.Ok(t, err) testutil.Equals(t, "tes", string(content)) @@ -126,6 +136,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rcUnspecifiedLen, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, -1) testutil.Ok(t, err) defer func() { testutil.Ok(t, rcUnspecifiedLen.Close()) }() + + sz, err = TryToGetSize(rcUnspecifiedLen) + testutil.Ok(t, err) + testutil.Equals(t, int64(10), sz, "expected size to be equal to 10") + content, err = io.ReadAll(rcUnspecifiedLen) testutil.Ok(t, err) testutil.Equals(t, "test-data@", string(content)) @@ -141,6 +156,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rcLength, err := bkt.GetRange(ctx, "id1/obj_1.some", 3, 9999) testutil.Ok(t, err) defer func() { testutil.Ok(t, rcLength.Close()) }() + + sz, err = TryToGetSize(rcLength) + testutil.Ok(t, err) + testutil.Equals(t, int64(8), sz, "expected size to be equal to 8") + content, err = io.ReadAll(rcLength) testutil.Ok(t, err) testutil.Equals(t, "st-data@", string(content)) From 2947125233e6b3edace706149b6862e911458058 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Thu, 10 Oct 2024 12:07:04 +0530 Subject: [PATCH 126/130] add size for inmem Signed-off-by: Ashwanth Goli --- inmem.go | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/inmem.go b/inmem.go index 3f6f35e9..ed256c9c 100644 --- a/inmem.go +++ b/inmem.go @@ -119,7 +119,12 @@ func (b *InMemBucket) Get(_ context.Context, name string) (io.ReadCloser, error) return nil, errNotFound } - return io.NopCloser(bytes.NewReader(file)), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file)), + Size: func() (int64, error) { + return int64(len(file)), nil + }, + }, nil } // GetRange returns a new range reader for the given object name and range. @@ -136,15 +141,27 @@ func (b *InMemBucket) GetRange(_ context.Context, name string, off, length int64 } if int64(len(file)) < off { - return io.NopCloser(bytes.NewReader(nil)), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(nil)), + Size: func() (int64, error) { return 0, nil }, + }, nil } if length == -1 { - return io.NopCloser(bytes.NewReader(file[off:])), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file[off:])), + Size: func() (int64, error) { + return int64(len(file[off:])), nil + }, + }, nil } if length <= 0 { - return io.NopCloser(bytes.NewReader(nil)), errors.New("length cannot be smaller or equal 0") + // wrap with ObjectSizerReadCloser to return 0 size. + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(nil)), + Size: func() (int64, error) { return 0, nil }, + }, errors.New("length cannot be smaller or equal 0") } if int64(len(file)) <= off+length { @@ -152,7 +169,12 @@ func (b *InMemBucket) GetRange(_ context.Context, name string, off, length int64 length = int64(len(file)) - off } - return io.NopCloser(bytes.NewReader(file[off : off+length])), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file[off : off+length])), + Size: func() (int64, error) { + return length, nil + }, + }, nil } // Exists checks if the given directory exists in memory. From dfed39a7c13e36f49ad115c92f386307c170dd09 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Thu, 10 Oct 2024 12:40:30 +0530 Subject: [PATCH 127/130] return err if Size is not set Signed-off-by: Ashwanth Goli --- objstore.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/objstore.go b/objstore.go index 116538ca..5bce3ef6 100644 --- a/objstore.go +++ b/objstore.go @@ -837,5 +837,9 @@ type ObjectSizerReadCloser struct { // ObjectSize implement ObjectSizer. func (o ObjectSizerReadCloser) ObjectSize() (int64, error) { + if o.Size == nil { + return 0, errors.New("unknown size") + } + return o.Size() } From 168679cbbf200a7d9091065d7239ee14e5561190 Mon Sep 17 00:00:00 2001 From: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> Date: Thu, 24 Oct 2024 17:37:00 +0530 Subject: [PATCH 128/130] Added roundtripper wrapper as param (#150) * added roundtripper wrapper as param Signed-off-by: milinddethe15 * CHANGELOG and TODO Signed-off-by: milinddethe15 * fix lint Signed-off-by: milinddethe15 * Update CHANGELOG.md Signed-off-by: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> * refactor code Signed-off-by: milinddethe15 --------- Signed-off-by: milinddethe15 Signed-off-by: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> --- CHANGELOG.md | 1 + client/factory.go | 16 ++++++++-------- errutil/rt_error.go | 16 +++++++++++++++- providers/azure/azure.go | 11 ++++------- providers/azure/azure_test.go | 7 ++----- providers/azure/helpers.go | 5 ++++- providers/bos/bos.go | 2 +- providers/cos/cos.go | 19 +++++++++++-------- providers/cos/cos_test.go | 6 ++---- providers/gcs/gcs.go | 17 +++++++++-------- providers/gcs/gcs_test.go | 6 ++---- providers/obs/obs.go | 2 +- providers/oci/oci.go | 15 +++++++++------ providers/oci/oci_test.go | 7 ++----- providers/oss/oss.go | 23 +++++++++++++++-------- providers/oss/oss_test.go | 6 ++---- providers/s3/s3.go | 13 +++++++------ providers/s3/s3_test.go | 6 ++---- providers/swift/swift.go | 20 ++++++++++---------- providers/swift/swift_test.go | 7 ++----- 20 files changed, 109 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4686a470..75cbd834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#116](https://github.com/thanos-io/objstore/pull/116) Azure: Add new storage_create_container configuration property - [#128](https://github.com/thanos-io/objstore/pull/128) GCS: Add support for `ChunkSize` for writer. - [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket +- [#150](https://github.com/thanos-io/objstore/pull/150) Add support for roundtripper wrapper. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/client/factory.go b/client/factory.go index bd345024..5fe5a741 100644 --- a/client/factory.go +++ b/client/factory.go @@ -50,7 +50,7 @@ type BucketConfig struct { // NewBucket initializes and returns new object storage clients. // NOTE: confContentYaml can contain secrets. -func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt http.RoundTripper) (objstore.Bucket, error) { +func NewBucket(logger log.Logger, confContentYaml []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (objstore.Bucket, error) { level.Info(logger).Log("msg", "loading bucket configuration") bucketConf := &BucketConfig{} if err := yaml.UnmarshalStrict(confContentYaml, bucketConf); err != nil { @@ -65,23 +65,23 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt h var bucket objstore.Bucket switch strings.ToUpper(string(bucketConf.Type)) { case string(GCS): - bucket, err = gcs.NewBucket(context.Background(), logger, config, component, rt) + bucket, err = gcs.NewBucket(context.Background(), logger, config, component, wrapRoundtripper) case string(S3): - bucket, err = s3.NewBucket(logger, config, component, rt) + bucket, err = s3.NewBucket(logger, config, component, wrapRoundtripper) case string(AZURE): - bucket, err = azure.NewBucket(logger, config, component, rt) + bucket, err = azure.NewBucket(logger, config, component, wrapRoundtripper) case string(SWIFT): - bucket, err = swift.NewContainer(logger, config, rt) + bucket, err = swift.NewContainer(logger, config, wrapRoundtripper) case string(COS): - bucket, err = cos.NewBucket(logger, config, component, rt) + bucket, err = cos.NewBucket(logger, config, component, wrapRoundtripper) case string(ALIYUNOSS): - bucket, err = oss.NewBucket(logger, config, component, rt) + bucket, err = oss.NewBucket(logger, config, component, wrapRoundtripper) case string(FILESYSTEM): bucket, err = filesystem.NewBucketFromConfig(config) case string(BOS): bucket, err = bos.NewBucket(logger, config, component) case string(OCI): - bucket, err = oci.NewBucket(logger, config, rt) + bucket, err = oci.NewBucket(logger, config, wrapRoundtripper) case string(OBS): bucket, err = obs.NewBucket(logger, config) default: diff --git a/errutil/rt_error.go b/errutil/rt_error.go index ad1309e1..b6b2e9c9 100644 --- a/errutil/rt_error.go +++ b/errutil/rt_error.go @@ -1,6 +1,16 @@ package errutil -import "net/http" +import ( + "net/http" + + "github.com/pkg/errors" +) + +var rtErr = errors.New("RoundTripper error") + +func IsMockedError(err error) bool { + return errors.Is(err, rtErr) +} // ErrorRoundTripper is a custom RoundTripper that always returns an error. type ErrorRoundTripper struct { @@ -10,3 +20,7 @@ type ErrorRoundTripper struct { func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { return nil, ert.Err } + +func WrapWithErrRoundtripper(rt http.RoundTripper) http.RoundTripper { + return &ErrorRoundTripper{Err: rtErr} +} diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 9a4e8518..e125ca35 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -146,7 +146,7 @@ type Bucket struct { } // NewBucket returns a new Bucket using the provided Azure config. -func NewBucket(logger log.Logger, azureConfig []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, azureConfig []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new Azure bucket connection", "component", component) conf, err := parseConfig(azureConfig) if err != nil { @@ -155,19 +155,16 @@ func NewBucket(logger log.Logger, azureConfig []byte, component string, rt http. if conf.MSIResource != "" { level.Warn(logger).Log("msg", "The field msi_resource has been deprecated and should no longer be set") } - return NewBucketWithConfig(logger, conf, component, rt) + return NewBucketWithConfig(logger, conf, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket using the provided Azure config struct. -func NewBucketWithConfig(logger log.Logger, conf Config, component string, rt http.RoundTripper) (*Bucket, error) { - if rt != nil { - conf.HTTPConfig.Transport = rt - } +func NewBucketWithConfig(logger log.Logger, conf Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if err := conf.validate(); err != nil { return nil, err } - containerClient, err := getContainerClient(conf) + containerClient, err := getContainerClient(conf, wrapRoundtripper) if err != nil { return nil, err } diff --git a/providers/azure/azure_test.go b/providers/azure/azure_test.go index 85533eaa..a96dcefb 100644 --- a/providers/azure/azure_test.go +++ b/providers/azure/azure_test.go @@ -9,7 +9,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" @@ -230,11 +229,9 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { cfg, err := parseConfig(validConfig) testutil.Ok(t, err) - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - - _, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) + _, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", errutil.WrapWithErrRoundtripper) // We expect an error from the RoundTripper testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 2915fbbb..deb86d03 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -19,7 +19,7 @@ import ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" -func getContainerClient(conf Config) (*container.Client, error) { +func getContainerClient(conf Config, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*container.Client, error) { var rt http.RoundTripper rt, err := exthttp.DefaultTransport(conf.HTTPConfig) if err != nil { @@ -28,6 +28,9 @@ func getContainerClient(conf Config) (*container.Client, error) { if conf.HTTPConfig.Transport != nil { rt = conf.HTTPConfig.Transport } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) + } opt := &container.ClientOptions{ ClientOptions: azcore.ClientOptions{ Retry: policy.RetryOptions{ diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 1f81e920..9faa93f3 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -66,7 +66,7 @@ func parseConfig(conf []byte) (Config, error) { // NewBucket new bos bucket. func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { - // TODO(https://github.com/thanos-io/objstore/pull/140): Add support for custom roundtripper. + // TODO(https://github.com/thanos-io/objstore/pull/150): Add support for roundtripper wrapper. if logger == nil { logger = log.NewNopLogger() } diff --git a/providers/cos/cos.go b/providers/cos/cos.go index f88a8e76..6bd39caa 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -95,7 +95,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided cos configuration. -func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if logger == nil { logger = log.NewNopLogger() } @@ -104,11 +104,11 @@ func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTr if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - return NewBucketWithConfig(logger, config, component, rt) + return NewBucketWithConfig(logger, config, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket using the provided cos config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate cos configuration") } @@ -127,19 +127,22 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt } } b := &cos.BaseURL{BucketURL: bucketURL} - var tpt http.RoundTripper - tpt, err = exthttp.DefaultTransport(config.HTTPConfig) + var rt http.RoundTripper + rt, err = exthttp.DefaultTransport(config.HTTPConfig) if err != nil { return nil, err } - if rt != nil { - tpt = rt + if config.HTTPConfig.Transport != nil { + rt = config.HTTPConfig.Transport + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) } client := cos.NewClient(b, &http.Client{ Transport: &cos.AuthorizationTransport{ SecretID: config.SecretId, SecretKey: config.SecretKey, - Transport: tpt, + Transport: rt, }, }) diff --git a/providers/cos/cos_test.go b/providers/cos/cos_test.go index f682aee9..4f7a56d6 100644 --- a/providers/cos/cos_test.go +++ b/providers/cos/cos_test.go @@ -10,7 +10,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/thanos-io/objstore/errutil" @@ -150,12 +149,11 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { SecretId: "sid", SecretKey: "skey", } - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", errutil.WrapWithErrRoundtripper) testutil.Ok(t, err) _, err = bkt.Get(context.Background(), "Test") // We expect an error from the RoundTripper testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index e022b14f..efb208e6 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -77,22 +77,20 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(ctx, logger, config, component, rt) + return NewBucketWithConfig(ctx, logger, config, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket with gcs Config struct. -func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if gc.Bucket == "" { return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") } - if rt != nil { - gc.HTTPConfig.Transport = rt - } + var opts []option.ClientOption // If ServiceAccount is provided, use them in GCS client, otherwise fallback to Google default logic. @@ -112,7 +110,7 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp if !gc.UseGRPC { var err error - opts, err = appendHttpOptions(gc, opts) + opts, err = appendHttpOptions(gc, opts, wrapRoundtripper) if err != nil { return nil, err } @@ -121,7 +119,7 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp return newBucket(ctx, logger, gc, opts) } -func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOption, error) { +func appendHttpOptions(gc Config, opts []option.ClientOption, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) ([]option.ClientOption, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. var rt http.RoundTripper @@ -132,6 +130,9 @@ func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOp if gc.HTTPConfig.Transport != nil { rt = gc.HTTPConfig.Transport } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) + } // GCS uses some defaults when "options.WithHTTPClient" is not used that are important when we call // htransport.NewTransport namely the scopes that are then used for OAth authentication. So to build our own diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 39b55041..80951d7a 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -15,7 +15,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/fullstorydev/emulators/storage/gcsemu" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/thanos-io/objstore/errutil" "google.golang.org/api/option" @@ -161,7 +160,6 @@ http_config: } func TestNewBucketWithErrorRoundTripper(t *testing.T) { - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} cfg := Config{ Bucket: "test-bucket", ServiceAccount: "", @@ -174,9 +172,9 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) testutil.Ok(t, err) - bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", errutil.WrapWithErrRoundtripper) testutil.Ok(t, err) _, err = bkt.Get(context.Background(), "test-bucket") testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/obs/obs.go b/providers/obs/obs.go index cb450365..4fb17baa 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -75,7 +75,7 @@ type Bucket struct { } func NewBucket(logger log.Logger, conf []byte) (*Bucket, error) { - // TODO(https://github.com/thanos-io/objstore/pull/140): Add support for custom roundtripper. + // TODO(https://github.com/thanos-io/objstore/pull/150): Add support for roundtripper wrapper. config, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 3bdf80f3..bc8a8bd9 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -298,7 +298,7 @@ func (b *Bucket) deleteBucket(ctx context.Context) (err error) { } // NewBucket returns a new Bucket using the provided oci config values. -func NewBucket(logger log.Logger, ociConfig []byte, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, ociConfig []byte, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new oci bucket connection") var config = DefaultConfig var configurationProvider common.ConfigurationProvider @@ -344,13 +344,16 @@ func NewBucket(logger log.Logger, ociConfig []byte, rt http.RoundTripper) (*Buck if err != nil { return nil, errors.Wrapf(err, "unable to create ObjectStorage client with the given oci configurations") } - - config.HTTPConfig.Transport = CustomTransport(config) - if rt != nil { - config.HTTPConfig.Transport = rt + var rt http.RoundTripper + rt = CustomTransport(config) + if config.HTTPConfig.Transport != nil { + rt = config.HTTPConfig.Transport + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) } httpClient := http.Client{ - Transport: config.HTTPConfig.Transport, + Transport: rt, Timeout: config.HTTPConfig.ClientTimeout, } client.HTTPClient = &httpClient diff --git a/providers/oci/oci_test.go b/providers/oci/oci_test.go index 0fc81ccf..6b44a0e7 100644 --- a/providers/oci/oci_test.go +++ b/providers/oci/oci_test.go @@ -5,7 +5,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/thanos-io/objstore/errutil" "gopkg.in/yaml.v2" ) @@ -38,10 +37,8 @@ G6aFKaqQfOXKCyWoUiVknQJAXrlgySFci/2ueKlIE1QqIiLSZ8V8OlpFLRnb1pzI ociConfig, err := yaml.Marshal(config) testutil.Ok(t, err) - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - - _, err = NewBucket(log.NewNopLogger(), ociConfig, rt) + _, err = NewBucket(log.NewNopLogger(), ociConfig, errutil.WrapWithErrRoundtripper) // We expect an error from the RoundTripper testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/oss/oss.go b/providers/oss/oss.go index d6e1bbf5..e01aff2e 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -23,6 +23,7 @@ import ( "gopkg.in/yaml.v2" "github.com/thanos-io/objstore/clientutil" + "github.com/thanos-io/objstore/exthttp" "github.com/thanos-io/objstore" ) @@ -159,26 +160,32 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt } // NewBucket returns a new Bucket using the provided oss config values. -func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { var config Config if err := yaml.Unmarshal(conf, &config); err != nil { return nil, errors.Wrap(err, "parse aliyun oss config file failed") } - return NewBucketWithConfig(logger, config, component, rt) + return NewBucketWithConfig(logger, config, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket using the provided oss config struct. -func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if err := validate(config); err != nil { return nil, err } - client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) - if rt != nil { - clientOption := func(client *alioss.Client) { - client.HTTPClient = &http.Client{Transport: rt} + var clientOptions []alioss.ClientOption + if wrapRoundtripper != nil { + rt, err := exthttp.DefaultTransport(exthttp.DefaultHTTPConfig) + if err != nil { + return nil, err } - client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret, clientOption) + clientOptions = append(clientOptions, func(client *alioss.Client) { + client.HTTPClient = &http.Client{ + Transport: wrapRoundtripper(rt), + } + }) } + client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret, clientOptions...) if err != nil { return nil, errors.Wrap(err, "create aliyun oss client failed") } diff --git a/providers/oss/oss_test.go b/providers/oss/oss_test.go index 99ad68a3..b43d3077 100644 --- a/providers/oss/oss_test.go +++ b/providers/oss/oss_test.go @@ -6,7 +6,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/thanos-io/objstore/errutil" ) @@ -17,12 +16,11 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { AccessKeySecret: "123", Bucket: "test", } - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", errutil.WrapWithErrRoundtripper) // We expect an error from the RoundTripper testutil.Ok(t, err) _, err = bkt.Get(context.Background(), "test") testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/s3/s3.go b/providers/s3/s3.go index eac8191a..8e5b8b56 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -176,13 +176,13 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(logger, config, component, rt) + return NewBucketWithConfig(logger, config, component, wrapRoundtripper) } type overrideSignerType struct { @@ -202,7 +202,7 @@ func (s *overrideSignerType) Retrieve() (credentials.Value, error) { } // NewBucketWithConfig returns a new Bucket using the provided s3 config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { var chain []credentials.Provider // TODO(bwplotka): Don't do flags as they won't scale, use actual params like v2, v4 instead @@ -242,9 +242,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt }), } } - if rt != nil { - config.HTTPConfig.Transport = rt - } + // Check if a roundtripper has been set in the config // otherwise build the default transport. var tpt http.RoundTripper @@ -255,6 +253,9 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt if config.HTTPConfig.Transport != nil { tpt = config.HTTPConfig.Transport } + if wrapRoundtripper != nil { + tpt = wrapRoundtripper(tpt) + } client, err := minio.New(config.Endpoint, &minio.Options{ Creds: credentials.NewChainCredentials(chain), diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index 2a44f0e0..3040cd81 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -16,7 +16,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/minio/minio-go/v7/pkg/encrypt" - "github.com/pkg/errors" "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" @@ -469,11 +468,10 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { cfg := DefaultConfig cfg.Endpoint = endpoint cfg.Bucket = "test" - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", errutil.WrapWithErrRoundtripper) testutil.Ok(t, err) _, err = bkt.Get(context.Background(), "test") // We expect an error from the RoundTripper testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index 44fa6ed0..e872728e 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -154,12 +154,12 @@ type Container struct { segmentsContainer string } -func NewContainer(logger log.Logger, conf []byte, rt http.RoundTripper) (*Container, error) { +func NewContainer(logger log.Logger, conf []byte, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Container, error) { sc, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parse config") } - return NewContainerFromConfig(logger, sc, false, rt) + return NewContainerFromConfig(logger, sc, false, wrapRoundtripper) } func ensureContainer(connection *swift.Connection, name string, createIfNotExist bool) error { @@ -178,22 +178,22 @@ func ensureContainer(connection *swift.Connection, name string, createIfNotExist return nil } -func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, rt http.RoundTripper) (*Container, error) { - if rt != nil { - sc.HTTPConfig.Transport = rt - } +func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Container, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. - var tpt http.RoundTripper - tpt, err := exthttp.DefaultTransport(sc.HTTPConfig) + var rt http.RoundTripper + rt, err := exthttp.DefaultTransport(sc.HTTPConfig) if err != nil { return nil, err } if sc.HTTPConfig.Transport != nil { - tpt = sc.HTTPConfig.Transport + rt = sc.HTTPConfig.Transport + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) } - connection := connectionFromConfig(sc, tpt) + connection := connectionFromConfig(sc, rt) if err := connection.Authenticate(); err != nil { return nil, errors.Wrap(err, "authentication") } diff --git a/providers/swift/swift_test.go b/providers/swift/swift_test.go index 629dfd77..b17a5e2b 100644 --- a/providers/swift/swift_test.go +++ b/providers/swift/swift_test.go @@ -9,7 +9,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/thanos-io/objstore/errutil" ) @@ -69,13 +68,11 @@ http_config: } func TestNewBucketWithErrorRoundTripper(t *testing.T) { - logger := log.NewNopLogger() - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} config := DefaultConfig config.AuthUrl = "http://identity.something.com/v3" - _, err := NewContainerFromConfig(logger, &config, false, rt) + _, err := NewContainerFromConfig(log.NewNopLogger(), &config, false, errutil.WrapWithErrRoundtripper) // We expect an error from the RoundTripper testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } From 31c0873d5266a6c58fe8fb5d6de01b66dcd50f5f Mon Sep 17 00:00:00 2001 From: Siavash Safi Date: Mon, 28 Oct 2024 15:45:11 +0100 Subject: [PATCH 129/130] fix objstore_bucket_operation_duration_seconds metrics fix histogram metrics for `get` and `getrange` operations. Signed-off-by: Siavash Safi --- objstore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/objstore.go b/objstore.go index 5bce3ef6..62f1c655 100644 --- a/objstore.go +++ b/objstore.go @@ -573,7 +573,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } - b.metrics.opsDuration.WithLabelValues(op).Observe(float64(time.Since(start))) + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return nil, err } return newTimingReader( @@ -600,7 +600,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } - b.metrics.opsDuration.WithLabelValues(op).Observe(float64(time.Since(start))) + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return nil, err } return newTimingReader( From cf96e342ed0d5977b1269c08d8c34ea9bd5ff442 Mon Sep 17 00:00:00 2001 From: Siavash Safi Date: Mon, 28 Oct 2024 15:59:44 +0100 Subject: [PATCH 130/130] add changelog Signed-off-by: Siavash Safi --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75cbd834..d2b1aaab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#145](https://github.com/thanos-io/objstore/pull/145) Include content length in the response of Get and GetRange. ### Fixed +- [#153](https://github.com/thanos-io/objstore/pull/153) Metrics: Fix `objstore_bucket_operation_duration_seconds_*` for `get` and `get_range` operations. - [#117](https://github.com/thanos-io/objstore/pull/117) Metrics: Fix `objstore_bucket_operation_failures_total` incorrectly incremented if context is cancelled while reading object contents. - [#115](https://github.com/thanos-io/objstore/pull/115) GCS: Fix creation of bucket with GRPC connections. Also update storage client to `v1.40.0`. - [#102](https://github.com/thanos-io/objstore/pull/102) Azure: bump azblob sdk to get concurrency fixes.