diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 354b51f087..c57255bacb 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -12,7 +12,6 @@ jobs: GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }} REGISTRY: gcr.io/kf-feast MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - DOCKER_BUILDKIT: '1' steps: - uses: actions/checkout@v2 - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master diff --git a/go.mod b/go.mod index 8f9d12581a..f0b1ef7568 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/gogo/protobuf v1.3.1 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/mock v1.2.0 - github.com/golang/protobuf v1.4.2 - github.com/google/go-cmp v0.4.0 + github.com/golang/protobuf v1.4.3 + github.com/google/go-cmp v0.5.0 github.com/huandu/xstrings v1.2.0 // indirect github.com/lyft/protoc-gen-validate v0.1.0 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect @@ -18,16 +18,16 @@ require ( github.com/mwitkow/go-proto-validators v0.2.0 // indirect github.com/pseudomuto/protoc-gen-doc v1.3.0 // indirect github.com/pseudomuto/protokit v0.2.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/cobra v0.0.4 github.com/spf13/viper v1.4.0 github.com/woop/protoc-gen-doc v1.3.0 // indirect go.opencensus.io v0.22.3 // indirect golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect - golang.org/x/net v0.0.0-20200513185701-a91f0712d120 - golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect - golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa // indirect + golang.org/x/net v0.0.0-20201021035429-f5854403a974 + golang.org/x/tools v0.0.0-20201202200335-bef1c476418a // indirect google.golang.org/grpc v1.29.1 - google.golang.org/protobuf v1.24.0 // indirect + google.golang.org/protobuf v1.25.0 // indirect gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.2.4 istio.io/gogo-genproto v0.0.0-20191212213402-78a529a42cd8 // indirect diff --git a/go.sum b/go.sum index 7038e78cbc..58e1f6d5e4 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -165,6 +167,7 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -290,6 +293,8 @@ github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNue github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -334,6 +339,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -356,6 +362,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -370,6 +377,7 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d h1:g9qWBGx4puODJTMVyoPrpoxPFgVGd+z1DZwjfRu4d0I= @@ -396,6 +404,7 @@ golang.org/x/net v0.0.0-20200320220750-118fecf932d8 h1:1+zQlQqEEhUeStBTi653GZAnA golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200513185701-a91f0712d120 h1:EZ3cVSzKOlJxAd8e8YAJ7no8nNypTxexh/YE/xW3ZEY= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -406,6 +415,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -428,6 +438,7 @@ golang.org/x/sys v0.0.0-20200321134203-328b4cd54aae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 h1:YTzHMGlqJu67/uEo1lBv0n3wBXhXNeUbB1XfN2vmTm0= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -435,6 +446,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -474,10 +486,13 @@ golang.org/x/tools v0.0.0-20200604042327-9b20fe4cabe8 h1:8Xr1qwxn90MXYKftwNxIO2g golang.org/x/tools v0.0.0-20200604042327-9b20fe4cabe8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa h1:mMXQKlWCw9mIWgVLLfiycDZjMHMMYqiuakI4E/l2xcA= golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201202200335-bef1c476418a h1:TYqOq/v+Ri5aADpldxXOj6PmvcPMOJbLjdALzZDQT2M= +golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= @@ -530,6 +545,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/sdk/go/client.go b/sdk/go/client.go index 7accfcd0c8..83638bb7d3 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -5,6 +5,7 @@ import ( "crypto/x509" "fmt" "github.com/feast-dev/feast/sdk/go/protos/feast/serving" + "github.com/opentracing-contrib/go-grpc" "github.com/opentracing/opentracing-go" "go.opencensus.io/plugin/ocgrpc" "google.golang.org/grpc" @@ -43,16 +44,30 @@ func NewGrpcClient(host string, port int) (*GrpcClient, error) { }) } +func NewTraceableGrpcClient(host string, port int, tracer opentracing.Tracer) (*GrpcClient, error) { + return NewSecureGrpcClient( + host, + port, + SecurityConfig{ + EnableTLS: false, + Credential: nil, + }, + grpc.WithUnaryInterceptor( + otgrpc.OpenTracingClientInterceptor(tracer)), + grpc.WithStreamInterceptor( + otgrpc.OpenTracingStreamClientInterceptor(tracer))) +} + // NewAuthGrpcClient constructs a secure client that uses security features (ie authentication). // host - hostname of the serving host/instance to connect to. // port - post of the host to service host/instancf to connect to. // securityConfig - security config configures client security. -func NewSecureGrpcClient(host string, port int, security SecurityConfig) (*GrpcClient, error) { +func NewSecureGrpcClient(host string, port int, security SecurityConfig, options ...grpc.DialOption) (*GrpcClient, error) { feastCli := &GrpcClient{} adr := fmt.Sprintf("%s:%d", host, port) // Compile grpc dial options from security config. - options := []grpc.DialOption{grpc.WithStatsHandler(&ocgrpc.ClientHandler{})} + options = append(options, grpc.WithStatsHandler(&ocgrpc.ClientHandler{})) // Configure client TLS. if !security.EnableTLS { options = append(options, grpc.WithInsecure()) @@ -90,8 +105,6 @@ func NewSecureGrpcClient(host string, port int, security SecurityConfig) (*GrpcC // GetOnlineFeatures gets the latest values of the request features from the Feast serving instance provided. func (fc *GrpcClient) GetOnlineFeatures(ctx context.Context, req *OnlineFeaturesRequest) ( *OnlineFeaturesResponse, error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "get_online_features") - defer span.Finish() featuresRequest, err := req.buildRequest() if err != nil { @@ -112,8 +125,6 @@ func (fc *GrpcClient) GetOnlineFeatures(ctx context.Context, req *OnlineFeatures // GetFeastServingInfo gets information about the feast serving instance this client is connected to. func (fc *GrpcClient) GetFeastServingInfo(ctx context.Context, in *serving.GetFeastServingInfoRequest) ( *serving.GetFeastServingInfoResponse, error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "get_info") - defer span.Finish() return fc.cli.GetFeastServingInfo(ctx, in) } diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index ab04cdeaa1..8827fba55e 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -9,7 +9,6 @@ import ( "github.com/feast-dev/feast/sdk/go/protos/feast/types" "github.com/golang/mock/gomock" "github.com/google/go-cmp/cmp" - "github.com/opentracing/opentracing-go" ) func TestGetOnlineFeatures(t *testing.T) { @@ -62,10 +61,9 @@ func TestGetOnlineFeatures(t *testing.T) { defer ctrl.Finish() cli := mock_serving.NewMockServingServiceClient(ctrl) ctx := context.Background() - _, traceCtx := opentracing.StartSpanFromContext(ctx, "get_online_features") rawRequest, _ := tc.req.buildRequest() resp := tc.want.RawResponse - cli.EXPECT().GetOnlineFeatures(traceCtx, rawRequest).Return(resp, nil).Times(1) + cli.EXPECT().GetOnlineFeatures(ctx, rawRequest).Return(resp, nil).Times(1) client := &GrpcClient{ cli: cli, diff --git a/sdk/go/go.mod b/sdk/go/go.mod index 6a3b3dd477..f531f5a718 100644 --- a/sdk/go/go.mod +++ b/sdk/go/go.mod @@ -6,6 +6,7 @@ require ( github.com/golang/mock v1.4.3 github.com/golang/protobuf v1.4.2 github.com/google/go-cmp v0.5.1 + github.com/opentracing-contrib/go-grpc v0.0.0-20200813121455-4a6760c71486 github.com/opentracing/opentracing-go v1.1.0 github.com/stretchr/testify v1.4.0 // indirect go.opencensus.io v0.22.4 diff --git a/sdk/go/go.sum b/sdk/go/go.sum index dba3041f04..d476294cc9 100644 --- a/sdk/go/go.sum +++ b/sdk/go/go.sum @@ -108,6 +108,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -117,6 +118,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/opentracing-contrib/go-grpc v0.0.0-20200813121455-4a6760c71486 h1:K35HCWaOTJIPW6cDHK4yj3QfRY/NhE0pBbfoc0M2NMQ= +github.com/opentracing-contrib/go-grpc v0.0.0-20200813121455-4a6760c71486/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -186,6 +189,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -357,6 +361,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= diff --git a/serving/pom.xml b/serving/pom.xml index 100eed3d37..e3d95ad740 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -169,21 +169,26 @@ joda-time joda-time - + io.jaegertracing jaeger-client - 0.31.0 + 1.3.2 io.opentracing opentracing-api - 0.31.0 + 0.33.0 io.opentracing opentracing-noop - 0.31.0 + 0.33.0 + + + io.opentracing.contrib + opentracing-grpc + 0.2.3 diff --git a/serving/src/main/java/feast/serving/config/InstrumentationConfig.java b/serving/src/main/java/feast/serving/config/InstrumentationConfig.java index 30269c5d0e..295b263f66 100644 --- a/serving/src/main/java/feast/serving/config/InstrumentationConfig.java +++ b/serving/src/main/java/feast/serving/config/InstrumentationConfig.java @@ -17,6 +17,7 @@ package feast.serving.config; import io.opentracing.Tracer; +import io.opentracing.contrib.grpc.TracingServerInterceptor; import io.opentracing.noop.NoopTracerFactory; import io.prometheus.client.exporter.MetricsServlet; import io.prometheus.client.hotspot.DefaultExports; @@ -54,4 +55,9 @@ public Tracer tracer() { return io.jaegertracing.Configuration.fromEnv(feastProperties.getTracing().getServiceName()) .getTracer(); } + + @Bean + public TracingServerInterceptor tracingInterceptor(Tracer tracer) { + return TracingServerInterceptor.newBuilder().withTracer(tracer).build(); + } } diff --git a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java index 01702d9f3e..13fa8f4f2e 100644 --- a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java +++ b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java @@ -35,9 +35,9 @@ import feast.serving.util.RequestHelper; import io.grpc.Status; import io.grpc.stub.StreamObserver; -import io.opentracing.Scope; import io.opentracing.Span; import io.opentracing.Tracer; +import io.opentracing.contrib.grpc.TracingServerInterceptor; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -47,7 +47,12 @@ import org.springframework.security.access.AccessDeniedException; import org.springframework.security.core.context.SecurityContextHolder; -@GrpcService(interceptors = {GrpcMessageInterceptor.class, GrpcMonitoringInterceptor.class}) +@GrpcService( + interceptors = { + TracingServerInterceptor.class, + GrpcMessageInterceptor.class, + GrpcMonitoringInterceptor.class + }) public class ServingServiceGRpcController extends ServingServiceImplBase { private static final Logger log = @@ -83,8 +88,7 @@ public void getFeastServingInfo( public void getOnlineFeatures( GetOnlineFeaturesRequest request, StreamObserver responseObserver) { - Span span = tracer.buildSpan("getOnlineFeatures").start(); - try (Scope scope = tracer.scopeManager().activate(span, false)) { + try { // authorize for the project in request object. if (request.getProject() != null && !request.getProject().isEmpty()) { // project set at root level overrides the project set at feature set level @@ -96,7 +100,11 @@ public void getOnlineFeatures( this.checkProjectAccess(request.getFeaturesList()); } RequestHelper.validateOnlineRequest(request); + Span span = tracer.buildSpan("getOnlineFeatures").start(); GetOnlineFeaturesResponse onlineFeatures = servingService.getOnlineFeatures(request); + if (span != null) { + span.finish(); + } responseObserver.onNext(onlineFeatures); responseObserver.onCompleted(); } catch (SpecRetrievalException e) { @@ -114,7 +122,6 @@ public void getOnlineFeatures( log.warn("Failed to get Online Features", e); responseObserver.onError(e); } - span.finish(); } @Override diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index a7d9d284aa..4af91d684a 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -16,24 +16,21 @@ */ package feast.serving.service; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Streams; import com.google.protobuf.Duration; import feast.common.models.Feature; -import feast.common.models.FeatureSet; import feast.proto.serving.ServingAPIProto.*; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; import feast.proto.types.ValueProto.Value; import feast.serving.specs.CachedSpecService; import feast.serving.util.Metrics; import feast.storage.api.retriever.FeatureSetRequest; import feast.storage.api.retriever.OnlineRetriever; import io.grpc.Status; -import io.opentracing.Scope; +import io.opentracing.Span; import io.opentracing.Tracer; import java.util.*; import java.util.stream.Collectors; @@ -66,86 +63,94 @@ public GetFeastServingInfoResponse getFeastServingInfo( /** {@inheritDoc} */ @Override public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { - try (Scope scope = tracer.buildSpan("getOnlineFeatures").startActive(true)) { - List entityRows = request.getEntityRowsList(); - // Collect the feature/entity value for each entity row in entityValueMap - Map> entityValuesMap = - entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); - // Collect the feature/entity status metadata for each entity row in entityValueMap - Map> entityStatusesMap = - entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); - // Collect featureRows retrieved for logging/tracing - List>> logFeatureRows = new LinkedList<>(); + List entityRows = request.getEntityRowsList(); + // Collect the feature/entity value for each entity row in entityValueMap + Map> entityValuesMap = + entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); + // Collect the feature/entity status metadata for each entity row in entityValueMap + Map> entityStatusesMap = + entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); + // Collect featureRows retrieved for logging/tracing + List>> logFeatureRows = new LinkedList<>(); - if (!request.getOmitEntitiesInResponse()) { - // Add entity row's fields as response fields - entityRows.forEach( - entityRow -> { - Map valueMap = entityRow.getFieldsMap(); - entityValuesMap.get(entityRow).putAll(valueMap); - entityStatusesMap.get(entityRow).putAll(getMetadataMap(valueMap, false, false)); - }); - } + if (!request.getOmitEntitiesInResponse()) { + // Add entity row's fields as response fields + entityRows.forEach( + entityRow -> { + Map valueMap = entityRow.getFieldsMap(); + entityValuesMap.get(entityRow).putAll(valueMap); + entityStatusesMap.get(entityRow).putAll(getMetadataMap(valueMap, false, false)); + }); + } - List featureSetRequests = - specService.getFeatureSets(request.getFeaturesList(), request.getProject()); - for (FeatureSetRequest featureSetRequest : featureSetRequests) { - // Pull feature rows for given entity rows from the feature/featureset specified in feature - // set request. - // from the configured online - List> featureRows = - retriever.getOnlineFeatures(entityRows, featureSetRequest); - // Check that feature row returned corresponds to a given entity row. - if (featureRows.size() != entityRows.size()) { - throw Status.INTERNAL - .withDescription( - "The no. of FeatureRow obtained from OnlineRetriever" - + "does not match no. of entityRow passed.") - .asRuntimeException(); - } + Span specServiceSpan = tracer.buildSpan("getFeatureSets").start(); + List featureSetRequests = + specService.getFeatureSets(request.getFeaturesList(), request.getProject()); + if (specServiceSpan != null) { + specServiceSpan.finish(); + } - Streams.zip(entityRows.stream(), featureRows.stream(), Pair::of) - .forEach( - entityFeaturePair -> { - EntityRow entityRow = entityFeaturePair.getLeft(); - Optional featureRow = entityFeaturePair.getRight(); - // Unpack feature field values and merge into entityValueMap - boolean isOutsideMaxAge = - checkOutsideMaxAge(featureSetRequest, entityRow, featureRow); - Map valueMap = - unpackValueMap(featureRow, featureSetRequest, isOutsideMaxAge); - entityValuesMap.get(entityRow).putAll(valueMap); + Span onlineRetrievalSpan = tracer.buildSpan("onlineRetrieval").start(); + if (onlineRetrievalSpan != null) { + onlineRetrievalSpan.setTag("entities", entityRows.size()); + onlineRetrievalSpan.setTag("feature sets", featureSetRequests.size()); + } + for (FeatureSetRequest featureSetRequest : featureSetRequests) { + // Pull feature rows for given entity rows from the feature/featureset specified in feature + // set request. + // from the configured online + List> featureRows = + retriever.getOnlineFeatures(entityRows, featureSetRequest); + // Check that feature row returned corresponds to a given entity row. + if (featureRows.size() != entityRows.size()) { + throw Status.INTERNAL + .withDescription( + "The no. of FeatureRow obtained from OnlineRetriever" + + "does not match no. of entityRow passed.") + .asRuntimeException(); + } - // Generate metadata for feature values and merge into entityFieldsMap - boolean isNotFound = featureRow.isEmpty(); - Map statusMap = - getMetadataMap(valueMap, isNotFound, isOutsideMaxAge); - entityStatusesMap.get(entityRow).putAll(statusMap); + Streams.zip(entityRows.stream(), featureRows.stream(), Pair::of) + .forEach( + entityFeaturePair -> { + EntityRow entityRow = entityFeaturePair.getLeft(); + Optional featureRow = entityFeaturePair.getRight(); + // Unpack feature field values and merge into entityValueMap + boolean isOutsideMaxAge = + checkOutsideMaxAge(featureSetRequest, entityRow, featureRow); + Map valueMap = + unpackValueMap(featureRow, featureSetRequest, isOutsideMaxAge); + entityValuesMap.get(entityRow).putAll(valueMap); - // Populate metrics/log request - populateCountMetrics(statusMap, featureSetRequest); - }); - populateRequestCountMetrics(featureSetRequest); - logFeatureRows.add(featureRows); - } - if (scope != null) { - logFeatureRowsTrace(scope, logFeatureRows, featureSetRequests); - } + // Generate metadata for feature values and merge into entityFieldsMap + boolean isNotFound = featureRow.isEmpty(); + Map statusMap = + getMetadataMap(valueMap, isNotFound, isOutsideMaxAge); + entityStatusesMap.get(entityRow).putAll(statusMap); - // Build response field values from entityValuesMap and entityStatusesMap - // Reponse field values should be in the same order as the entityRows provided by the user. - List fieldValuesList = - entityRows.stream() - .map( - entityRow -> { - return FieldValues.newBuilder() - .putAllFields(entityValuesMap.get(entityRow)) - .putAllStatuses(entityStatusesMap.get(entityRow)) - .build(); - }) - .collect(Collectors.toList()); - return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); + // Populate metrics/log request + populateCountMetrics(statusMap, featureSetRequest); + }); + populateRequestCountMetrics(featureSetRequest); + logFeatureRows.add(featureRows); } + if (onlineRetrievalSpan != null) { + onlineRetrievalSpan.finish(); + } + + // Build response field values from entityValuesMap and entityStatusesMap + // Reponse field values should be in the same order as the entityRows provided by the user. + List fieldValuesList = + entityRows.stream() + .map( + entityRow -> { + return FieldValues.newBuilder() + .putAllFields(entityValuesMap.get(entityRow)) + .putAllStatuses(entityStatusesMap.get(entityRow)) + .build(); + }) + .collect(Collectors.toList()); + return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); } /** @@ -254,40 +259,6 @@ private static boolean checkOutsideMaxAge( return timeDifference > maxAge.getSeconds(); } - private void logFeatureRowsTrace( - Scope scope, - List>> logFeatureRows, - List featureSetRequests) { - List> loggableFeatureRows = - Streams.zip( - logFeatureRows.stream(), - featureSetRequests.stream(), - (featureRows, featureSetRequest) -> { - FeatureRow.Builder nullFeatureRowBuilder = - FeatureRow.newBuilder() - .setFeatureSet( - FeatureSet.getFeatureSetStringRef(featureSetRequest.getSpec())); - for (FeatureReference featureReference : - featureSetRequest.getFeatureReferences()) { - nullFeatureRowBuilder.addFields( - Field.newBuilder().setName(featureReference.getName())); - } - - // log null feature row when feature row is empty - return featureRows.stream() - .map( - featureRow -> { - return (featureRow.isEmpty()) - ? nullFeatureRowBuilder.build() - : featureRow.get(); - }) - .collect(Collectors.toList()); - }) - .collect(Collectors.toList()); - - scope.span().log(ImmutableMap.of("event", "featureRows", "value", loggableFeatureRows)); - } - private void populateCountMetrics( Map statusMap, FeatureSetRequest featureSetRequest) { String project = featureSetRequest.getSpec().getProject();