diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cfe1acd3..858a56497 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Update Go version and base Ubuntu image. [#821](https://github.com/elastic/package-registry/pull/821) * Add support for "threat_intel" category. [#841](https://github.com/elastic/package-registry/pull/841) +* Instrument package registry with Prometheus metrics. [#827](https://github.com/elastic/package-registry/pull/827) ### Deprecated diff --git a/README.md b/README.md index c859ab92d..dfeda39c9 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Endpoints: * `/search`: Search for packages. By default returns all the most recent packages available. * `/categories`: List of the existing package categories and how many packages are in each category. * `/package/{name}/{version}`: Info about a package -* `/epr/{name}/{name}-{version}.tar.gz`: Download a package +* `/epr/{name}/{name}-{version}.zip`: Download a package Examples for each API endpoint can be found here: https://github.com/elastic/package-registry/tree/main/docs/api @@ -236,6 +236,19 @@ It will be listening in the given address. You can read more about this profiler and the available endpoints in the [pprof documentation](https://pkg.go.dev/net/http/pprof). +## Metrics + +Package registry can be instrumented to expose Prometheus metrics under `/metrics` endpoint. +By default this endpoint is disabled. + +To enable this instrumentation, the required address (host and port) where this endpoint needs +to run must be set using the parameter `metrics-address` (or the `EPR_METRICS_ADDRESS` environment variable). +For example: + +``` +package-registry --metrics-address 0.0.0.0:9000 +``` + ## Release New versions of the package registry need to be released from time to time. The following steps should be followed to create a new release: diff --git a/go.mod b/go.mod index d1c726e0f..f997d3ca6 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 github.com/magefile/mage v1.13.0 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.12.2 github.com/stretchr/testify v1.8.0 go.elastic.co/apm v1.15.0 go.elastic.co/apm/module/apmgorilla v1.15.0 @@ -28,6 +29,8 @@ require ( cloud.google.com/go/iam v0.3.0 // indirect cloud.google.com/go/pubsub v1.23.0 // indirect github.com/armon/go-radix v1.0.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/elastic/go-sysinfo v1.7.1 // indirect github.com/elastic/go-windows v1.0.1 // indirect @@ -41,8 +44,11 @@ require ( github.com/gorilla/handlers v1.5.1 // indirect github.com/jcchavezs/porto v0.3.0 // indirect github.com/kr/pretty v0.3.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pkg/xattr v0.4.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/sirupsen/logrus v1.8.1 // indirect diff --git a/go.sum b/go.sum index 813af8df4..364e48a52 100644 --- a/go.sum +++ b/go.sum @@ -71,14 +71,25 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -126,6 +137,14 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -177,6 +196,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -231,9 +251,19 @@ github.com/jcchavezs/porto v0.3.0/go.mod h1:fESH0gzDHiutHRdX2hv27ojnOVFco37hg1W6 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -244,6 +274,15 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magefile/mage v1.13.0 h1:XtLJl8bcCM7EFoO8FyH8XK3t7G5hQAeK+i4tq+veT9M= github.com/magefile/mage v1.13.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -252,8 +291,27 @@ github.com/pkg/xattr v0.4.7 h1:XoA3KzmFvyPlH4RwX5eMcgtzcaGBaSvgt3IoFQfbrmQ= github.com/pkg/xattr v0.4.7/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -262,10 +320,14 @@ github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBO github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -309,6 +371,7 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -355,6 +418,7 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVD golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -362,6 +426,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -388,6 +453,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -432,9 +498,12 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -446,6 +515,7 @@ golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -458,6 +528,8 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -474,6 +546,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -486,6 +559,7 @@ golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -754,14 +828,19 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/indexer.go b/indexer.go index 2568b3acb..8a91d547d 100644 --- a/indexer.go +++ b/indexer.go @@ -6,7 +6,9 @@ package main import ( "context" + "time" + "github.com/elastic/package-registry/metrics" "github.com/elastic/package-registry/packages" ) @@ -32,6 +34,8 @@ func (c CombinedIndexer) Init(ctx context.Context) error { } func (c CombinedIndexer) Get(ctx context.Context, opts *packages.GetOptions) (packages.Packages, error) { + start := time.Now() + defer metrics.StorageIndexerGetDurationSeconds.Observe(time.Since(start).Seconds()) var packages packages.Packages for _, indexer := range c { p, err := indexer.Get(ctx, opts) diff --git a/main.go b/main.go index 191045ec1..6b8201b2e 100644 --- a/main.go +++ b/main.go @@ -19,25 +19,30 @@ import ( gstorage "cloud.google.com/go/storage" "github.com/gorilla/mux" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "go.elastic.co/apm" "go.elastic.co/apm/module/apmgorilla" "go.uber.org/zap" ucfgYAML "github.com/elastic/go-ucfg/yaml" + "github.com/elastic/package-registry/metrics" "github.com/elastic/package-registry/packages" "github.com/elastic/package-registry/storage" "github.com/elastic/package-registry/util" ) const ( - serviceName = "package-registry" - version = "1.9.1" + serviceName = "package-registry" + version = "1.9.1" + defaultInstanceName = "localhost" ) var ( address string httpProfAddress string + metricsAddress string tlsCertFile string tlsKeyFile string @@ -45,6 +50,8 @@ var ( dryRun bool configPath string + printVersionInfo bool + featureStorageIndexer bool storageIndexerBucketInternal string storageEndpoint string @@ -59,7 +66,9 @@ var ( ) func init() { + flag.BoolVar(&printVersionInfo, "version", false, "Print Elastic Package Registry version") flag.StringVar(&address, "address", "localhost:8080", "Address of the package-registry service.") + flag.StringVar(&metricsAddress, "metrics-address", "", "Address to expose the Prometheus metrics.") flag.StringVar(&tlsCertFile, "tls-cert", "", "Path of the TLS certificate.") flag.StringVar(&tlsKeyFile, "tls-key", "", "Path of the TLS key.") flag.StringVar(&configPath, "config", "config.yml", "Path to the configuration file.") @@ -86,15 +95,27 @@ type Config struct { func main() { parseFlags() + if printVersionInfo { + fmt.Printf("Elastic Package Registry version %v\n", version) + os.Exit(0) + } + logger := util.Logger() defer logger.Sync() + config := mustLoadConfig(logger) + if dryRun { + logger.Info("Running dry-run mode") + _ = initIndexers(context.Background(), logger, config) + os.Exit(0) + } + logger.Info("Package registry started") defer logger.Info("Package registry stopped") initHttpProf(logger) - server := initServer(logger) + server := initServer(logger, config) go func() { err := runServer(server) if err != nil && err != http.ErrServerClosed { @@ -102,6 +123,8 @@ func main() { } }() + initMetricsServer(logger) + stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt, syscall.SIGTERM) <-stop @@ -126,17 +149,38 @@ func initHttpProf(logger *zap.Logger) { }() } -func initServer(logger *zap.Logger) *http.Server { - apmTracer := initAPMTracer(logger) - tx := apmTracer.StartTransaction("initServer", "backend.init") - defer tx.End() +func getHostname() string { + hostname, err := os.Hostname() + if err != nil { + return defaultInstanceName + } + return hostname +} - ctx := apm.ContextWithTransaction(context.TODO(), tx) +func initMetricsServer(logger *zap.Logger) { + if metricsAddress == "" { + return + } - config := mustLoadConfig(logger) + hostname := getHostname() + + metrics.ServiceInfo.With(prometheus.Labels{"version": version, "instance": hostname}).Set(1) + + logger.Info("Starting http metrics in " + metricsAddress) + go func() { + router := http.NewServeMux() + router.Handle("/metrics", promhttp.Handler()) + err := http.ListenAndServe(metricsAddress, router) + if err != nil { + logger.Fatal("failed to start Prometheus metrics endpoint", zap.Error(err)) + } + }() +} + +func initIndexers(ctx context.Context, logger *zap.Logger, config *Config) CombinedIndexer { packagesBasePaths := getPackagesBasePaths(config) - var indexers []Indexer + var indexers CombinedIndexer if featureStorageIndexer { storageClient, err := gstorage.NewClient(ctx) if err != nil { @@ -154,10 +198,17 @@ func initServer(logger *zap.Logger) *http.Server { combinedIndexer := NewCombinedIndexer(indexers...) ensurePackagesAvailable(ctx, logger, combinedIndexer) - // If -dry-run=true is set, service stops here after validation - if dryRun { - os.Exit(0) - } + return combinedIndexer +} + +func initServer(logger *zap.Logger, config *Config) *http.Server { + apmTracer := initAPMTracer(logger) + tx := apmTracer.StartTransaction("initServer", "backend.init") + defer tx.End() + + ctx := apm.ContextWithTransaction(context.TODO(), tx) + + combinedIndexer := initIndexers(ctx, logger, config) router := mustLoadRouter(logger, config, combinedIndexer) apmgorilla.Instrument(router, apmgorilla.WithTracer(apmTracer)) @@ -223,6 +274,8 @@ func getPackagesBasePaths(config *Config) []string { func printConfig(logger *zap.Logger, config *Config) { logger.Info("Packages paths: " + strings.Join(config.PackagePaths, ", ")) + logger.Info("Cache time for /: " + config.CacheTimeIndex.String()) + logger.Info("Cache time for /index.json: " + config.CacheTimeIndex.String()) logger.Info("Cache time for /search: " + config.CacheTimeSearch.String()) logger.Info("Cache time for /categories: " + config.CacheTimeCategories.String()) logger.Info("Cache time for all others: " + config.CacheTimeCatchAll.String()) @@ -244,6 +297,7 @@ func ensurePackagesAvailable(ctx context.Context, logger *zap.Logger, indexer In } logger.Info(fmt.Sprintf("%v package manifests loaded", len(packages))) + metrics.NumberIndexedPackages.Set(float64(len(packages))) } func mustLoadRouter(logger *zap.Logger, config *Config, indexer Indexer) *mux.Router { @@ -270,7 +324,6 @@ func getRouter(logger *zap.Logger, config *Config, indexer Indexer) (*mux.Router staticHandler := staticHandler(indexer, config.CacheTimeCatchAll) router := mux.NewRouter().StrictSlash(true) - router.HandleFunc("/", indexHandlerFunc) router.HandleFunc("/index.json", indexHandlerFunc) router.HandleFunc("/search", searchHandler(indexer, config.CacheTimeSearch)) @@ -282,6 +335,9 @@ func getRouter(logger *zap.Logger, config *Config, indexer Indexer) (*mux.Router router.HandleFunc(packageIndexRouterPath, packageIndexHandler) router.HandleFunc(staticRouterPath, staticHandler) router.Use(util.LoggingMiddleware(logger)) + if metricsAddress != "" { + router.Use(metrics.MetricsMiddleware()) + } router.NotFoundHandler = http.Handler(notFoundHandler(fmt.Errorf("404 page not found"))) return router, nil } diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 000000000..aafc95a62 --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,135 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const metricsNamespace = "epr" + +var ServiceInfo = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Name: "service_info", + Help: "Version information about this binary.", + }, + []string{"version", "instance"}, +) + +var ( + NumberIndexedPackages = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Name: "number_indexed_packages", + Help: "A gauge for number of indexed packages.", + }) + + StorageRequestsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Name: "storage_requests_total", + Help: "A counter for requests performed to the storage.", + }, + []string{"location", "component"}, + ) + + StorageIndexerUpdateIndexSuccessTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Name: "storage_indexer_update_index_success_total", + Help: "A counter for updates of the cursor.", + }, + ) + + StorageIndexerUpdateIndexErrorsTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Name: "storage_indexer_update_index_error_total", + Help: "A counter for all the update index processes that finished with error.", + }, + ) + + StorageIndexerUpdateIndexDurationSeconds = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Name: "storage_indexer_update_index_duration_seconds", + Help: "A histogram of latencies for update index processes run by the indexer.", + }, + ) + + StorageIndexerGetDurationSeconds = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Name: "storage_indexer_get_duration_seconds", + Help: "A histogram of latencies for get processes run by the indexer.", + }, + ) +) + +var ( + httpInFlightRequests = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Name: "in_flight_requests", + Help: "A gauge of requests currently being served by the http server.", + }) + + httpRequestsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Name: "http_requests_total", + Help: "A counter for requests to the http server.", + }, + []string{"code", "method", "path"}, + ) + + httpRequestDurationSeconds = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Name: "http_request_duration_seconds", + Help: "A histogram of latencies for requests to the http server.", + }, + []string{"code", "method", "path"}, + ) + + httpRequestSizeBytes = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Name: "http_request_size_bytes", + Help: "A histogram of sizes of requests to the http server.", + Buckets: []float64{ + 16, + 32, + 64, + 128, + 256, + 512, + 1024, /* 1KiB */ + 64 * 1024, /* 64KiB */ + 256 * 1024, /* 256KiB */ + }, + }, + []string{"code", "method", "path"}, + ) + + httpResponseSizeBytes = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Name: "http_response_size_bytes", + Help: "A histogram of response sizes for requests to the http server.", + Buckets: []float64{ + 16, + 32, + 64, + 128, + 256, + 512, + 1024, /* 1KiB */ + 64 * 1024, /* 64KiB */ + 256 * 1024, /* 256KiB */ + }, + }, + []string{"code", "method", "path"}, + ) +) diff --git a/metrics/middleware.go b/metrics/middleware.go new file mode 100644 index 000000000..3e7d3740d --- /dev/null +++ b/metrics/middleware.go @@ -0,0 +1,52 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package metrics + +import ( + "net/http" + + "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// MetricsMiddleware is a middleware used to measure every request received +func MetricsMiddleware() mux.MiddlewareFunc { + // Rergister all metrics + prometheus.MustRegister(ServiceInfo) + + prometheus.MustRegister(httpInFlightRequests) + prometheus.MustRegister(httpRequestsTotal) + prometheus.MustRegister(httpRequestDurationSeconds) + prometheus.MustRegister(httpRequestSizeBytes) + prometheus.MustRegister(httpResponseSizeBytes) + + prometheus.MustRegister(NumberIndexedPackages) + prometheus.MustRegister(StorageRequestsTotal) + prometheus.MustRegister(StorageIndexerGetDurationSeconds) + prometheus.MustRegister(StorageIndexerUpdateIndexDurationSeconds) + prometheus.MustRegister(StorageIndexerUpdateIndexSuccessTotal) + prometheus.MustRegister(StorageIndexerUpdateIndexErrorsTotal) + + return func(next http.Handler) http.Handler { + handler := next + + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + route := mux.CurrentRoute(req) + path, err := route.GetPathTemplate() + if err != nil { + path = "unknown" + } + labels := prometheus.Labels{"path": path} + + handler = promhttp.InstrumentHandlerCounter(httpRequestsTotal.MustCurryWith(labels), handler) + handler = promhttp.InstrumentHandlerDuration(httpRequestDurationSeconds.MustCurryWith(labels), handler) + handler = promhttp.InstrumentHandlerInFlight(httpInFlightRequests, handler) + handler = promhttp.InstrumentHandlerRequestSize(httpRequestSizeBytes.MustCurryWith(labels), handler) + handler = promhttp.InstrumentHandlerResponseSize(httpResponseSizeBytes.MustCurryWith(labels), handler) + handler.ServeHTTP(w, req) + }) + } +} diff --git a/packages/http.go b/packages/http.go index 71c5cff7c..a2583cad8 100644 --- a/packages/http.go +++ b/packages/http.go @@ -11,7 +11,10 @@ import ( "go.elastic.co/apm" "go.uber.org/zap" + "github.com/prometheus/client_golang/prometheus" + "github.com/elastic/package-registry/archiver" + "github.com/elastic/package-registry/metrics" "github.com/elastic/package-registry/util" ) @@ -22,9 +25,11 @@ func ServePackage(w http.ResponseWriter, r *http.Request, p *Package) { if p.RemoteResolver() != nil { p.RemoteResolver().RedirectArtifactsHandler(w, r, p) + metrics.StorageRequestsTotal.With(prometheus.Labels{"location": "remote", "component": "artifacts"}).Inc() return } serveLocalPackage(w, r, p, p.BasePath) + metrics.StorageRequestsTotal.With(prometheus.Labels{"location": "local", "component": "artifacts"}).Inc() } // ServePackageSignature is used by signaturesHandler to serve signatures. @@ -34,9 +39,11 @@ func ServePackageSignature(w http.ResponseWriter, r *http.Request, p *Package) { if p.RemoteResolver() != nil { p.RemoteResolver().RedirectSignaturesHandler(w, r, p) + metrics.StorageRequestsTotal.With(prometheus.Labels{"location": "remote", "component": "signatures"}).Inc() return } serveLocalPackage(w, r, p, p.BasePath+".sig") + metrics.StorageRequestsTotal.With(prometheus.Labels{"location": "local", "component": "signatures"}).Inc() } func serveLocalPackage(w http.ResponseWriter, r *http.Request, p *Package, packagePath string) { @@ -75,6 +82,7 @@ func ServePackageResource(w http.ResponseWriter, r *http.Request, p *Package, pa if p.RemoteResolver() != nil { p.RemoteResolver().RedirectStaticHandler(w, r, p, packageFilePath) + metrics.StorageRequestsTotal.With(prometheus.Labels{"location": "remote", "component": "static"}).Inc() return } @@ -111,4 +119,5 @@ func ServePackageResource(w http.ResponseWriter, r *http.Request, p *Package, pa defer f.Close() http.ServeContent(w, r, packageFilePath, stat.ModTime(), f) + metrics.StorageRequestsTotal.With(prometheus.Labels{"location": "local", "component": "static"}).Inc() } diff --git a/storage/indexer.go b/storage/indexer.go index 3f968869e..909bed777 100644 --- a/storage/indexer.go +++ b/storage/indexer.go @@ -16,6 +16,7 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" + "github.com/elastic/package-registry/metrics" "github.com/elastic/package-registry/packages" "github.com/elastic/package-registry/util" ) @@ -128,14 +129,18 @@ func (i *Indexer) watchIndices(ctx context.Context) { func (i *Indexer) updateIndex(ctx context.Context) error { logger := util.Logger() logger.Debug("Update indices") + start := time.Now() + defer metrics.StorageIndexerUpdateIndexDurationSeconds.Observe(time.Since(start).Seconds()) bucketName, rootStoragePath, err := extractBucketNameFromURL(i.options.PackageStorageBucketInternal) if err != nil { + metrics.StorageIndexerUpdateIndexErrorsTotal.Inc() return errors.Wrapf(err, "can't extract bucket name from URL (url: %s)", i.options.PackageStorageBucketInternal) } storageCursor, err := loadCursor(ctx, i.storageClient, bucketName, rootStoragePath) if err != nil { + metrics.StorageIndexerUpdateIndexErrorsTotal.Inc() return errors.Wrap(err, "can't load latest cursor") } @@ -147,12 +152,14 @@ func (i *Indexer) updateIndex(ctx context.Context) error { anIndex, err := loadSearchIndexAll(ctx, i.storageClient, bucketName, rootStoragePath, *storageCursor) if err != nil { + metrics.StorageIndexerUpdateIndexErrorsTotal.Inc() return errors.Wrapf(err, "can't load the search-index-all index content") } logger.Info("Downloaded new search-index-all index", zap.String("index.packages.size", fmt.Sprintf("%d", len(anIndex.Packages)))) refreshedList, err := i.transformSearchIndexAllToPackages(*anIndex) if err != nil { + metrics.StorageIndexerUpdateIndexErrorsTotal.Inc() return errors.Wrap(err, "can't transform the search-index-all") } @@ -160,6 +167,8 @@ func (i *Indexer) updateIndex(ctx context.Context) error { defer i.m.Unlock() i.cursor = storageCursor.Current i.packageList = refreshedList + metrics.StorageIndexerUpdateIndexSuccessTotal.Inc() + metrics.NumberIndexedPackages.Set(float64(len(i.packageList))) return nil } diff --git a/util/logging.go b/util/logging.go index 0995b2e51..4b74edc77 100644 --- a/util/logging.go +++ b/util/logging.go @@ -97,13 +97,13 @@ func LoggingMiddleware(logger *zap.Logger) mux.MiddlewareFunc { logger = logger.Named("http").WithOptions(zap.WithCaller(false)) return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Do not log requests to the health endpoint - if r.RequestURI == "/health" { + switch r.RequestURI { + case "/health": + // Do not log requests to these endpoints next.ServeHTTP(w, r) - return + default: + logRequest(logger, next, w, r) } - - logRequest(logger, next, w, r) }) } }