Merge branch 'master' into linasn/dbnode-metrics

* master: (26 commits)
  [dbnode][m3ninx] Rename doc.Document -> doc.Metadata (#3062)
  Don't attempt to lint genny gen'd code (#3063)
  [coordinator] Add ability to turn off rule matching cache (#3059)
  [r2ctl] Bump urijs from 1.19.1 to 1.19.5 in /src/ctl/ui (#3055)
  [aggregator/client] Metric for dropped metrics (#3054)
  [query] Add Graphite support for ** with metric path selectors (#3020)
  [tools] Fix failing linter (#3052)
  [query] implemented Graphite's `powSeries` function (#3038)
  [coordinator] Add Graphite Carbon rewrite cleanup integration test (#3049)
  [coordinator] Add Graphite rewrite cleanup directive for cleansing incoming metrics (#3047)
  [coordinator] Add Graphite Carbon ingest contains rule matcher for fast match (#3046)
  [query] Update Graphite quoted strings to just escape quotes and a flag for previous escape behavior (#3022)
  [query] Graphite match regexp all values for __g0__ tag matcher instead of match field (#3021)
  [coordinator] Add Carbon ingest latency metrics (#3045)
  [aggregator] Fix panic on invalid unaggregated payloads (#3041)
  [coordinator] Ignore rollup rule storage policies when deciding to utilize auto-mapping rules (#3044)
  [dbnode] Support checkpointing in cold flusher and preparing index writer per block (#3040)
  [dbnode] Client borrow connection API (#3019)
  [DOCS] Add back FAQ and troubleshooting sections (#3029)
  [DOCS] SEO and config changes (#3024)
  ...

.golangci.yml

@@ -34,7 +34,8 @@ run:
   # won't be reported. Default value is empty list, but there is
   # no need to include all autogenerated files, we confidently recognize
   # autogenerated files. If it's not please let us know.
-  skip-files: []
+  skip-files:
+    - ".*_gen.go$"
 
   # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
   # If invoked with -mod=readonly, the go command is disallowed from the implicit
@@ -113,7 +114,7 @@ linters-settings:
   lll:
     # max line length, lines longer will be reported. Default is 120.
     # '\t' is counted as 1 character by default, and can be changed with the tab-width option
-    line-length: 100
+    line-length: 120
     # tab width in spaces. Default to 1.
     tab-width: 1
   unused:

go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
 	github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d // indirect
 	github.com/DataDog/datadog-go v3.7.1+incompatible // indirect
-	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/MichaelTJones/pcg v0.0.0-20180122055547-df440c6ed7ed
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
@@ -23,7 +22,6 @@ require (
 	github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect
 	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/fatih/color v1.10.0 // indirect
 	github.com/fortytw2/leaktest v1.2.1-0.20180901000122-b433bbd6d743
 	github.com/fossas/fossa-cli v1.0.30
 	github.com/garethr/kubeval v0.0.0-20180821130434-c44f5193dc94
@@ -36,6 +34,7 @@ require (
 	github.com/golang/mock v1.4.4
 	github.com/golang/protobuf v1.4.2
 	github.com/golang/snappy v0.0.1
+	github.com/golangci/golangci-lint v1.33.0
 	github.com/google/go-cmp v0.5.2
 	github.com/google/go-jsonnet v0.16.0
 	github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f // indirect
@@ -47,7 +46,6 @@ require (
 	github.com/influxdata/influxdb v1.7.7
 	github.com/jhump/protoreflect v1.6.1
 	github.com/json-iterator/go v1.1.9
-	github.com/kr/text v0.2.0 // indirect
 	github.com/leanovate/gopter v0.2.8
 	github.com/lib/pq v1.6.0 // indirect
 	github.com/lightstep/lightstep-tracer-go v0.18.1
@@ -66,9 +64,6 @@ require (
 	github.com/m3dbx/vellum v0.0.0-20201119082309-5b47f7a70f69
 	github.com/mauricelam/genny v0.0.0-20180903214747-eb2c5232c885
 	github.com/mjibson/esc v0.1.0
-	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
-	github.com/onsi/ginkgo v1.14.1 // indirect
-	github.com/onsi/gomega v1.10.2 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opencontainers/runc v0.1.1 // indirect
 	github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9
@@ -92,7 +87,6 @@ require (
 	github.com/rveen/ogdl v0.0.0-20200522080342-eeeda1a978e7 // indirect
 	github.com/sergi/go-diff v1.1.0
 	github.com/shirou/gopsutil v2.20.5+incompatible // indirect
-	github.com/sirupsen/logrus v1.7.0 // indirect
 	github.com/spf13/cast v1.3.1-0.20190531151931-f31dc0aaab5a // indirect
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -123,7 +117,6 @@ require (
 	golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634
 	golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752
 	google.golang.org/grpc v1.29.1
-	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/go-ini/ini.v1 v1.57.0 // indirect
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 	gopkg.in/go-playground/validator.v9 v9.7.0
@@ -133,7 +126,6 @@ require (
 	gopkg.in/vmihailenco/msgpack.v2 v2.8.3
 	gopkg.in/yaml.v2 v2.3.0
 	gotest.tools v2.2.0+incompatible
-	honnef.co/go/tools v0.0.1-2020.1.6 // indirect
 )
 
 // branch 0.9.3-pool-read-binary-3

netlify.toml

@@ -30,17 +30,17 @@
 
 [[redirects]]
   from = "/docs/how_to/single_node"
-  to = "docs/quickstart"
+  to = "/docs/quickstart"
 
 [[redirects]]
-  from="docs/how_to/kubernetes"
-  to="/docs/getting_started/kubernetes_cluster"
+  from="/docs/how_to/kubernetes"
+  to="/docs/cluster/kubernetes_cluster"
 
 [[redirects]]
-  from="docs/quickstart/kubernetes"
+  from="/docs/quickstart/kubernetes"
   to="/docs/cluster/kubernetes_cluster"
 
 # TODO: Fix this with new content type
 [[redirects]]
   from = "/talks"
-  to = "docs/overview/media"
+  to = "/docs/overview/media"

scripts/docker-integration-tests/carbon/expected/dbaz.json

@@ -0,0 +1,9 @@
+[
+  {
+    "id": "d.bar.baz",
+    "text": "baz",
+    "leaf": 1,
+    "expandable": 0,
+    "allowChildren": 0
+  }
+]

scripts/docker-integration-tests/carbon/expected/ebaz.json

@@ -0,0 +1,9 @@
+[
+  {
+    "id": "e.bar.baz",
+    "text": "baz",
+    "leaf": 1,
+    "expandable": 0,
+    "allowChildren": 0
+  }
+]

scripts/docker-integration-tests/carbon/expected/fbaz.json

@@ -0,0 +1,9 @@
+[
+  {
+    "id": "f.bar.baz",
+    "text": "baz",
+    "leaf": 1,
+    "expandable": 0,
+    "allowChildren": 0
+  }
+]

scripts/docker-integration-tests/carbon/expected/gbaz.json

@@ -0,0 +1,9 @@
+[
+  {
+    "id": "g.bar.baz",
+    "text": "baz",
+    "leaf": 1,
+    "expandable": 0,
+    "allowChildren": 0
+  }
+]

scripts/docker-integration-tests/carbon/expected/hbarbaz.json

@@ -0,0 +1,9 @@
+[
+  {
+    "id": "h.bar_baz",
+    "text": "bar_baz",
+    "leaf": 1,
+    "expandable": 0,
+    "allowChildren": 0
+  }
+]

scripts/docker-integration-tests/carbon/expected/ibarbaz.json

@@ -0,0 +1,9 @@
+[
+  {
+    "id": "i.bar_baz",
+    "text": "bar_baz",
+    "leaf": 1,
+    "expandable": 0,
+    "allowChildren": 0
+  }
+]

scripts/docker-integration-tests/carbon/m3coordinator.yml

@@ -14,6 +14,8 @@ clusters:
 carbon:
   ingester:
     listenAddress: "0.0.0.0:7204"
+    rewrite:
+      cleanup: true
     rules:
       - pattern: .*min.aggregate.*
         aggregation:

scripts/docker-integration-tests/carbon/test.sh

@@ -53,7 +53,7 @@ t=$(date +%s)
 echo "foo.min.aggregate.baz 41 $t" | nc 0.0.0.0 7204
 echo "foo.min.aggregate.baz 42 $t" | nc 0.0.0.0 7204
 echo "Attempting to read min aggregated carbon metric"
-ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon foo.min.aggregate.baz 41
+ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.aggregate.baz' 41"
 
 echo "Writing out a carbon metric that should not be aggregated"
 t=$(date +%s)
@@ -64,41 +64,76 @@ t=$(date +%s)
 echo "foo.min.already-aggregated.baz 42 $t" | nc 0.0.0.0 7204
 echo "foo.min.already-aggregated.baz 43 $t" | nc 0.0.0.0 7204
 echo "Attempting to read unaggregated carbon metric"
-ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon foo.min.already-aggregated.baz 43
+ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.already-aggregated.baz' 43"
 
 echo "Writing out a carbon metric that should should use the default mean aggregation"
 t=$(date +%s)
 # Mean of 10 and 20 is 15. Same comment as the min aggregation test above.
 echo "foo.min.catch-all.baz 10 $t" | nc 0.0.0.0 7204
 echo "foo.min.catch-all.baz 20 $t" | nc 0.0.0.0 7204
 echo "Attempting to read mean aggregated carbon metric"
-ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon foo.min.catch-all.baz 15
+ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.catch-all.baz' 15"
 
 # Test writing and reading IDs with colons in them.
 t=$(date +%s)
 echo "foo.bar:baz.qux 42 $t" | nc 0.0.0.0 7204
-ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon 'foo.bar:*.*' 42
+ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.bar:*.*' 42"
 
 # Test writing and reading IDs with a single element.
 t=$(date +%s)
 echo "quail 42 $t" | nc 0.0.0.0 7204
-ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon 'quail' 42
+ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'quail' 42"
 
+# Test using "**" in queries
 t=$(date +%s)
+echo "qux.pos1-a.pos2-0 1 $t" | nc 0.0.0.0 7204
+echo "qux.pos1-a.pos2-1 1 $t" | nc 0.0.0.0 7204
+echo "qux.pos1-b.pos2-0 1 $t" | nc 0.0.0.0 7204
+echo "qux.pos1-b.pos2-1 1 $t" | nc 0.0.0.0 7204
+echo "qux.pos1-c.pos2-0 1 $t" | nc 0.0.0.0 7204
+echo "qux.pos1-c.pos2-1 1 $t" | nc 0.0.0.0 7204
+ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(qux**)' 6"
+ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(qux.pos1-a**)' 2"
+ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos1-a**)' 2"
+ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos2-1**)' 3"
+ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos2-1)' 3"
+
+t=$(date +%s)
+
+# Test basic cases
 echo "a 0 $t"             | nc 0.0.0.0 7204
 echo "a.bar 0 $t"         | nc 0.0.0.0 7204
 echo "a.biz 0 $t"         | nc 0.0.0.0 7204
 echo "a.biz.cake 0 $t"    | nc 0.0.0.0 7204
 echo "a.bar.caw.daz 0 $t" | nc 0.0.0.0 7204
 echo "a.bag 0 $t"         | nc 0.0.0.0 7204
 echo "c:bar.c:baz 0 $t"   | nc 0.0.0.0 7204
-ATTEMPTS=10 TIMEOUT=1 retry_with_backoff find_carbon a* a.json
-ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.b* a.b.json
-ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.ba[rg] a.ba.json
-ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.b*.c* a.b.c.json
-ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.b*.caw.* a.b.c.d.json
-ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon x none.json
-ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.d none.json
-ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon *.*.*.*.* none.json
-ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon c:* cbar.json
-ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon c:bar.* cbaz.json
+
+# Test rewrite multiple dots
+echo "d..bar.baz 0 $t"    | nc 0.0.0.0 7204
+echo "e.bar...baz 0 $t"   | nc 0.0.0.0 7204
+
+# Test rewrite leading or trailing dots
+echo "..f.bar.baz 0 $t"   | nc 0.0.0.0 7204
+echo "g.bar.baz.. 0 $t"   | nc 0.0.0.0 7204
+
+# Test rewrite bad chars
+echo "h.bar@@baz 0 $t"    | nc 0.0.0.0 7204
+echo "i.bar!!baz 0 $t"    | nc 0.0.0.0 7204
+
+ATTEMPTS=10 TIMEOUT=1 retry_with_backoff "find_carbon 'a*' a.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*' ab.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.ba[rg]' aba.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*.c*' abc.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*.caw.*' abcd.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'x' none.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.d' none.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon '*.*.*.*.*' none.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'c:*' cbar.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'c:bar.*' cbaz.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'd.bar.*' dbaz.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'e.bar.*' ebaz.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'f.bar.*' fbaz.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'g.bar.*' gbaz.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'h.bar*' hbarbaz.json"
+ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'i.bar*' ibarbaz.json"

scripts/docker-integration-tests/coordinator_noop/test.sh

@@ -43,7 +43,7 @@ if ! curl -vvvsSf localhost:7201/api/v1/services/m3coordinator/placement; then
   exit 1
 fi
 
-QUERY_EXP='{"error":"operation not valid for noop client"}'
+QUERY_EXP='{"status":"error","error":"operation not valid for noop client"}'
 RES=$(curl "localhost:7201/m3query/api/v1/query_range?start=$(date '+%s')&end=$(date '+%s')&step=10&query=foo")
 if [[ "$RES" != "$QUERY_EXP" ]]; then
   echo "Expected resp '$QUERY_EXP', GOT '$RES'"

site/config/_default/config.toml

@@ -32,7 +32,6 @@
     target = "archetypes"
 
 ignoreFiles = [ "\\.ttf$", "\\.woff$", "\\.woff2$", "\\.eot$" ]
-enableRobotsTXT = true
 metaDataFormat = "yaml"
 # theme = "docs-theme"
 # baseURL = "/"
@@ -44,8 +43,8 @@ defaultContentLanguageInSubdir = true
 
 [languages]
 [languages.en]
-title = "Kubernetes"
-description = "Production-Grade Container Orchestration"
+title = "M3 Documentation"
+description = "M3 is a Prometheus compatible, easy to adopt metrics engine that provides visibility for some of the world’s largest brands."
 languageName ="English"
 # Weight used for sorting.
 weight = 1
@@ -110,9 +109,10 @@ offlineSearch = false
   # Prefix URL to edit current page. Will display an "Edit this page" button on top right hand corner of every page. 
   # Useful to give opportunity to people to create merge request for your doc.
   # See the config.toml file from this documentation site to have an example.
-  # TODO: pattern to branch?
-  # TODO: bring back
-  # editURL = "https://github.com/m3db/m3/tree/master/site/content/"
+
+  helpBlock = true
+  editURL = "https://github.com/m3db/m3/tree/master/site/content/"
+
   # Author of the site, will be used in meta information
   author = "The M3 team and community"
   # Description of the site, will be used in meta information

site/config/production/config.toml

@@ -52,6 +52,7 @@ ignoreFiles = [ "\\.ttf$", "\\.woff$", "\\.woff2$", "\\.eot$" ]
 languageCode = "en-US"
 defaultContentLanguage = "en"
 # staticDir = ["static"]
+enableRobotsTXT = true
 
 metaDataFormat = "yaml"
 defaultContentLanguageInSubdir= true
@@ -101,11 +102,7 @@ offlineSearch = false
   # Prefix URL to edit current page. Will display an "Edit this page" button on top right hand corner of every page. 
   # Useful to give opportunity to people to create merge request for your doc.
   # See the config.toml file from this documentation site to have an example.
-  # TODO: pattern to branch?
-  # TODO: bring back
-  # editURL = "https://github.com/m3db/m3/tree/master/site/content/"
-  # Shows a checkmark for visited pages on the menu
-  showVisitedLinks = false
+
   # Disable search function. It will hide search bar
   disableSearch = false
   # Javascript and CSS cache are automatically busted when new version of site is generated. 
@@ -124,7 +121,7 @@ offlineSearch = false
   # Order sections in menu by "weight" or "title". Default to "weight"
   ordersectionsby = "weight"
   # Change default color scheme with a variant one. Can be "red", "blue", "green".
-  themeVariant = "blue"
+  # themeVariant = "blue"
   twitter = "m3db_io"
   disableHomeIcon = true
 

site/content/cluster/binaries_cluster.md

@@ -1,10 +1,9 @@
 ---
 linktitle: "Binaries"
+title: Creating an M3 Cluster with Binaries
 weight: 2
 ---
 
-# Creating an M3 Cluster with Binaries
-
 This guide shows you the steps involved in creating an M3 cluster using M3 binaries, typically you would automate this with infrastructure as code tools such as Terraform or [Kubernetes](/docs/operator).
 
 {{% notice note %}}

site/content/cluster/kubernetes_cluster.md

@@ -1,10 +1,9 @@
 ---
 linktitle: "Kubernetes"
+title: Creating an M3 Cluster with Kubernetes
 weight: 1
 ---
 
-# Creating an M3 Cluster with Kubernetes
-
 This guide shows you how to create an M3 cluster of 3 nodes, designed to run locally on the same machine. It is designed to show you how M3 and Kubernetes can work together, but not as a production example.
 
 {{% notice note %}}