Merge branch 'v0.6.0'

CHANGELOG.md

@@ -1,6 +1,32 @@
-# Release 0.5.1 #
+# Release 0.6.0 #
+
+## Incompatible changes ##
+* aggregation options now have separate section. Changed parameters:
+    * `metrics.aggregation-mode` -> `aggregation.mode`
+    * `metrics.aggregation-threads` -> `aggregation.threads`
+    * `metrics.update-counter-threshold` -> `aggregate.update-count-threshold` (note `count` instead of `counter`)
+
+* a list of particular aggregates is now customizable for ms-type metrics
+* aggregate naming is now customizable through `prefix-replacements`, `tag-replacements` and `postfix-replacements` in `aggregate` section
+
+* update counter is now an aggregate. Options changed:
+    * `metrics.update-counter-prefix` moved to `prefix-replacements` naturally
+    * `metrics.update-counter-suffix` moved to `postfix-replacements` naturally
 
-* fixed bug with parallel aggregation not working in separate thread pool mode
+## Major changes ##
+* graphite-style tags are now supported:
+    * aggregates can be placed in tags instead of postfixes (see `aggregate.mode` parameter)
+    * a new option `metrics.create-untagged-copy` is available to create a copy of untagged metric for compatibility
+      the option is false by default and needs to be enabled explicitly
+* timestamp rounding option is available as `aggregate.round-timestamp` to better match aggregation delays with rounding at receiver
+
+See config.toml for configuration instructions
+
+## Minor changes
+* more unit testing
+
+# Release 0.5.1 #
+* Fixed bug where parallel aggregation was panicking if turned on
 
 # Release 0.5.0 #
 

Cargo.toml

@@ -1,11 +1,15 @@
 [package]
 name = "bioyino"
-version = "0.5.1"
+version = "0.6.0"
 authors = ["Sergey Noskov aka Albibek <albibek@gmail.com>"]
 description = "StatsD-compatible, high-performance, fault-tolerant metric aggregator"
 edition = "2018"
 build = "build.rs"
 
+[profile.release]
+lto = true
+overflow-checks = false
+
 [dependencies]
 lazy_static="^1.1"
 clap="^2.31"
@@ -30,14 +34,16 @@ slog="^2.4"
 slog-term="^2.4"
 slog-async="^2.3"
 slog-scope="^4.0"
+slog-stdlog="^4.0"
+log="^0.4"
 toml="^0.5"
 ftoa = "^0.1"
 capnp = "^0.10"
 capnp-futures = "^0.10"
 raft-tokio = { git = "https://github.com/Albibek/raft-tokio" }
 rand = "^0.6"
 rayon = "^1.0"
-bioyino-metric = "^0.1"
+bioyino-metric = "^0.2"
 
 [build-dependencies]
 capnpc = "^0.10"

README.md

@@ -1,3 +1,5 @@
+# This branch is in development now. Please do not use it yet.
+
 # Bioyino #
 The StatsD server written in Rust
 

config.toml

@@ -26,47 +26,14 @@ stats-prefix = "resources.monitoring.bioyino"
 # What consensus to use: "consul", "internal" or "none"
 consensus = "none"
 
-[metrics]
-# Should we provide metrics that update more than update-counter-threshold times diring aggregation interval
-count-updates = true
-
-# Prefix for metric update statistics (no trailing dot!)
-update-counter-prefix = "resources.monitoring.bioyino.updates"
-
-# Suffix for metric update statistics (no leading dot!)
-update-counter-suffix = ""
-
-# Minimal update counter to be reported
-update-counter-threshold = 200
-
-# Aggregation can be made in two ways giving a tradeoff between the aggregation speed and
-# the probability of loosing metrics. See doc/aggregation.md for more information.
-# Available variants here: "single", "common", "separate"
-#aggregation-mode = "single"
-
-# Number of threads to use for aggregation in "separate" mode
-#aggregation-threads = (unset by default)
-
-# Process buffers from different hosts separately, this gives more guarantee to parse
-# metrics from different hosts correctly. Can play bad if lots of metrics is received from a single host, set
-# it to false if you have such use case
-# consistent-parsing = true
-
-# Log all buffers being dropped due to parsing errors. Can be very spammy.
-# log-parse-errors = false
-
-# Size of buffer that parser considers invalid. Used to avoid DoS attacks on parser.
-# Increase this if you have metrics taking more than 1000 bytes
-# max-unparsed-buffer = 1000
-
 [carbon]
 
 # IP and port of the carbon-protocol backend to send aggregated data to
 address = "127.0.0.1:2003"
 
 # Address to bind carbon client to when connecting
 # default: not specified, so no bind happens
-#bind_address = "127.0.0.1:2003"
+#bind-address = "127.0.0.1:2003"
 
 # How often to send metrics to carbon backend, ms
 interval = 30000
@@ -145,6 +112,84 @@ nodes = []
 # Interval to send snapshots to nodes, ms
 snapshot-interval = 1000
 
+[metrics]
+# Process buffers from different hosts separately, this gives more guarantee to parse
+# metrics from different hosts correctly. Can play bad if lots of metrics is received from a single host, set
+# it to false if you have such use case
+# consistent-parsing = true
+
+# Log all buffers being dropped due to parsing errors. Can be very spammy.
+# log-parse-errors = false
+
+# Size of buffer that parser considers invalid. Used to avoid DoS attacks on parser.
+# Increase this if you have metrics taking more than 1000 bytes
+# max-unparsed-buffer = 10000
+
+# Size of tags part of a metric (after semicolon character, not including the leading semicolon itself)
+# max-tags-len = 9000
+
+# Since tagged metric becomes a totally different metric in many systems the timeseries for such metrics
+# can be broken. To avoid this for the time the metrics is adopted, this option allows to create a copy of
+# such tagged metric, but without tags
+# Please note that this is effectively the double amount of memory and processing times
+# create-untagged-copy = false
+
+[aggregation]
+# round timestamp to interval of aggregation
+# possible values:
+# "up" - round to uppor bound of interval
+# "down" - round to lower bound
+# "no" - use exact timestamp, i.e. no rounding
+# round-timestamp = "no"
+
+# the threading mode of aggregation:
+# single - in a single thread
+# common - in the general thread pool along with parsing ang aggregating metrics
+# separate - in a separate thread pool
+mode = "single"
+
+# available values: smart (explained below), name, tag, both
+# smart means uto detection based on existence of tags in the metric,
+# so, for example sum for `some.metric` will be aggregated as `some.metric.sum`
+# but for `some.metric;tag=value` it will automatically become `some.metric;tag=value;aggregate=sum`
+destination = "smart"
+
+# updates aggregate is usually a debug value than some real one
+# also thi is the only one used for every type of metric, not only ms
+# so it's reasonable to avoid receiving a doubled amount of all metrics
+# This option allows to receive only metrics updated too often
+update-count-threshold = 200
+
+# a list of aggregates gatheredfor each ms-typed metric
+# please, note that setting this value will define the exact way, so, i.e. ["mean"] will gather ONLY ONE aggregate
+#
+# To add new percentiles an integer value with "precentile-" prefix should be used.
+# It will then be "appended" (in reality this is not string based calculation) with "0."
+# For example, "percentile-9999" will become 0.9999th percentile
+#
+# this is the full list of default values
+#ms-aggregates = [ "count", "last", "min", "max", "sum", "median", "mean", "updates", "percentile-75", "percentile-95", "percentile-98", "percentile-99", "percentile-999" ]
+
+# these attributes allow to change naming of aggregates in name postfix, prefix or in tags
+# by default names are the same as aggregate names, except for percentiles
+# percentile postfixes by default look like "percentile.99"
+
+# NOT defaults, just example
+#postfix-replacements = { "min" = "lower", "max" = "upper", "percentile-50" = "percentile.50" }
+
+# prefix specific aggregates with specified string
+# all prefixes are empty by default
+# NOTE: you most probably want EITHER prefix or postfix, so having prefix specified here,
+# don't forget to explicitly set postfix to ""
+#
+# NOT defaults, just example
+#prefix-replacements = { "updates" = "resources.monitoring.bioyino.updates" }
+
+#tag-replacements = { "min" = "lower", "max" = "upper", "percentile-50" = "percentile.50" }
+
+# the default tag name taken when replacement is not specified
+# tag-name = "aggregate"
+
 # Settings for internal Raft
 [raft]
 # Defer start of raft consensus to avoid node becoming leader too early
@@ -165,7 +210,8 @@ snapshot-interval = 1000
 # this-node = <empty>
 
 # A map of other raft nodes. Keys are in form of hostname:port or IP:port
-# values are integers
+# values are integer IDs
+# this map is crucial to have the same address <-> ID mappings for all nodes
 nodes = {}
 
 # allow binding raft outgoing connnections to specific IP

rustfmt.toml

@@ -1,2 +1,2 @@
-max_width=1000
-ideal_width=100
+max_width=160
+ideal_width=80