From b4c068cb60793ffe52d71400e0aea537ac7d9acc Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 3 Dec 2020 13:42:00 +0530
Subject: [PATCH 01/53] Implement basic scaffolding for rate_limit processor

---
 .../rate_limit/algorithm/algorithm.go         | 50 +++++++++++
 libbeat/processors/rate_limit/config.go       | 35 ++++++++
 .../rate_limit/docs/rate_limit.asciidoc       | 63 +++++++++++++
 libbeat/processors/rate_limit/limit.go        | 80 +++++++++++++++++
 libbeat/processors/rate_limit/rate_limit.go   | 89 +++++++++++++++++++
 5 files changed, 317 insertions(+)
 create mode 100644 libbeat/processors/rate_limit/algorithm/algorithm.go
 create mode 100644 libbeat/processors/rate_limit/config.go
 create mode 100644 libbeat/processors/rate_limit/docs/rate_limit.asciidoc
 create mode 100644 libbeat/processors/rate_limit/limit.go
 create mode 100644 libbeat/processors/rate_limit/rate_limit.go

diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/rate_limit/algorithm/algorithm.go
new file mode 100644
index 000000000000..7829e10436d4
--- /dev/null
+++ b/libbeat/processors/rate_limit/algorithm/algorithm.go
@@ -0,0 +1,50 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package algorithm
+
+import (
+	"fmt"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/processors/rate_limit"
+)
+
+var registry = map[string]Algorithm{}
+
+type Config struct {
+	Limit  rate_limit.Limit
+	Config common.Config
+}
+
+type Algorithm interface {
+	ID() string
+	Configure(Config)
+	IsAllowed(string) bool
+}
+
+func Register(algorithm Algorithm) {
+	registry[algorithm.ID()] = algorithm
+}
+
+func Factory(id string) (Algorithm, error) {
+	if algorithm, found := registry[id]; found {
+		return algorithm, nil
+	}
+
+	return nil, fmt.Errorf("rate limiting algorithm '%v' not implemented", id)
+}
diff --git a/libbeat/processors/rate_limit/config.go b/libbeat/processors/rate_limit/config.go
new file mode 100644
index 000000000000..d24c9402d9a0
--- /dev/null
+++ b/libbeat/processors/rate_limit/config.go
@@ -0,0 +1,35 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package rate_limit
+
+import (
+	"github.com/elastic/beats/v7/libbeat/common"
+)
+
+// Config for rate limit processor.
+type Config struct {
+	Limit     Limit                  `config:"limit" validate:"required"`
+	Fields    []string               `config:"fields"`
+	Algorithm common.ConfigNamespace `config:"algorithm"`
+}
+
+func defaultConfig() Config {
+	return Config{
+		// TODO: set default algorithm
+	}
+}
diff --git a/libbeat/processors/rate_limit/docs/rate_limit.asciidoc b/libbeat/processors/rate_limit/docs/rate_limit.asciidoc
new file mode 100644
index 000000000000..6555968f79c6
--- /dev/null
+++ b/libbeat/processors/rate_limit/docs/rate_limit.asciidoc
@@ -0,0 +1,63 @@
+[[rate_limit]]
+=== Rate limit the flow of events
+
+++++
+<titleabbrev>rate_limit</titleabbrev>
+++++
+
+The `rate_limit` processor limits the rate of event flow based on
+the specified configuration.
+
+[source,yaml]
+-----------------------------------------------------
+processors:
+- rate_limit:
+   limit: "10000/m"
+-----------------------------------------------------
+
+[source,yaml]
+-----------------------------------------------------
+processors:
+- rate_limit:
+   fields:
+   - "cloudfoundry.org.name"
+   limit: "400/s"
+-----------------------------------------------------
+
+[source,yaml]
+-----------------------------------------------------
+processors:
+- if.equals.cloudfoundry.org.name: "acme"
+  then:
+  - rate_limit:
+      limit: "500/s"
+-----------------------------------------------------
+
+The following settings are supported:
+
+`limit`:: The rate limit. Supported time units for the rate are `s` (per second), `m` (per minute), and `h` (per hour).
+`fields`:: (Optional) List of fields. The rate limit will be applied to each distinct value derived by combining the values of these fields.
+`algorithm`:: (Optional) The rate limiting algorithm to use. Supported algorithms are `token_bucket`, `leaky_bucket_queue`, and `sliding_window`. Default is `token_bucket`.
+
+Some rate limiting algorithms support additional settings.
+
+[source,yaml]
+-----------------------------------------------------
+processors:
+- rate_limit:
+   limit: "10000/m"
+   algorithm:
+     token_bucket: ~
+-----------------------------------------------------
+
+[source,yaml]
+-----------------------------------------------------
+processors:
+- rate_limit:
+   limit: "10000/m"
+   algorithm:
+     token_bucket:
+       burst_multiplier: 1.0
+-----------------------------------------------------
+
+`algorithm.token_bucket.burst_multiplier`:: How much burstability of events should the token bucket allow for. Optional. Default = 1.0 which means no burstability. TODO: explain better.
diff --git a/libbeat/processors/rate_limit/limit.go b/libbeat/processors/rate_limit/limit.go
new file mode 100644
index 000000000000..25f9238c0407
--- /dev/null
+++ b/libbeat/processors/rate_limit/limit.go
@@ -0,0 +1,80 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package rate_limit
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+type unit string
+
+const (
+	unitPerSecond unit = "s"
+	unitPerMinute unit = "m"
+	unitPerHour   unit = "h"
+)
+
+type Limit struct {
+	value float64
+	unit  unit
+}
+
+// Unpack creates a limit from the given string
+func (l *Limit) Unpack(str string) error {
+	parts := strings.Split(str, "/")
+	if len(parts) != 2 {
+		// TODO: make custom error?
+		return fmt.Errorf(`rate limit in invalid format: %v. Must be specified as "number/unit"`, str)
+	}
+
+	valueStr := strings.TrimSpace(parts[0])
+	unitStr := strings.TrimSpace(parts[1])
+
+	v, err := strconv.ParseFloat(valueStr, 8)
+	if err != nil {
+		return fmt.Errorf(`rate limit's limit component is not numeric: %v`, valueStr)
+	}
+
+	if allowed := []unit{unitPerSecond, unitPerMinute, unitPerHour}; !contains(allowed, unitStr) {
+		allowedStrs := make([]string, len(allowed))
+		for _, a := range allowed {
+			allowedStrs = append(allowedStrs, "/"+string(a))
+		}
+
+		return fmt.Errorf(`rate limit's unit component must be specified as one of: %v`, strings.Join(allowedStrs, ","))
+	}
+
+	u := unit(unitStr)
+
+	l.value = v
+	l.unit = u
+
+	return nil
+}
+
+func contains(allowed []unit, candidate string) bool {
+	for _, a := range allowed {
+		if candidate == string(a) {
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
new file mode 100644
index 000000000000..d767ce35d73a
--- /dev/null
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -0,0 +1,89 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package rate_limit
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/processors"
+	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/algorithm"
+	jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor"
+)
+
+func init() {
+	processors.RegisterPlugin("rate_limit", New)
+	jsprocessor.RegisterPlugin("Fingerprint", New)
+}
+
+const processorName = "rate_limit"
+
+type rateLimit struct {
+	config    Config
+	algorithm algorithm.Algorithm
+}
+
+// New constructs a new rate limit processor.
+func New(cfg *common.Config) (processors.Processor, error) {
+	config := defaultConfig()
+	if err := cfg.Unpack(&config); err != nil {
+		// TODO: make custom error: errConfigUnpack?
+		return nil, errors.Wrap(err, "could not unpack processor configuration")
+	}
+
+	algo, err := algorithm.Factory(config.Algorithm.Name())
+	if err != nil {
+		return nil, errors.Wrap(err, "could not instantiate rate limiting algorithm")
+	}
+
+	algo.Configure(algorithm.Config{
+		Limit:  config.Limit,
+		Config: *config.Algorithm.Config(),
+	})
+
+	// TODO: flesh out fields
+	p := &rateLimit{
+		config:    config,
+		algorithm: algo,
+	}
+
+	return p, nil
+}
+
+// Run applies the configured rate limit to the given event. If the event is within the
+// configured rate limit, it is returned as-is. If not, nil is returned.
+func (p *rateLimit) Run(event *beat.Event) (*beat.Event, error) {
+	key := "" // TODO: construct key from event fields + config
+
+	if p.algorithm.IsAllowed(key) {
+		return event, nil
+	}
+
+	// TODO: log that event is being dropped
+	return nil, nil
+}
+
+func (p *rateLimit) String() string {
+	return fmt.Sprintf(
+		"%v=[limit=[%v],fields=[%v],algorithm=[%v]]",
+		processorName, p.config.Limit, p.config.Fields, p.config.Algorithm.Name(),
+	)
+}

From 2400b672696bf274e85c7c994245d4bdccf15bcf Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 3 Dec 2020 14:26:10 +0530
Subject: [PATCH 02/53] Fixing import cycle

---
 libbeat/processors/rate_limit/algorithm/algorithm.go   | 3 +--
 libbeat/processors/rate_limit/{ => algorithm}/limit.go | 2 +-
 libbeat/processors/rate_limit/config.go                | 3 ++-
 3 files changed, 4 insertions(+), 4 deletions(-)
 rename libbeat/processors/rate_limit/{ => algorithm}/limit.go (99%)

diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/rate_limit/algorithm/algorithm.go
index 7829e10436d4..2e070d5cca70 100644
--- a/libbeat/processors/rate_limit/algorithm/algorithm.go
+++ b/libbeat/processors/rate_limit/algorithm/algorithm.go
@@ -21,13 +21,12 @@ import (
 	"fmt"
 
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/processors/rate_limit"
 )
 
 var registry = map[string]Algorithm{}
 
 type Config struct {
-	Limit  rate_limit.Limit
+	Limit  Limit
 	Config common.Config
 }
 
diff --git a/libbeat/processors/rate_limit/limit.go b/libbeat/processors/rate_limit/algorithm/limit.go
similarity index 99%
rename from libbeat/processors/rate_limit/limit.go
rename to libbeat/processors/rate_limit/algorithm/limit.go
index 25f9238c0407..e244ccdcf9c8 100644
--- a/libbeat/processors/rate_limit/limit.go
+++ b/libbeat/processors/rate_limit/algorithm/limit.go
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package rate_limit
+package algorithm
 
 import (
 	"fmt"
diff --git a/libbeat/processors/rate_limit/config.go b/libbeat/processors/rate_limit/config.go
index d24c9402d9a0..847e77ce2661 100644
--- a/libbeat/processors/rate_limit/config.go
+++ b/libbeat/processors/rate_limit/config.go
@@ -19,11 +19,12 @@ package rate_limit
 
 import (
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/algorithm"
 )
 
 // Config for rate limit processor.
 type Config struct {
-	Limit     Limit                  `config:"limit" validate:"required"`
+	Limit     algorithm.Limit        `config:"limit" validate:"required"`
 	Fields    []string               `config:"fields"`
 	Algorithm common.ConfigNamespace `config:"algorithm"`
 }

From b9028cdcdf2e88b8c540154f6ebc84901ac832a0 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 3 Dec 2020 14:31:05 +0530
Subject: [PATCH 03/53] Adding skeleton for token bucket algo

---
 .../rate_limit/algorithm/algorithm.go         |  2 +-
 .../rate_limit/algorithm/token_bucket.go      | 23 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 libbeat/processors/rate_limit/algorithm/token_bucket.go

diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/rate_limit/algorithm/algorithm.go
index 2e070d5cca70..465398dc71a8 100644
--- a/libbeat/processors/rate_limit/algorithm/algorithm.go
+++ b/libbeat/processors/rate_limit/algorithm/algorithm.go
@@ -32,7 +32,7 @@ type Config struct {
 
 type Algorithm interface {
 	ID() string
-	Configure(Config)
+	Configure(Config) error
 	IsAllowed(string) bool
 }
 
diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
new file mode 100644
index 000000000000..53d9fbb2c24d
--- /dev/null
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -0,0 +1,23 @@
+package algorithm
+
+func init() {
+	Register(&tokenBucket{})
+}
+
+type tokenBucket struct {
+	// TODO: flesh out
+}
+
+func (t *tokenBucket) ID() string {
+	return "token_bucket"
+}
+
+func (t *tokenBucket) Configure(config Config) error {
+	// TODO: flesh out
+	return nil
+}
+
+func (t *tokenBucket) IsAllowed(key string) bool {
+	// TODO: flesh out
+	return true
+}

From dce6dae450479a9ac58a3d5addf9fcdab39be43c Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 12:19:18 +0530
Subject: [PATCH 04/53] Set default algorithm in default config

---
 libbeat/processors/rate_limit/config.go     | 21 ++++++++++++++++++---
 libbeat/processors/rate_limit/rate_limit.go |  8 ++++++--
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/libbeat/processors/rate_limit/config.go b/libbeat/processors/rate_limit/config.go
index 847e77ce2661..8213111c7a81 100644
--- a/libbeat/processors/rate_limit/config.go
+++ b/libbeat/processors/rate_limit/config.go
@@ -18,6 +18,8 @@
 package rate_limit
 
 import (
+	"github.com/pkg/errors"
+
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/algorithm"
 )
@@ -29,8 +31,21 @@ type Config struct {
 	Algorithm common.ConfigNamespace `config:"algorithm"`
 }
 
-func defaultConfig() Config {
-	return Config{
-		// TODO: set default algorithm
+func defaultConfig() (*Config, error) {
+	cfg, err := common.NewConfigFrom(map[string]interface{}{
+		"algorithm": map[string]interface{}{
+			"token_bucket": "~",
+		},
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "could not parse default configuration")
 	}
+
+	var config Config
+	if err := cfg.Unpack(&config); err != nil {
+		return nil, errors.Wrap(err, "could not unpack default configuration")
+	}
+
+	return &config, nil
 }
diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index d767ce35d73a..3c135eb5f4aa 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -43,7 +43,11 @@ type rateLimit struct {
 
 // New constructs a new rate limit processor.
 func New(cfg *common.Config) (processors.Processor, error) {
-	config := defaultConfig()
+	config, err := defaultConfig()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create default configuration")
+	}
+
 	if err := cfg.Unpack(&config); err != nil {
 		// TODO: make custom error: errConfigUnpack?
 		return nil, errors.Wrap(err, "could not unpack processor configuration")
@@ -61,7 +65,7 @@ func New(cfg *common.Config) (processors.Processor, error) {
 
 	// TODO: flesh out fields
 	p := &rateLimit{
-		config:    config,
+		config:    *config,
 		algorithm: algo,
 	}
 

From ca41a82b1e7e4b43ca0fcc37c82e0f2bfe467b62 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 12:40:47 +0530
Subject: [PATCH 05/53] Using an algo constructor

---
 .../processors/rate_limit/algorithm/algorithm.go | 16 ++++++++--------
 .../rate_limit/algorithm/token_bucket.go         | 11 +++--------
 libbeat/processors/rate_limit/rate_limit.go      |  4 ++--
 3 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/rate_limit/algorithm/algorithm.go
index 465398dc71a8..65a7a0214ccb 100644
--- a/libbeat/processors/rate_limit/algorithm/algorithm.go
+++ b/libbeat/processors/rate_limit/algorithm/algorithm.go
@@ -23,7 +23,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 )
 
-var registry = map[string]Algorithm{}
+var registry = make(map[string]constructor, 0)
 
 type Config struct {
 	Limit  Limit
@@ -31,18 +31,18 @@ type Config struct {
 }
 
 type Algorithm interface {
-	ID() string
-	Configure(Config) error
 	IsAllowed(string) bool
 }
 
-func Register(algorithm Algorithm) {
-	registry[algorithm.ID()] = algorithm
+type constructor func(Config) Algorithm
+
+func Register(id string, ctor constructor) {
+	registry[id] = ctor
 }
 
-func Factory(id string) (Algorithm, error) {
-	if algorithm, found := registry[id]; found {
-		return algorithm, nil
+func Factory(id string) (constructor, error) {
+	if ctor, found := registry[id]; found {
+		return ctor, nil
 	}
 
 	return nil, fmt.Errorf("rate limiting algorithm '%v' not implemented", id)
diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 53d9fbb2c24d..5d9af24e9fee 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -1,20 +1,15 @@
 package algorithm
 
 func init() {
-	Register(&tokenBucket{})
+	Register("token_bucket", newTokenBucket)
 }
 
 type tokenBucket struct {
 	// TODO: flesh out
 }
 
-func (t *tokenBucket) ID() string {
-	return "token_bucket"
-}
-
-func (t *tokenBucket) Configure(config Config) error {
-	// TODO: flesh out
-	return nil
+func newTokenBucket(config Config) Algorithm {
+	return &tokenBucket{}
 }
 
 func (t *tokenBucket) IsAllowed(key string) bool {
diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index 3c135eb5f4aa..5e4f751eefae 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -53,12 +53,12 @@ func New(cfg *common.Config) (processors.Processor, error) {
 		return nil, errors.Wrap(err, "could not unpack processor configuration")
 	}
 
-	algo, err := algorithm.Factory(config.Algorithm.Name())
+	algoCtor, err := algorithm.Factory(config.Algorithm.Name())
 	if err != nil {
 		return nil, errors.Wrap(err, "could not instantiate rate limiting algorithm")
 	}
 
-	algo.Configure(algorithm.Config{
+	algo := algoCtor(algorithm.Config{
 		Limit:  config.Limit,
 		Config: *config.Algorithm.Config(),
 	})

From fb960e57e289d6b30f2269975af811c60501f6a2 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 13:28:34 +0530
Subject: [PATCH 06/53] Implement token bucket rate limiting algorithm

---
 .../rate_limit/algorithm/algorithm.go         |  2 +-
 .../processors/rate_limit/algorithm/limit.go  | 15 +++-
 .../rate_limit/algorithm/token_bucket.go      | 74 ++++++++++++++++++-
 libbeat/processors/rate_limit/config.go       |  2 +-
 4 files changed, 86 insertions(+), 7 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/rate_limit/algorithm/algorithm.go
index 65a7a0214ccb..c70c08548c02 100644
--- a/libbeat/processors/rate_limit/algorithm/algorithm.go
+++ b/libbeat/processors/rate_limit/algorithm/algorithm.go
@@ -26,7 +26,7 @@ import (
 var registry = make(map[string]constructor, 0)
 
 type Config struct {
-	Limit  Limit
+	Limit  Rate
 	Config common.Config
 }
 
diff --git a/libbeat/processors/rate_limit/algorithm/limit.go b/libbeat/processors/rate_limit/algorithm/limit.go
index e244ccdcf9c8..756cfee5d39e 100644
--- a/libbeat/processors/rate_limit/algorithm/limit.go
+++ b/libbeat/processors/rate_limit/algorithm/limit.go
@@ -31,13 +31,13 @@ const (
 	unitPerHour   unit = "h"
 )
 
-type Limit struct {
+type Rate struct {
 	value float64
 	unit  unit
 }
 
 // Unpack creates a limit from the given string
-func (l *Limit) Unpack(str string) error {
+func (l *Rate) Unpack(str string) error {
 	parts := strings.Split(str, "/")
 	if len(parts) != 2 {
 		// TODO: make custom error?
@@ -69,6 +69,17 @@ func (l *Limit) Unpack(str string) error {
 	return nil
 }
 
+func (l *Rate) valuePerSecond() float64 {
+	switch l.unit {
+	case unitPerSecond:
+		return l.value
+	case unitPerMinute:
+		return l.value / 60
+	case unitPerHour:
+		return l.value / (60 * 60)
+	}
+}
+
 func contains(allowed []unit, candidate string) bool {
 	for _, a := range allowed {
 		if candidate == string(a) {
diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 5d9af24e9fee..0050549d085a 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -1,18 +1,86 @@
 package algorithm
 
+import (
+	"time"
+)
+
 func init() {
 	Register("token_bucket", newTokenBucket)
 }
 
+type bucket struct {
+	tokens        float64
+	lastReplenish time.Time
+}
+
 type tokenBucket struct {
-	// TODO: flesh out
+	limit   Rate
+	depth   float64
+	buckets map[string]bucket
 }
 
 func newTokenBucket(config Config) Algorithm {
-	return &tokenBucket{}
+	return &tokenBucket{
+		config.Limit,
+		config.Limit.value * 1, // TODO: replace 1 with burstability multiplier
+		make(map[string]bucket, 0),
+	}
 }
 
 func (t *tokenBucket) IsAllowed(key string) bool {
-	// TODO: flesh out
+	t.replenishBuckets()
+
+	b := t.getBucket(key)
+	allowed := b.withdraw()
+
+	return allowed
+}
+
+func (t *tokenBucket) getBucket(key string) bucket {
+	b, exists := t.buckets[key]
+	if !exists {
+		b = bucket{
+			tokens:        t.depth,
+			lastReplenish: time.Now(),
+		}
+		t.buckets[key] = b
+	}
+
+	return b
+
+}
+
+func (b *bucket) withdraw() bool {
+	if b.tokens == 0 {
+		return false
+	}
+	b.tokens--
 	return true
 }
+
+func (b *bucket) replenish(rate Rate) {
+	secsSinceLastReplenish := time.Now().Sub(b.lastReplenish).Seconds()
+	tokensToReplenish := secsSinceLastReplenish * rate.valuePerSecond()
+
+	b.tokens += tokensToReplenish
+	b.lastReplenish = time.Now()
+}
+
+func (t *tokenBucket) replenishBuckets() {
+	toDelete := make([]string, 0)
+
+	// Replenish all buckets with tokens at the rate limit
+	for key, b := range t.buckets {
+		b.replenish(t.limit)
+
+		// If bucket is full, flag it for deletion
+		if b.tokens >= t.depth {
+			toDelete = append(toDelete, key)
+		}
+	}
+
+	// Cleanup full buckets to free up memory
+	for _, key := range toDelete {
+		delete(t.buckets, key)
+	}
+}
diff --git a/libbeat/processors/rate_limit/config.go b/libbeat/processors/rate_limit/config.go
index 8213111c7a81..55401b4075e1 100644
--- a/libbeat/processors/rate_limit/config.go
+++ b/libbeat/processors/rate_limit/config.go
@@ -26,7 +26,7 @@ import (
 
 // Config for rate limit processor.
 type Config struct {
-	Limit     algorithm.Limit        `config:"limit" validate:"required"`
+	Limit     algorithm.Rate         `config:"limit" validate:"required"`
 	Fields    []string               `config:"fields"`
 	Algorithm common.ConfigNamespace `config:"algorithm"`
 }

From 80a37f83377d4fdca1444cd58505ca5c8efd8ff0 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 15:25:39 +0530
Subject: [PATCH 07/53] Resolving some TODOs

---
 .../rate_limit/algorithm/algorithm.go         |  2 +-
 .../processors/rate_limit/algorithm/limit.go  |  3 +-
 .../rate_limit/algorithm/token_bucket.go      | 16 +++++--
 libbeat/processors/rate_limit/config.go       |  6 ++-
 libbeat/processors/rate_limit/rate_limit.go   | 42 +++++++++++++++++--
 5 files changed, 59 insertions(+), 10 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/rate_limit/algorithm/algorithm.go
index c70c08548c02..ee3aed13d6d6 100644
--- a/libbeat/processors/rate_limit/algorithm/algorithm.go
+++ b/libbeat/processors/rate_limit/algorithm/algorithm.go
@@ -34,7 +34,7 @@ type Algorithm interface {
 	IsAllowed(string) bool
 }
 
-type constructor func(Config) Algorithm
+type constructor func(Config) (Algorithm, error)
 
 func Register(id string, ctor constructor) {
 	registry[id] = ctor
diff --git a/libbeat/processors/rate_limit/algorithm/limit.go b/libbeat/processors/rate_limit/algorithm/limit.go
index 756cfee5d39e..74a78bbed37e 100644
--- a/libbeat/processors/rate_limit/algorithm/limit.go
+++ b/libbeat/processors/rate_limit/algorithm/limit.go
@@ -40,7 +40,6 @@ type Rate struct {
 func (l *Rate) Unpack(str string) error {
 	parts := strings.Split(str, "/")
 	if len(parts) != 2 {
-		// TODO: make custom error?
 		return fmt.Errorf(`rate limit in invalid format: %v. Must be specified as "number/unit"`, str)
 	}
 
@@ -78,6 +77,8 @@ func (l *Rate) valuePerSecond() float64 {
 	case unitPerHour:
 		return l.value / (60 * 60)
 	}
+
+	return 0
 }
 
 func contains(allowed []unit, candidate string) bool {
diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 0050549d085a..57e7d4b86ce5 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -2,6 +2,8 @@ package algorithm
 
 import (
 	"time"
+
+	"github.com/pkg/errors"
 )
 
 func init() {
@@ -19,12 +21,20 @@ type tokenBucket struct {
 	buckets map[string]bucket
 }
 
-func newTokenBucket(config Config) Algorithm {
+func newTokenBucket(config Config) (Algorithm, error) {
+	var cfg struct {
+		BurstMultipler float64 `config:"burst_multiplier"`
+	}
+
+	if err := config.Config.Unpack(&cfg); err != nil {
+		return nil, errors.Wrap(err, "could not unpack token_bucket algorithm configuration")
+	}
+
 	return &tokenBucket{
 		config.Limit,
-		config.Limit.value * 1, // TODO: replace 1 with burstability multiplier
+		config.Limit.value * cfg.BurstMultipler,
 		make(map[string]bucket, 0),
-	}
+	}, nil
 }
 
 func (t *tokenBucket) IsAllowed(key string) bool {
diff --git a/libbeat/processors/rate_limit/config.go b/libbeat/processors/rate_limit/config.go
index 55401b4075e1..ead95aa54d7d 100644
--- a/libbeat/processors/rate_limit/config.go
+++ b/libbeat/processors/rate_limit/config.go
@@ -34,7 +34,9 @@ type Config struct {
 func defaultConfig() (*Config, error) {
 	cfg, err := common.NewConfigFrom(map[string]interface{}{
 		"algorithm": map[string]interface{}{
-			"token_bucket": "~",
+			"token_bucket": map[string]interface{}{
+				"burst_multiplier": 1.0,
+			},
 		},
 	})
 
@@ -47,5 +49,7 @@ func defaultConfig() (*Config, error) {
 		return nil, errors.Wrap(err, "could not unpack default configuration")
 	}
 
+	config.Fields = make([]string, 0)
+
 	return &config, nil
 }
diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index 5e4f751eefae..5d0fa3d75aaa 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -19,11 +19,14 @@ package rate_limit
 
 import (
 	"fmt"
+	"sort"
+	"strings"
 
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/processors"
 	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/algorithm"
 	jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor"
@@ -39,6 +42,7 @@ const processorName = "rate_limit"
 type rateLimit struct {
 	config    Config
 	algorithm algorithm.Algorithm
+	logger    *logp.Logger
 }
 
 // New constructs a new rate limit processor.
@@ -49,7 +53,6 @@ func New(cfg *common.Config) (processors.Processor, error) {
 	}
 
 	if err := cfg.Unpack(&config); err != nil {
-		// TODO: make custom error: errConfigUnpack?
 		return nil, errors.Wrap(err, "could not unpack processor configuration")
 	}
 
@@ -58,15 +61,18 @@ func New(cfg *common.Config) (processors.Processor, error) {
 		return nil, errors.Wrap(err, "could not instantiate rate limiting algorithm")
 	}
 
-	algo := algoCtor(algorithm.Config{
+	algo, err := algoCtor(algorithm.Config{
 		Limit:  config.Limit,
 		Config: *config.Algorithm.Config(),
 	})
+	if err != nil {
+		return nil, errors.Wrap(err, "could not construct rate limit algorithm")
+	}
 
-	// TODO: flesh out fields
 	p := &rateLimit{
 		config:    *config,
 		algorithm: algo,
+		logger:    logp.NewLogger("rate_limit"),
 	}
 
 	return p, nil
@@ -75,9 +81,13 @@ func New(cfg *common.Config) (processors.Processor, error) {
 // Run applies the configured rate limit to the given event. If the event is within the
 // configured rate limit, it is returned as-is. If not, nil is returned.
 func (p *rateLimit) Run(event *beat.Event) (*beat.Event, error) {
-	key := "" // TODO: construct key from event fields + config
+	key, err := p.makeKey(event)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not make key")
+	}
 
 	if p.algorithm.IsAllowed(key) {
+		p.logger.Debugf("event [%v] dropped by rate_limit processor", event)
 		return event, nil
 	}
 
@@ -91,3 +101,27 @@ func (p *rateLimit) String() string {
 		processorName, p.config.Limit, p.config.Fields, p.config.Algorithm.Name(),
 	)
 }
+
+func (p *rateLimit) makeKey(event *beat.Event) (string, error) {
+	var key string
+
+	if len(p.config.Fields) > 0 {
+		sort.Strings(p.config.Fields)
+		values := make([]string, len(p.config.Fields))
+		for _, field := range p.config.Fields {
+			value, err := event.GetValue(field)
+			if err != nil && err == common.ErrKeyNotFound {
+				value = ""
+			}
+			if err != nil {
+				return "", errors.Wrapf(err, "error getting value of field: %v", field)
+			}
+
+			// TODO: check that the value is a scalar?
+			values = append(values, fmt.Sprintf("%v", value))
+		}
+		key = strings.Join(values, "_")
+	}
+
+	return key, nil
+}

From 8b550d32bb0c117f2da018e39010c3fc3153780a Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 15:27:59 +0530
Subject: [PATCH 08/53] Adding license header

---
 .../rate_limit/algorithm/token_bucket.go        | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 57e7d4b86ce5..e466e8dcf611 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -1,3 +1,20 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package algorithm
 
 import (

From bff5ee7e99932396778ee979f4ff0ab41adbb14b Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 15:28:54 +0530
Subject: [PATCH 09/53] Removing old TODO comment

---
 libbeat/processors/rate_limit/rate_limit.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index 5d0fa3d75aaa..cb5c4c3f65b8 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -91,7 +91,6 @@ func (p *rateLimit) Run(event *beat.Event) (*beat.Event, error) {
 		return event, nil
 	}
 
-	// TODO: log that event is being dropped
 	return nil, nil
 }
 

From 752ff9cc08623585041c5b7d8642525f9c5ff18f Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 17:48:27 +0530
Subject: [PATCH 10/53] Adding tests

---
 .../rate_limit/algorithm/token_bucket.go      |   7 +-
 libbeat/processors/rate_limit/config.go       |  23 ++-
 libbeat/processors/rate_limit/rate_limit.go   |  12 +-
 .../processors/rate_limit/rate_limit_test.go  | 150 ++++++++++++++++++
 4 files changed, 169 insertions(+), 23 deletions(-)
 create mode 100644 libbeat/processors/rate_limit/rate_limit_test.go

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index e466e8dcf611..02f2ce68ff41 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -40,7 +40,7 @@ type tokenBucket struct {
 
 func newTokenBucket(config Config) (Algorithm, error) {
 	var cfg struct {
-		BurstMultipler float64 `config:"burst_multiplier"`
+		BurstMultiplier float64 `config:"burst_multiplier"`
 	}
 
 	if err := config.Config.Unpack(&cfg); err != nil {
@@ -49,7 +49,7 @@ func newTokenBucket(config Config) (Algorithm, error) {
 
 	return &tokenBucket{
 		config.Limit,
-		config.Limit.value * cfg.BurstMultipler,
+		config.Limit.value * cfg.BurstMultiplier,
 		make(map[string]bucket, 0),
 	}, nil
 }
@@ -59,6 +59,7 @@ func (t *tokenBucket) IsAllowed(key string) bool {
 
 	b := t.getBucket(key)
 	allowed := b.withdraw()
+	t.buckets[key] = b
 
 	return allowed
 }
@@ -78,7 +79,7 @@ func (t *tokenBucket) getBucket(key string) bucket {
 }
 
 func (b *bucket) withdraw() bool {
-	if b.tokens == 0 {
+	if b.tokens < 1 {
 		return false
 	}
 	b.tokens--
diff --git a/libbeat/processors/rate_limit/config.go b/libbeat/processors/rate_limit/config.go
index ead95aa54d7d..9bc64d88b893 100644
--- a/libbeat/processors/rate_limit/config.go
+++ b/libbeat/processors/rate_limit/config.go
@@ -31,25 +31,20 @@ type Config struct {
 	Algorithm common.ConfigNamespace `config:"algorithm"`
 }
 
-func defaultConfig() (*Config, error) {
-	cfg, err := common.NewConfigFrom(map[string]interface{}{
-		"algorithm": map[string]interface{}{
+func (c *Config) SetDefaults() error {
+	if c.Algorithm.Name() == "" {
+		cfg, err := common.NewConfigFrom(map[string]interface{}{
 			"token_bucket": map[string]interface{}{
 				"burst_multiplier": 1.0,
 			},
-		},
-	})
+		})
 
-	if err != nil {
-		return nil, errors.Wrap(err, "could not parse default configuration")
-	}
+		if err != nil {
+			return errors.Wrap(err, "could not parse default configuration")
+		}
 
-	var config Config
-	if err := cfg.Unpack(&config); err != nil {
-		return nil, errors.Wrap(err, "could not unpack default configuration")
+		c.Algorithm.Unpack(cfg)
 	}
 
-	config.Fields = make([]string, 0)
-
-	return &config, nil
+	return nil
 }
diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index cb5c4c3f65b8..5bda5041ee84 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -47,15 +47,15 @@ type rateLimit struct {
 
 // New constructs a new rate limit processor.
 func New(cfg *common.Config) (processors.Processor, error) {
-	config, err := defaultConfig()
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create default configuration")
-	}
-
+	var config Config
 	if err := cfg.Unpack(&config); err != nil {
 		return nil, errors.Wrap(err, "could not unpack processor configuration")
 	}
 
+	if err := config.SetDefaults(); err != nil {
+		return nil, errors.Wrap(err, "could not set default configuration")
+	}
+
 	algoCtor, err := algorithm.Factory(config.Algorithm.Name())
 	if err != nil {
 		return nil, errors.Wrap(err, "could not instantiate rate limiting algorithm")
@@ -70,7 +70,7 @@ func New(cfg *common.Config) (processors.Processor, error) {
 	}
 
 	p := &rateLimit{
-		config:    *config,
+		config:    config,
 		algorithm: algo,
 		logger:    logp.NewLogger("rate_limit"),
 	}
diff --git a/libbeat/processors/rate_limit/rate_limit_test.go b/libbeat/processors/rate_limit/rate_limit_test.go
new file mode 100644
index 000000000000..85df647cf72b
--- /dev/null
+++ b/libbeat/processors/rate_limit/rate_limit_test.go
@@ -0,0 +1,150 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package rate_limit
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/common"
+)
+
+func TestNew(t *testing.T) {
+	cases := map[string]struct {
+		config common.MapStr
+		err    string
+	}{
+		"default": {
+			common.MapStr{},
+			"",
+		},
+		"unknown_algo": {
+			common.MapStr{
+				"algorithm": common.MapStr{
+					"foobar": common.MapStr{},
+				},
+			},
+			"rate limiting algorithm 'foobar' not implemented",
+		},
+	}
+
+	for name, test := range cases {
+		t.Run(name, func(t *testing.T) {
+			config := common.MustNewConfigFrom(test.config)
+			_, err := New(config)
+			if test.err == "" {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err, test.err)
+			}
+		})
+	}
+}
+
+func TestRateLimit(t *testing.T) {
+	inEvents := []beat.Event{
+		{
+			Timestamp: time.Now(),
+			Fields: common.MapStr{
+				"foo": "bar",
+			},
+		},
+		{
+			Timestamp: time.Now(),
+			Fields: common.MapStr{
+				"baz": "qux",
+			},
+		},
+		{
+			Timestamp: time.Now(),
+			Fields: common.MapStr{
+				"dog": "seger",
+			},
+		},
+		{
+			Timestamp: time.Now(),
+			Fields: common.MapStr{
+				"cat": "garfield",
+			},
+		},
+	}
+
+	cases := map[string]struct {
+		config    common.MapStr
+		inEvents  []beat.Event
+		delay     time.Duration
+		outEvents []beat.Event
+	}{
+		"rate_0": {
+			config:    common.MapStr{},
+			inEvents:  inEvents,
+			outEvents: []beat.Event{},
+		},
+		"rate_1_per_min": {
+			config: common.MapStr{
+				"limit": "1/m",
+			},
+			inEvents:  inEvents,
+			outEvents: inEvents[0:1],
+		},
+		"rate_2_per_min": {
+			config: common.MapStr{
+				"limit": "2/m",
+			},
+			inEvents:  inEvents,
+			outEvents: inEvents[0:2],
+		},
+		"rate_5_per_min": {
+			config: common.MapStr{
+				"limit": "5/m",
+			},
+			inEvents:  inEvents,
+			outEvents: inEvents,
+		},
+		"rate_2_per_sec": {
+			config: common.MapStr{
+				"limit": "2/s",
+			},
+			delay:     400 * time.Millisecond,
+			inEvents:  inEvents,
+			outEvents: append(inEvents[0:2], inEvents[3]),
+		},
+	}
+
+	for name, test := range cases {
+		t.Run(name, func(t *testing.T) {
+			p, err := New(common.MustNewConfigFrom(test.config))
+			require.NoError(t, err)
+
+			out := make([]beat.Event, 0)
+			for _, in := range test.inEvents {
+				o, err := p.Run(&in)
+				require.NoError(t, err)
+				if o != nil {
+					out = append(out, *o)
+				}
+				time.Sleep(test.delay)
+			}
+
+			require.Equal(t, test.outEvents, out)
+		})
+	}
+}

From 028b67a2e5d7026a35858e80a195860334feabba Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 17:49:48 +0530
Subject: [PATCH 11/53] Reverting to previous logic

---
 libbeat/processors/rate_limit/algorithm/token_bucket.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 02f2ce68ff41..35f5e60a2e40 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -79,7 +79,7 @@ func (t *tokenBucket) getBucket(key string) bucket {
 }
 
 func (b *bucket) withdraw() bool {
-	if b.tokens < 1 {
+	if b.tokens == 0 {
 		return false
 	}
 	b.tokens--

From d08d9a681d34557b2b81592db1e77ecdb8c8358b Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 18:03:10 +0530
Subject: [PATCH 12/53] Adding CHANGELOG entry

---
 CHANGELOG.next.asciidoc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 7fa7be11f936..2ad46a0803eb 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -549,6 +549,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve event normalization performance {pull}22974[22974]
 - Add tini as init system in docker images {pull}22137[22137]
 - Added "add_network_direction" processor for determining perimeter-based network direction. {pull}23076[23076]
+- Added new `rate_limit` processor for enforcing rate limits on event throughput. {pull}22883[22883]
 
 *Auditbeat*
 

From 0a6b0cdf02bf05a9e6cf6127088a136db314fb16 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 18:03:18 +0530
Subject: [PATCH 13/53] Adding TODOs for more tests

---
 libbeat/processors/rate_limit/rate_limit_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libbeat/processors/rate_limit/rate_limit_test.go b/libbeat/processors/rate_limit/rate_limit_test.go
index 85df647cf72b..4b695a9dab47 100644
--- a/libbeat/processors/rate_limit/rate_limit_test.go
+++ b/libbeat/processors/rate_limit/rate_limit_test.go
@@ -127,6 +127,8 @@ func TestRateLimit(t *testing.T) {
 			inEvents:  inEvents,
 			outEvents: append(inEvents[0:2], inEvents[3]),
 		},
+		// TODO: add test for burst multiplier
+		// TODO: add test with fields
 	}
 
 	for name, test := range cases {

From c9852c9ba0e6b018c2d9016b772fe83ee8c72237 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 18:06:13 +0530
Subject: [PATCH 14/53] Fixing comment

---
 libbeat/processors/rate_limit/algorithm/limit.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/processors/rate_limit/algorithm/limit.go b/libbeat/processors/rate_limit/algorithm/limit.go
index 74a78bbed37e..f2a15d556054 100644
--- a/libbeat/processors/rate_limit/algorithm/limit.go
+++ b/libbeat/processors/rate_limit/algorithm/limit.go
@@ -36,7 +36,7 @@ type Rate struct {
 	unit  unit
 }
 
-// Unpack creates a limit from the given string
+// Unpack creates a Rate from the given string
 func (l *Rate) Unpack(str string) error {
 	parts := strings.Split(str, "/")
 	if len(parts) != 2 {

From 1fb20e3b90bf8f3071115cfac322b953bb30e556 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 18:07:28 +0530
Subject: [PATCH 15/53] Fixing error messages

---
 libbeat/processors/rate_limit/algorithm/limit.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/limit.go b/libbeat/processors/rate_limit/algorithm/limit.go
index f2a15d556054..d93eb12b1065 100644
--- a/libbeat/processors/rate_limit/algorithm/limit.go
+++ b/libbeat/processors/rate_limit/algorithm/limit.go
@@ -40,7 +40,7 @@ type Rate struct {
 func (l *Rate) Unpack(str string) error {
 	parts := strings.Split(str, "/")
 	if len(parts) != 2 {
-		return fmt.Errorf(`rate limit in invalid format: %v. Must be specified as "number/unit"`, str)
+		return fmt.Errorf(`rate in invalid format: %v. Must be specified as "number/unit"`, str)
 	}
 
 	valueStr := strings.TrimSpace(parts[0])
@@ -48,7 +48,7 @@ func (l *Rate) Unpack(str string) error {
 
 	v, err := strconv.ParseFloat(valueStr, 8)
 	if err != nil {
-		return fmt.Errorf(`rate limit's limit component is not numeric: %v`, valueStr)
+		return fmt.Errorf(`rate's value component is not numeric: %v`, valueStr)
 	}
 
 	if allowed := []unit{unitPerSecond, unitPerMinute, unitPerHour}; !contains(allowed, unitStr) {
@@ -57,7 +57,7 @@ func (l *Rate) Unpack(str string) error {
 			allowedStrs = append(allowedStrs, "/"+string(a))
 		}
 
-		return fmt.Errorf(`rate limit's unit component must be specified as one of: %v`, strings.Join(allowedStrs, ","))
+		return fmt.Errorf(`rate's unit component must be specified as one of: %v`, strings.Join(allowedStrs, ","))
 	}
 
 	u := unit(unitStr)

From 8e8a2042b41704a6b0e33be3e392414f6d9011e2 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 18:10:00 +0530
Subject: [PATCH 16/53] Fixing comment

---
 libbeat/processors/rate_limit/algorithm/token_bucket.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 35f5e60a2e40..2fb522a6d993 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -97,7 +97,7 @@ func (b *bucket) replenish(rate Rate) {
 func (t *tokenBucket) replenishBuckets() {
 	toDelete := make([]string, 0)
 
-	// Replenish all buckets with tokens at the rate limit
+	// Add tokens to all buckets according to the rate limit.
 	for key, b := range t.buckets {
 		b.replenish(t.limit)
 

From dfc5907b512df98d48f93ccbe10124c5364ec29a Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 18:17:00 +0530
Subject: [PATCH 17/53] Fleshing out godoc comments

---
 .../processors/rate_limit/algorithm/algorithm.go | 16 ++++++++++++++--
 .../rate_limit/algorithm/token_bucket.go         |  2 +-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/rate_limit/algorithm/algorithm.go
index ee3aed13d6d6..46e53110747a 100644
--- a/libbeat/processors/rate_limit/algorithm/algorithm.go
+++ b/libbeat/processors/rate_limit/algorithm/algorithm.go
@@ -25,21 +25,33 @@ import (
 
 var registry = make(map[string]constructor, 0)
 
+// Config for rate limit algorithm.
 type Config struct {
-	Limit  Rate
+	// Limit is the rate limit to be enforced by the algorithm.
+	Limit Rate
+
+	// Config is any algorithm-specific additional configuration.
 	Config common.Config
 }
 
+// Algorithm is the interface that all rate limiting algorithms must
+// conform to.
 type Algorithm interface {
+	// IsAllowed accepts a key and returns whether that key is allowed
+	// (true) or not (false). If a key is allowed, it means it is NOT
+	// rate limited. If a key is not allowed, it means it is being rate
+	// limited.
 	IsAllowed(string) bool
 }
 
 type constructor func(Config) (Algorithm, error)
 
-func Register(id string, ctor constructor) {
+func register(id string, ctor constructor) {
 	registry[id] = ctor
 }
 
+// Factory returns the requested rate limiting algorithm, if one is found. If not found,
+// an error is returned.
 func Factory(id string) (constructor, error) {
 	if ctor, found := registry[id]; found {
 		return ctor, nil
diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 2fb522a6d993..b8827d5c0654 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -24,7 +24,7 @@ import (
 )
 
 func init() {
-	Register("token_bucket", newTokenBucket)
+	register("token_bucket", newTokenBucket)
 }
 
 type bucket struct {

From fa63dbbc38e5b06198195e234d3f123bd70b2ad6 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 18:17:07 +0530
Subject: [PATCH 18/53] Fixing logger location

---
 libbeat/processors/rate_limit/rate_limit.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index 5bda5041ee84..27b0e020de31 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -87,10 +87,10 @@ func (p *rateLimit) Run(event *beat.Event) (*beat.Event, error) {
 	}
 
 	if p.algorithm.IsAllowed(key) {
-		p.logger.Debugf("event [%v] dropped by rate_limit processor", event)
 		return event, nil
 	}
 
+	p.logger.Debugf("event [%v] dropped by rate_limit processor", event)
 	return nil, nil
 }
 

From 189a170494aefd1fb3ecead328938babb8e50c4e Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 18:20:29 +0530
Subject: [PATCH 19/53] Fixing up docs a bit

---
 libbeat/processors/rate_limit/docs/rate_limit.asciidoc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libbeat/processors/rate_limit/docs/rate_limit.asciidoc b/libbeat/processors/rate_limit/docs/rate_limit.asciidoc
index 6555968f79c6..1666c27f8c19 100644
--- a/libbeat/processors/rate_limit/docs/rate_limit.asciidoc
+++ b/libbeat/processors/rate_limit/docs/rate_limit.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>rate_limit</titleabbrev>
 ++++
 
-The `rate_limit` processor limits the rate of event flow based on
+The `rate_limit` processor limits the throughput of events based on
 the specified configuration.
 
 [source,yaml]
@@ -37,9 +37,9 @@ The following settings are supported:
 
 `limit`:: The rate limit. Supported time units for the rate are `s` (per second), `m` (per minute), and `h` (per hour).
 `fields`:: (Optional) List of fields. The rate limit will be applied to each distinct value derived by combining the values of these fields.
-`algorithm`:: (Optional) The rate limiting algorithm to use. Supported algorithms are `token_bucket`, `leaky_bucket_queue`, and `sliding_window`. Default is `token_bucket`.
+`algorithm`:: (Optional) The rate limiting algorithm to use. The only supported algorithm at the moment is `token_bucket`.
 
-Some rate limiting algorithms support additional settings.
+Rate limiting algorithms may support additional, algorithm-specific settings.
 
 [source,yaml]
 -----------------------------------------------------

From 1e0c6ecf33339bc5dc8e49548bc1bdba72b9d38e Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 18:42:46 +0530
Subject: [PATCH 20/53] Adding test for "fields" config

---
 libbeat/processors/rate_limit/rate_limit.go   |  8 ++++----
 .../processors/rate_limit/rate_limit_test.go  | 19 ++++++++++++++-----
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index 27b0e020de31..1c1df91a6b8c 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -109,12 +109,12 @@ func (p *rateLimit) makeKey(event *beat.Event) (string, error) {
 		values := make([]string, len(p.config.Fields))
 		for _, field := range p.config.Fields {
 			value, err := event.GetValue(field)
-			if err != nil && err == common.ErrKeyNotFound {
-				value = ""
-			}
-			if err != nil {
+			if err != nil && err != common.ErrKeyNotFound {
 				return "", errors.Wrapf(err, "error getting value of field: %v", field)
 			}
+			if err != common.ErrKeyNotFound {
+				value = ""
+			}
 
 			// TODO: check that the value is a scalar?
 			values = append(values, fmt.Sprintf("%v", value))
diff --git a/libbeat/processors/rate_limit/rate_limit_test.go b/libbeat/processors/rate_limit/rate_limit_test.go
index 4b695a9dab47..e1c97250ac1b 100644
--- a/libbeat/processors/rate_limit/rate_limit_test.go
+++ b/libbeat/processors/rate_limit/rate_limit_test.go
@@ -70,19 +70,20 @@ func TestRateLimit(t *testing.T) {
 		{
 			Timestamp: time.Now(),
 			Fields: common.MapStr{
-				"baz": "qux",
+				"foo": "bar",
+				"baz": "mosquito",
 			},
 		},
 		{
 			Timestamp: time.Now(),
 			Fields: common.MapStr{
-				"dog": "seger",
+				"baz": "qux",
 			},
 		},
 		{
 			Timestamp: time.Now(),
 			Fields: common.MapStr{
-				"cat": "garfield",
+				"foo": "seger",
 			},
 		},
 	}
@@ -125,10 +126,18 @@ func TestRateLimit(t *testing.T) {
 			},
 			delay:     400 * time.Millisecond,
 			inEvents:  inEvents,
-			outEvents: append(inEvents[0:2], inEvents[3]),
+			outEvents: []beat.Event{inEvents[0], inEvents[1], inEvents[3]},
+		},
+		"with_fields": {
+			config: common.MapStr{
+				"limit":  "1/s",
+				"fields": []string{"foo"},
+			},
+			delay:     400 * time.Millisecond,
+			inEvents:  inEvents,
+			outEvents: []beat.Event{inEvents[0], inEvents[2], inEvents[3]},
 		},
 		// TODO: add test for burst multiplier
-		// TODO: add test with fields
 	}
 
 	for name, test := range cases {

From 8639e3b26b6f1d38328458dcbd6e969de2d1db0d Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Mon, 7 Dec 2020 18:46:30 +0530
Subject: [PATCH 21/53] WIP: adding test for burst multiplier

---
 libbeat/processors/rate_limit/rate_limit_test.go | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/libbeat/processors/rate_limit/rate_limit_test.go b/libbeat/processors/rate_limit/rate_limit_test.go
index e1c97250ac1b..1818e7c3edbe 100644
--- a/libbeat/processors/rate_limit/rate_limit_test.go
+++ b/libbeat/processors/rate_limit/rate_limit_test.go
@@ -137,7 +137,15 @@ func TestRateLimit(t *testing.T) {
 			inEvents:  inEvents,
 			outEvents: []beat.Event{inEvents[0], inEvents[2], inEvents[3]},
 		},
-		// TODO: add test for burst multiplier
+		"with_burst": { // FIXME: test is not working as expected
+			config: common.MapStr{
+				"limit":            "2/s",
+				"burst_multiplier": 2,
+			},
+			delay:     400 * time.Millisecond,
+			inEvents:  inEvents,
+			outEvents: inEvents,
+		},
 	}
 
 	for name, test := range cases {

From a9faa7e53e381c861615b6a77bd4dc5e7fd955ff Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 12:39:48 +0530
Subject: [PATCH 22/53] Return pointer to bucket from getBucket

---
 libbeat/processors/rate_limit/algorithm/token_bucket.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index b8827d5c0654..507fe5c7a5f3 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -59,12 +59,11 @@ func (t *tokenBucket) IsAllowed(key string) bool {
 
 	b := t.getBucket(key)
 	allowed := b.withdraw()
-	t.buckets[key] = b
 
 	return allowed
 }
 
-func (t *tokenBucket) getBucket(key string) bucket {
+func (t *tokenBucket) getBucket(key string) *bucket {
 	b, exists := t.buckets[key]
 	if !exists {
 		b = bucket{
@@ -74,8 +73,7 @@ func (t *tokenBucket) getBucket(key string) bucket {
 		t.buckets[key] = b
 	}
 
-	return b
-
+	return &b
 }
 
 func (b *bucket) withdraw() bool {

From 193cb1f1069c121512e85bd69690ce6b62c6b432 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 13:07:59 +0530
Subject: [PATCH 23/53] Keep pointers to buckets in map to avoid map
 reassignment

---
 libbeat/processors/rate_limit/algorithm/token_bucket.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 507fe5c7a5f3..963423200673 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -35,7 +35,7 @@ type bucket struct {
 type tokenBucket struct {
 	limit   Rate
 	depth   float64
-	buckets map[string]bucket
+	buckets map[string]*bucket
 }
 
 func newTokenBucket(config Config) (Algorithm, error) {
@@ -50,7 +50,7 @@ func newTokenBucket(config Config) (Algorithm, error) {
 	return &tokenBucket{
 		config.Limit,
 		config.Limit.value * cfg.BurstMultiplier,
-		make(map[string]bucket, 0),
+		make(map[string]*bucket, 0),
 	}, nil
 }
 
@@ -66,14 +66,14 @@ func (t *tokenBucket) IsAllowed(key string) bool {
 func (t *tokenBucket) getBucket(key string) *bucket {
 	b, exists := t.buckets[key]
 	if !exists {
-		b = bucket{
+		b = &bucket{
 			tokens:        t.depth,
 			lastReplenish: time.Now(),
 		}
 		t.buckets[key] = b
 	}
 
-	return &b
+	return b
 }
 
 func (b *bucket) withdraw() bool {

From 7597c6727b8ddb71570e73cf137b9860661972b9 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 13:08:19 +0530
Subject: [PATCH 24/53] Fix test

---
 libbeat/processors/rate_limit/rate_limit_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/processors/rate_limit/rate_limit_test.go b/libbeat/processors/rate_limit/rate_limit_test.go
index 1818e7c3edbe..3a6c3f2514cd 100644
--- a/libbeat/processors/rate_limit/rate_limit_test.go
+++ b/libbeat/processors/rate_limit/rate_limit_test.go
@@ -124,7 +124,7 @@ func TestRateLimit(t *testing.T) {
 			config: common.MapStr{
 				"limit": "2/s",
 			},
-			delay:     400 * time.Millisecond,
+			delay:     200 * time.Millisecond,
 			inEvents:  inEvents,
 			outEvents: []beat.Event{inEvents[0], inEvents[1], inEvents[3]},
 		},

From f918076a195624c716089e789b696b76fc6f430e Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 13:08:25 +0530
Subject: [PATCH 25/53] Fix logic as we cannot allow withdrawal of fractional
 (<1) tokens

---
 libbeat/processors/rate_limit/algorithm/token_bucket.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 963423200673..e0f91e5180ca 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -77,7 +77,7 @@ func (t *tokenBucket) getBucket(key string) *bucket {
 }
 
 func (b *bucket) withdraw() bool {
-	if b.tokens == 0 {
+	if b.tokens < 1 {
 		return false
 	}
 	b.tokens--

From ae35015f6b5c579ce01cd471a1e6383e3c6f6a95 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 13:19:52 +0530
Subject: [PATCH 26/53] Move burst multiplier default to token_bucket algo

---
 libbeat/processors/rate_limit/algorithm/token_bucket.go | 4 +++-
 libbeat/processors/rate_limit/config.go                 | 4 +---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index e0f91e5180ca..d851281da5ba 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -39,8 +39,10 @@ type tokenBucket struct {
 }
 
 func newTokenBucket(config Config) (Algorithm, error) {
-	var cfg struct {
+	cfg := struct {
 		BurstMultiplier float64 `config:"burst_multiplier"`
+	}{
+		BurstMultiplier: 1.0,
 	}
 
 	if err := config.Config.Unpack(&cfg); err != nil {
diff --git a/libbeat/processors/rate_limit/config.go b/libbeat/processors/rate_limit/config.go
index 9bc64d88b893..6a1231a23732 100644
--- a/libbeat/processors/rate_limit/config.go
+++ b/libbeat/processors/rate_limit/config.go
@@ -34,9 +34,7 @@ type Config struct {
 func (c *Config) SetDefaults() error {
 	if c.Algorithm.Name() == "" {
 		cfg, err := common.NewConfigFrom(map[string]interface{}{
-			"token_bucket": map[string]interface{}{
-				"burst_multiplier": 1.0,
-			},
+			"token_bucket": map[string]interface{}{},
 		})
 
 		if err != nil {

From 1a172293649614469fb8652cc1fd38b91a2a789d Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 15:41:19 +0530
Subject: [PATCH 27/53] Implementing GC

---
 .../rate_limit/algorithm/token_bucket.go      | 80 +++++++++++++++----
 1 file changed, 66 insertions(+), 14 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index d851281da5ba..df7f5749e890 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -36,13 +36,42 @@ type tokenBucket struct {
 	limit   Rate
 	depth   float64
 	buckets map[string]*bucket
+
+	// GC thresholds and metrics
+	gc struct {
+		thresholds tokenBucketGCConfig
+		metrics    tokenBucketGCConfig
+	}
+}
+
+type tokenBucketGCConfig struct {
+	// NumCalls is the number of calls made to IsAllowed. When more than
+	// the specified number of calls are made, GC is performed.
+	NumCalls int `config:"num_calls"`
+
+	// NumBuckets is the number of buckets being utilized by the token
+	// bucket algorithm. When more than the specified number are utilized,
+	// GC is performed.
+	NumBuckets int `config:"num_buckets"`
+}
+
+type tokenBucketConfig struct {
+	BurstMultiplier float64 `config:"burst_multiplier"`
+
+	// GC governs when completely filled token buckets must be deleted
+	// to free up memory. GC is performed when _any_ of the GC conditions
+	// below are met. After each GC, counters corresponding to _each_ of
+	// the GC conditions below are reset.
+	GC tokenBucketGCConfig `config:"gc"`
 }
 
 func newTokenBucket(config Config) (Algorithm, error) {
-	cfg := struct {
-		BurstMultiplier float64 `config:"burst_multiplier"`
-	}{
+	cfg := tokenBucketConfig{
 		BurstMultiplier: 1.0,
+		GC: tokenBucketGCConfig{
+			NumCalls:   10000,
+			NumBuckets: 1000,
+		},
 	}
 
 	if err := config.Config.Unpack(&cfg); err != nil {
@@ -53,28 +82,42 @@ func newTokenBucket(config Config) (Algorithm, error) {
 		config.Limit,
 		config.Limit.value * cfg.BurstMultiplier,
 		make(map[string]*bucket, 0),
+		struct {
+			thresholds tokenBucketGCConfig
+			metrics    tokenBucketGCConfig
+		}{
+			thresholds: tokenBucketGCConfig{
+				NumCalls:   cfg.GC.NumCalls,
+				NumBuckets: cfg.GC.NumBuckets,
+			},
+		},
 	}, nil
 }
 
 func (t *tokenBucket) IsAllowed(key string) bool {
-	t.replenishBuckets()
+	t.runGC()
 
 	b := t.getBucket(key)
 	allowed := b.withdraw()
 
+	t.gc.metrics.NumCalls++
 	return allowed
 }
 
 func (t *tokenBucket) getBucket(key string) *bucket {
 	b, exists := t.buckets[key]
-	if !exists {
-		b = &bucket{
-			tokens:        t.depth,
-			lastReplenish: time.Now(),
-		}
-		t.buckets[key] = b
+	if exists {
+		b.replenish(t.limit)
+		return b
+	}
+
+	b = &bucket{
+		tokens:        t.depth,
+		lastReplenish: time.Now(),
 	}
+	t.buckets[key] = b
 
+	t.gc.metrics.NumBuckets++
 	return b
 }
 
@@ -94,14 +137,19 @@ func (b *bucket) replenish(rate Rate) {
 	b.lastReplenish = time.Now()
 }
 
-func (t *tokenBucket) replenishBuckets() {
-	toDelete := make([]string, 0)
+func (t *tokenBucket) runGC() {
+	// Don't run GC if thresholds haven't been crossed.
+	if (t.gc.metrics.NumBuckets < t.gc.thresholds.NumBuckets) &&
+		(t.gc.metrics.NumCalls < t.gc.thresholds.NumCalls) {
+		return
+	}
 
-	// Add tokens to all buckets according to the rate limit.
+	// Add tokens to all buckets according to the rate limit
+	// and flag full buckets for deletion.
+	toDelete := make([]string, 0)
 	for key, b := range t.buckets {
 		b.replenish(t.limit)
 
-		// If bucket is full, flag it for deletion
 		if b.tokens >= t.depth {
 			toDelete = append(toDelete, key)
 		}
@@ -111,4 +159,8 @@ func (t *tokenBucket) replenishBuckets() {
 	for _, key := range toDelete {
 		delete(t.buckets, key)
 	}
+
+	// Reset GC metrics
+	t.gc.metrics.NumCalls = 0
+	t.gc.metrics.NumBuckets = len(t.buckets)
 }

From 44c31e422862bed575c4e69ff71918c2114a2f59 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 16:00:32 +0530
Subject: [PATCH 28/53] Making the factory take the algo config and return the
 algo

---
 .../rate_limit/algorithm/algorithm.go           | 17 +++++++++++++----
 libbeat/processors/rate_limit/rate_limit.go     | 12 ++++--------
 2 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/rate_limit/algorithm/algorithm.go
index 46e53110747a..f8414aa4c48b 100644
--- a/libbeat/processors/rate_limit/algorithm/algorithm.go
+++ b/libbeat/processors/rate_limit/algorithm/algorithm.go
@@ -20,6 +20,8 @@ package algorithm
 import (
 	"fmt"
 
+	"github.com/pkg/errors"
+
 	"github.com/elastic/beats/v7/libbeat/common"
 )
 
@@ -52,10 +54,17 @@ func register(id string, ctor constructor) {
 
 // Factory returns the requested rate limiting algorithm, if one is found. If not found,
 // an error is returned.
-func Factory(id string) (constructor, error) {
-	if ctor, found := registry[id]; found {
-		return ctor, nil
+func Factory(id string, config Config) (Algorithm, error) {
+	var ctor constructor
+	var found bool
+	if ctor, found = registry[id]; !found {
+		return nil, fmt.Errorf("rate limiting algorithm '%v' not implemented", id)
+	}
+
+	algorithm, err := ctor(config)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not construct algorithm")
 	}
 
-	return nil, fmt.Errorf("rate limiting algorithm '%v' not implemented", id)
+	return algorithm, nil
 }
diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index 1c1df91a6b8c..55297be7d808 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -56,17 +56,13 @@ func New(cfg *common.Config) (processors.Processor, error) {
 		return nil, errors.Wrap(err, "could not set default configuration")
 	}
 
-	algoCtor, err := algorithm.Factory(config.Algorithm.Name())
-	if err != nil {
-		return nil, errors.Wrap(err, "could not instantiate rate limiting algorithm")
-	}
-
-	algo, err := algoCtor(algorithm.Config{
+	algoConfig := algorithm.Config{
 		Limit:  config.Limit,
 		Config: *config.Algorithm.Config(),
-	})
+	}
+	algo, err := algorithm.Factory(config.Algorithm.Name(), algoConfig)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not construct rate limit algorithm")
+		return nil, errors.Wrap(err, "could not construct rate limiting algorithm")
 	}
 
 	p := &rateLimit{

From c3be26516009101c29440a888154f9f163413c5a Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 16:02:26 +0530
Subject: [PATCH 29/53] Reduce nesting level

---
 libbeat/processors/rate_limit/rate_limit.go | 34 ++++++++++-----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index 55297be7d808..dace22f57713 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -98,25 +98,25 @@ func (p *rateLimit) String() string {
 }
 
 func (p *rateLimit) makeKey(event *beat.Event) (string, error) {
-	var key string
-
-	if len(p.config.Fields) > 0 {
-		sort.Strings(p.config.Fields)
-		values := make([]string, len(p.config.Fields))
-		for _, field := range p.config.Fields {
-			value, err := event.GetValue(field)
-			if err != nil && err != common.ErrKeyNotFound {
-				return "", errors.Wrapf(err, "error getting value of field: %v", field)
-			}
-			if err != common.ErrKeyNotFound {
-				value = ""
-			}
-
-			// TODO: check that the value is a scalar?
-			values = append(values, fmt.Sprintf("%v", value))
+	if len(p.config.Fields) == 0 {
+		return "", nil
+	}
+
+	sort.Strings(p.config.Fields)
+	values := make([]string, len(p.config.Fields))
+	for _, field := range p.config.Fields {
+		value, err := event.GetValue(field)
+		if err != nil && err != common.ErrKeyNotFound {
+			return "", errors.Wrapf(err, "error getting value of field: %v", field)
 		}
-		key = strings.Join(values, "_")
+		if err != common.ErrKeyNotFound {
+			value = ""
+		}
+
+		// TODO: check that the value is a scalar?
+		values = append(values, fmt.Sprintf("%v", value))
 	}
+	key := strings.Join(values, "_")
 
 	return key, nil
 }

From 56e4731133f659992f8cadb7616f1999534a8dfe Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 16:15:15 +0530
Subject: [PATCH 30/53] Use mitchellh/hashstructure

---
 libbeat/processors/rate_limit/algorithm/algorithm.go |  2 +-
 .../processors/rate_limit/algorithm/token_bucket.go  | 10 +++++-----
 libbeat/processors/rate_limit/rate_limit.go          | 12 +++++-------
 libbeat/processors/rate_limit/rate_limit_test.go     |  2 +-
 4 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/rate_limit/algorithm/algorithm.go
index f8414aa4c48b..ebda78116d27 100644
--- a/libbeat/processors/rate_limit/algorithm/algorithm.go
+++ b/libbeat/processors/rate_limit/algorithm/algorithm.go
@@ -43,7 +43,7 @@ type Algorithm interface {
 	// (true) or not (false). If a key is allowed, it means it is NOT
 	// rate limited. If a key is not allowed, it means it is being rate
 	// limited.
-	IsAllowed(string) bool
+	IsAllowed(uint64) bool
 }
 
 type constructor func(Config) (Algorithm, error)
diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index df7f5749e890..d2a48779bf61 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -35,7 +35,7 @@ type bucket struct {
 type tokenBucket struct {
 	limit   Rate
 	depth   float64
-	buckets map[string]*bucket
+	buckets map[uint64]*bucket
 
 	// GC thresholds and metrics
 	gc struct {
@@ -81,7 +81,7 @@ func newTokenBucket(config Config) (Algorithm, error) {
 	return &tokenBucket{
 		config.Limit,
 		config.Limit.value * cfg.BurstMultiplier,
-		make(map[string]*bucket, 0),
+		make(map[uint64]*bucket, 0),
 		struct {
 			thresholds tokenBucketGCConfig
 			metrics    tokenBucketGCConfig
@@ -94,7 +94,7 @@ func newTokenBucket(config Config) (Algorithm, error) {
 	}, nil
 }
 
-func (t *tokenBucket) IsAllowed(key string) bool {
+func (t *tokenBucket) IsAllowed(key uint64) bool {
 	t.runGC()
 
 	b := t.getBucket(key)
@@ -104,7 +104,7 @@ func (t *tokenBucket) IsAllowed(key string) bool {
 	return allowed
 }
 
-func (t *tokenBucket) getBucket(key string) *bucket {
+func (t *tokenBucket) getBucket(key uint64) *bucket {
 	b, exists := t.buckets[key]
 	if exists {
 		b.replenish(t.limit)
@@ -146,7 +146,7 @@ func (t *tokenBucket) runGC() {
 
 	// Add tokens to all buckets according to the rate limit
 	// and flag full buckets for deletion.
-	toDelete := make([]string, 0)
+	toDelete := make([]uint64, 0)
 	for key, b := range t.buckets {
 		b.replenish(t.limit)
 
diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index dace22f57713..1b29cd9bad5a 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -20,8 +20,8 @@ package rate_limit
 import (
 	"fmt"
 	"sort"
-	"strings"
 
+	"github.com/mitchellh/hashstructure"
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
@@ -97,9 +97,9 @@ func (p *rateLimit) String() string {
 	)
 }
 
-func (p *rateLimit) makeKey(event *beat.Event) (string, error) {
+func (p *rateLimit) makeKey(event *beat.Event) (uint64, error) {
 	if len(p.config.Fields) == 0 {
-		return "", nil
+		return 0, nil
 	}
 
 	sort.Strings(p.config.Fields)
@@ -107,16 +107,14 @@ func (p *rateLimit) makeKey(event *beat.Event) (string, error) {
 	for _, field := range p.config.Fields {
 		value, err := event.GetValue(field)
 		if err != nil && err != common.ErrKeyNotFound {
-			return "", errors.Wrapf(err, "error getting value of field: %v", field)
+			return 0, errors.Wrapf(err, "error getting value of field: %v", field)
 		}
 		if err != common.ErrKeyNotFound {
 			value = ""
 		}
 
-		// TODO: check that the value is a scalar?
 		values = append(values, fmt.Sprintf("%v", value))
 	}
-	key := strings.Join(values, "_")
 
-	return key, nil
+	return hashstructure.Hash(values, nil)
 }
diff --git a/libbeat/processors/rate_limit/rate_limit_test.go b/libbeat/processors/rate_limit/rate_limit_test.go
index 3a6c3f2514cd..af25a206b4eb 100644
--- a/libbeat/processors/rate_limit/rate_limit_test.go
+++ b/libbeat/processors/rate_limit/rate_limit_test.go
@@ -137,7 +137,7 @@ func TestRateLimit(t *testing.T) {
 			inEvents:  inEvents,
 			outEvents: []beat.Event{inEvents[0], inEvents[2], inEvents[3]},
 		},
-		"with_burst": { // FIXME: test is not working as expected
+		"with_burst": {
 			config: common.MapStr{
 				"limit":            "2/s",
 				"burst_multiplier": 2,

From ae8874bf49c2069710b84ca60d0edde40b8bc893 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 16:49:46 +0530
Subject: [PATCH 31/53] Using sync.Map

---
 .../rate_limit/algorithm/token_bucket.go      | 35 ++++++++++++-------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index d2a48779bf61..43cae6e39b7e 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -18,6 +18,7 @@
 package algorithm
 
 import (
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -35,7 +36,7 @@ type bucket struct {
 type tokenBucket struct {
 	limit   Rate
 	depth   float64
-	buckets map[uint64]*bucket
+	buckets sync.Map
 
 	// GC thresholds and metrics
 	gc struct {
@@ -81,7 +82,7 @@ func newTokenBucket(config Config) (Algorithm, error) {
 	return &tokenBucket{
 		config.Limit,
 		config.Limit.value * cfg.BurstMultiplier,
-		make(map[uint64]*bucket, 0),
+		sync.Map{},
 		struct {
 			thresholds tokenBucketGCConfig
 			metrics    tokenBucketGCConfig
@@ -105,18 +106,17 @@ func (t *tokenBucket) IsAllowed(key uint64) bool {
 }
 
 func (t *tokenBucket) getBucket(key uint64) *bucket {
-	b, exists := t.buckets[key]
+	v, exists := t.buckets.LoadOrStore(key, &bucket{
+		tokens:        t.depth,
+		lastReplenish: time.Now(),
+	})
+	b := v.(*bucket)
+
 	if exists {
 		b.replenish(t.limit)
 		return b
 	}
 
-	b = &bucket{
-		tokens:        t.depth,
-		lastReplenish: time.Now(),
-	}
-	t.buckets[key] = b
-
 	t.gc.metrics.NumBuckets++
 	return b
 }
@@ -147,20 +147,29 @@ func (t *tokenBucket) runGC() {
 	// Add tokens to all buckets according to the rate limit
 	// and flag full buckets for deletion.
 	toDelete := make([]uint64, 0)
-	for key, b := range t.buckets {
+	numBuckets := 0
+	t.buckets.Range(func(k, v interface{}) bool {
+		key := k.(uint64)
+		b := v.(*bucket)
+
 		b.replenish(t.limit)
 
 		if b.tokens >= t.depth {
 			toDelete = append(toDelete, key)
 		}
-	}
+
+		numBuckets++
+
+		return true
+	})
 
 	// Cleanup full buckets to free up memory
 	for _, key := range toDelete {
-		delete(t.buckets, key)
+		t.buckets.Delete(key)
+		numBuckets--
 	}
 
 	// Reset GC metrics
 	t.gc.metrics.NumCalls = 0
-	t.gc.metrics.NumBuckets = len(t.buckets)
+	t.gc.metrics.NumBuckets = numBuckets
 }

From 900019c3023edd1c78ee219d035b58fed79358ad Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 16:51:03 +0530
Subject: [PATCH 32/53] Removing algorithm and algorithm options from
 documentation

---
 .../rate_limit/docs/rate_limit.asciidoc       | 24 -------------------
 1 file changed, 24 deletions(-)

diff --git a/libbeat/processors/rate_limit/docs/rate_limit.asciidoc b/libbeat/processors/rate_limit/docs/rate_limit.asciidoc
index 1666c27f8c19..5a63a6d54d0a 100644
--- a/libbeat/processors/rate_limit/docs/rate_limit.asciidoc
+++ b/libbeat/processors/rate_limit/docs/rate_limit.asciidoc
@@ -37,27 +37,3 @@ The following settings are supported:
 
 `limit`:: The rate limit. Supported time units for the rate are `s` (per second), `m` (per minute), and `h` (per hour).
 `fields`:: (Optional) List of fields. The rate limit will be applied to each distinct value derived by combining the values of these fields.
-`algorithm`:: (Optional) The rate limiting algorithm to use. The only supported algorithm at the moment is `token_bucket`.
-
-Rate limiting algorithms may support additional, algorithm-specific settings.
-
-[source,yaml]
------------------------------------------------------
-processors:
-- rate_limit:
-   limit: "10000/m"
-   algorithm:
-     token_bucket: ~
------------------------------------------------------
-
-[source,yaml]
------------------------------------------------------
-processors:
-- rate_limit:
-   limit: "10000/m"
-   algorithm:
-     token_bucket:
-       burst_multiplier: 1.0
------------------------------------------------------
-
-`algorithm.token_bucket.burst_multiplier`:: How much burstability of events should the token bucket allow for. Optional. Default = 1.0 which means no burstability. TODO: explain better.

From 58124f553ede329ecdb4d30bca1457b55a5b214e Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 17:55:12 +0530
Subject: [PATCH 33/53] Mocking clock

---
 .../rate_limit/algorithm/algorithm.go         |  4 ++++
 .../rate_limit/algorithm/token_bucket.go      | 22 ++++++++++++++-----
 libbeat/processors/rate_limit/clock/clock.go  | 10 +++++++++
 .../processors/rate_limit/clock/fake_clock.go | 19 ++++++++++++++++
 .../processors/rate_limit/clock/real_clock.go | 13 +++++++++++
 libbeat/processors/rate_limit/rate_limit.go   | 12 ++++++++++
 .../processors/rate_limit/rate_limit_test.go  |  8 ++++++-
 7 files changed, 81 insertions(+), 7 deletions(-)
 create mode 100644 libbeat/processors/rate_limit/clock/clock.go
 create mode 100644 libbeat/processors/rate_limit/clock/fake_clock.go
 create mode 100644 libbeat/processors/rate_limit/clock/real_clock.go

diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/rate_limit/algorithm/algorithm.go
index ebda78116d27..e84eef2efaf3 100644
--- a/libbeat/processors/rate_limit/algorithm/algorithm.go
+++ b/libbeat/processors/rate_limit/algorithm/algorithm.go
@@ -23,6 +23,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/clock"
 )
 
 var registry = make(map[string]constructor, 0)
@@ -44,6 +45,9 @@ type Algorithm interface {
 	// rate limited. If a key is not allowed, it means it is being rate
 	// limited.
 	IsAllowed(uint64) bool
+
+	// SetClock allows test code to inject a fake clock.
+	SetClock(clock.Clock)
 }
 
 type constructor func(Config) (Algorithm, error)
diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 43cae6e39b7e..81abb4f4ce0d 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -22,6 +22,8 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/clock"
 )
 
 func init() {
@@ -43,6 +45,8 @@ type tokenBucket struct {
 		thresholds tokenBucketGCConfig
 		metrics    tokenBucketGCConfig
 	}
+
+	clock clock.Clock
 }
 
 type tokenBucketGCConfig struct {
@@ -92,6 +96,7 @@ func newTokenBucket(config Config) (Algorithm, error) {
 				NumBuckets: cfg.GC.NumBuckets,
 			},
 		},
+		clock.RealClock{},
 	}, nil
 }
 
@@ -105,15 +110,20 @@ func (t *tokenBucket) IsAllowed(key uint64) bool {
 	return allowed
 }
 
+// SetClock allows test code to inject a fake clock
+func (t *tokenBucket) SetClock(c clock.Clock) {
+	t.clock = c
+}
+
 func (t *tokenBucket) getBucket(key uint64) *bucket {
 	v, exists := t.buckets.LoadOrStore(key, &bucket{
 		tokens:        t.depth,
-		lastReplenish: time.Now(),
+		lastReplenish: t.clock.Now(),
 	})
 	b := v.(*bucket)
 
 	if exists {
-		b.replenish(t.limit)
+		b.replenish(t.limit, t.clock)
 		return b
 	}
 
@@ -129,12 +139,12 @@ func (b *bucket) withdraw() bool {
 	return true
 }
 
-func (b *bucket) replenish(rate Rate) {
-	secsSinceLastReplenish := time.Now().Sub(b.lastReplenish).Seconds()
+func (b *bucket) replenish(rate Rate, clock clock.Clock) {
+	secsSinceLastReplenish := clock.Now().Sub(b.lastReplenish).Seconds()
 	tokensToReplenish := secsSinceLastReplenish * rate.valuePerSecond()
 
 	b.tokens += tokensToReplenish
-	b.lastReplenish = time.Now()
+	b.lastReplenish = clock.Now()
 }
 
 func (t *tokenBucket) runGC() {
@@ -152,7 +162,7 @@ func (t *tokenBucket) runGC() {
 		key := k.(uint64)
 		b := v.(*bucket)
 
-		b.replenish(t.limit)
+		b.replenish(t.limit, t.clock)
 
 		if b.tokens >= t.depth {
 			toDelete = append(toDelete, key)
diff --git a/libbeat/processors/rate_limit/clock/clock.go b/libbeat/processors/rate_limit/clock/clock.go
new file mode 100644
index 000000000000..09ff724ced18
--- /dev/null
+++ b/libbeat/processors/rate_limit/clock/clock.go
@@ -0,0 +1,10 @@
+package clock
+
+import (
+	"time"
+)
+
+type Clock interface {
+	Now() time.Time
+	Sleep(duration time.Duration)
+}
diff --git a/libbeat/processors/rate_limit/clock/fake_clock.go b/libbeat/processors/rate_limit/clock/fake_clock.go
new file mode 100644
index 000000000000..1276e3bca09b
--- /dev/null
+++ b/libbeat/processors/rate_limit/clock/fake_clock.go
@@ -0,0 +1,19 @@
+package clock
+
+import "time"
+
+type FakeClock struct {
+	now time.Time
+}
+
+func (f *FakeClock) Now() time.Time {
+	return f.now
+}
+
+func (f *FakeClock) SetNow(now time.Time) {
+	f.now = now
+}
+
+func (f *FakeClock) Sleep(d time.Duration) {
+	f.now = f.now.Add(d)
+}
diff --git a/libbeat/processors/rate_limit/clock/real_clock.go b/libbeat/processors/rate_limit/clock/real_clock.go
new file mode 100644
index 000000000000..d9624483c3ec
--- /dev/null
+++ b/libbeat/processors/rate_limit/clock/real_clock.go
@@ -0,0 +1,13 @@
+package clock
+
+import "time"
+
+type RealClock struct{}
+
+func (r RealClock) Now() time.Time {
+	return time.Now()
+}
+
+func (r RealClock) Sleep(d time.Duration) {
+	time.Sleep(d)
+}
diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index 1b29cd9bad5a..6240bce76799 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -29,6 +29,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/processors"
 	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/algorithm"
+	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/clock"
 	jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor"
 )
 
@@ -43,6 +44,9 @@ type rateLimit struct {
 	config    Config
 	algorithm algorithm.Algorithm
 	logger    *logp.Logger
+
+	// For testing
+	c clock.Clock
 }
 
 // New constructs a new rate limit processor.
@@ -71,6 +75,8 @@ func New(cfg *common.Config) (processors.Processor, error) {
 		logger:    logp.NewLogger("rate_limit"),
 	}
 
+	p.setClock(clock.RealClock{})
+
 	return p, nil
 }
 
@@ -118,3 +124,9 @@ func (p *rateLimit) makeKey(event *beat.Event) (uint64, error) {
 
 	return hashstructure.Hash(values, nil)
 }
+
+// setClock allows test code to inject a fake clock
+func (p *rateLimit) setClock(c clock.Clock) {
+	p.c = c
+	p.algorithm.SetClock(c)
+}
diff --git a/libbeat/processors/rate_limit/rate_limit_test.go b/libbeat/processors/rate_limit/rate_limit_test.go
index af25a206b4eb..bc386a333869 100644
--- a/libbeat/processors/rate_limit/rate_limit_test.go
+++ b/libbeat/processors/rate_limit/rate_limit_test.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/clock"
 )
 
 func TestNew(t *testing.T) {
@@ -153,6 +154,11 @@ func TestRateLimit(t *testing.T) {
 			p, err := New(common.MustNewConfigFrom(test.config))
 			require.NoError(t, err)
 
+			fakeClock := clock.FakeClock{}
+			fakeClock.SetNow(time.Now())
+
+			p.(*rateLimit).setClock(&fakeClock)
+
 			out := make([]beat.Event, 0)
 			for _, in := range test.inEvents {
 				o, err := p.Run(&in)
@@ -160,7 +166,7 @@ func TestRateLimit(t *testing.T) {
 				if o != nil {
 					out = append(out, *o)
 				}
-				time.Sleep(test.delay)
+				fakeClock.Sleep(test.delay)
 			}
 
 			require.Equal(t, test.outEvents, out)

From eb81d1354b86b2b1b4e762819abd0b656419c3fb Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 17:59:01 +0530
Subject: [PATCH 34/53] Adding license headers

---
 libbeat/processors/rate_limit/clock/clock.go    | 17 +++++++++++++++++
 .../processors/rate_limit/clock/fake_clock.go   | 17 +++++++++++++++++
 .../processors/rate_limit/clock/real_clock.go   | 17 +++++++++++++++++
 3 files changed, 51 insertions(+)

diff --git a/libbeat/processors/rate_limit/clock/clock.go b/libbeat/processors/rate_limit/clock/clock.go
index 09ff724ced18..d40467105e3a 100644
--- a/libbeat/processors/rate_limit/clock/clock.go
+++ b/libbeat/processors/rate_limit/clock/clock.go
@@ -1,3 +1,20 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package clock
 
 import (
diff --git a/libbeat/processors/rate_limit/clock/fake_clock.go b/libbeat/processors/rate_limit/clock/fake_clock.go
index 1276e3bca09b..d64bf4c35e6f 100644
--- a/libbeat/processors/rate_limit/clock/fake_clock.go
+++ b/libbeat/processors/rate_limit/clock/fake_clock.go
@@ -1,3 +1,20 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package clock
 
 import "time"
diff --git a/libbeat/processors/rate_limit/clock/real_clock.go b/libbeat/processors/rate_limit/clock/real_clock.go
index d9624483c3ec..81b7183ed7fb 100644
--- a/libbeat/processors/rate_limit/clock/real_clock.go
+++ b/libbeat/processors/rate_limit/clock/real_clock.go
@@ -1,3 +1,20 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package clock
 
 import "time"

From 742ed41d4ec5e7a87734f8b0e6cc7ba4bb34cf4e Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 20:23:06 +0530
Subject: [PATCH 35/53] Using atomic.Uints for metrics counters

---
 .../rate_limit/algorithm/token_bucket.go      | 39 +++++++++++--------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 81abb4f4ce0d..7ce7659625e7 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/elastic/beats/v7/libbeat/common/atomic"
 	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/clock"
 )
 
@@ -43,7 +44,10 @@ type tokenBucket struct {
 	// GC thresholds and metrics
 	gc struct {
 		thresholds tokenBucketGCConfig
-		metrics    tokenBucketGCConfig
+		metrics    struct {
+			numCalls   atomic.Uint
+			numBuckets atomic.Uint
+		}
 	}
 
 	clock clock.Clock
@@ -52,12 +56,12 @@ type tokenBucket struct {
 type tokenBucketGCConfig struct {
 	// NumCalls is the number of calls made to IsAllowed. When more than
 	// the specified number of calls are made, GC is performed.
-	NumCalls int `config:"num_calls"`
+	NumCalls uint `config:"num_calls"`
 
 	// NumBuckets is the number of buckets being utilized by the token
 	// bucket algorithm. When more than the specified number are utilized,
 	// GC is performed.
-	NumBuckets int `config:"num_buckets"`
+	NumBuckets uint `config:"num_buckets"`
 }
 
 type tokenBucketConfig struct {
@@ -84,19 +88,22 @@ func newTokenBucket(config Config) (Algorithm, error) {
 	}
 
 	return &tokenBucket{
-		config.Limit,
-		config.Limit.value * cfg.BurstMultiplier,
-		sync.Map{},
-		struct {
+		limit:   config.Limit,
+		depth:   config.Limit.value * cfg.BurstMultiplier,
+		buckets: sync.Map{},
+		gc: struct {
 			thresholds tokenBucketGCConfig
-			metrics    tokenBucketGCConfig
+			metrics    struct {
+				numCalls   atomic.Uint
+				numBuckets atomic.Uint
+			}
 		}{
 			thresholds: tokenBucketGCConfig{
 				NumCalls:   cfg.GC.NumCalls,
 				NumBuckets: cfg.GC.NumBuckets,
 			},
 		},
-		clock.RealClock{},
+		clock: clock.RealClock{},
 	}, nil
 }
 
@@ -106,7 +113,7 @@ func (t *tokenBucket) IsAllowed(key uint64) bool {
 	b := t.getBucket(key)
 	allowed := b.withdraw()
 
-	t.gc.metrics.NumCalls++
+	t.gc.metrics.numCalls.Inc()
 	return allowed
 }
 
@@ -127,7 +134,7 @@ func (t *tokenBucket) getBucket(key uint64) *bucket {
 		return b
 	}
 
-	t.gc.metrics.NumBuckets++
+	t.gc.metrics.numBuckets.Inc()
 	return b
 }
 
@@ -149,15 +156,15 @@ func (b *bucket) replenish(rate Rate, clock clock.Clock) {
 
 func (t *tokenBucket) runGC() {
 	// Don't run GC if thresholds haven't been crossed.
-	if (t.gc.metrics.NumBuckets < t.gc.thresholds.NumBuckets) &&
-		(t.gc.metrics.NumCalls < t.gc.thresholds.NumCalls) {
+	if (t.gc.metrics.numBuckets.Load() < t.gc.thresholds.NumBuckets) &&
+		(t.gc.metrics.numCalls.Load() < t.gc.thresholds.NumCalls) {
 		return
 	}
 
 	// Add tokens to all buckets according to the rate limit
 	// and flag full buckets for deletion.
 	toDelete := make([]uint64, 0)
-	numBuckets := 0
+	var numBuckets uint
 	t.buckets.Range(func(k, v interface{}) bool {
 		key := k.(uint64)
 		b := v.(*bucket)
@@ -180,6 +187,6 @@ func (t *tokenBucket) runGC() {
 	}
 
 	// Reset GC metrics
-	t.gc.metrics.NumCalls = 0
-	t.gc.metrics.NumBuckets = numBuckets
+	t.gc.metrics.numCalls = atomic.MakeUint(0)
+	t.gc.metrics.numBuckets = atomic.MakeUint(numBuckets)
 }

From b4990a02cc481861fcf2825a8040fb492ad4d491 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 20:33:50 +0530
Subject: [PATCH 36/53] Fixing logic

---
 libbeat/processors/rate_limit/rate_limit.go | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index 6240bce76799..dc0cd775a3e7 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -112,10 +112,11 @@ func (p *rateLimit) makeKey(event *beat.Event) (uint64, error) {
 	values := make([]string, len(p.config.Fields))
 	for _, field := range p.config.Fields {
 		value, err := event.GetValue(field)
-		if err != nil && err != common.ErrKeyNotFound {
-			return 0, errors.Wrapf(err, "error getting value of field: %v", field)
-		}
-		if err != common.ErrKeyNotFound {
+		if err != nil {
+			if err != common.ErrKeyNotFound {
+				return 0, errors.Wrapf(err, "error getting value of field: %v", field)
+			}
+
 			value = ""
 		}
 

From 40e1382e67e891623e4c8ad4b467ba3467d882ec Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 20:51:38 +0530
Subject: [PATCH 37/53] Use github.com/jonboulle/clockwork

---
 go.mod                                        |  1 +
 go.sum                                        |  3 ++
 .../rate_limit/algorithm/algorithm.go         |  4 +--
 .../rate_limit/algorithm/token_bucket.go      | 10 +++---
 libbeat/processors/rate_limit/clock/clock.go  | 27 --------------
 .../processors/rate_limit/clock/fake_clock.go | 36 -------------------
 .../processors/rate_limit/clock/real_clock.go | 30 ----------------
 libbeat/processors/rate_limit/rate_limit.go   |  8 ++---
 .../processors/rate_limit/rate_limit_test.go  |  9 +++--
 9 files changed, 19 insertions(+), 109 deletions(-)
 delete mode 100644 libbeat/processors/rate_limit/clock/clock.go
 delete mode 100644 libbeat/processors/rate_limit/clock/fake_clock.go
 delete mode 100644 libbeat/processors/rate_limit/clock/real_clock.go

diff --git a/go.mod b/go.mod
index afd34ca9cf39..61ab57020b6a 100644
--- a/go.mod
+++ b/go.mod
@@ -104,6 +104,7 @@ require (
 	github.com/insomniacslk/dhcp v0.0.0-20180716145214-633285ba52b2
 	github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5
 	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901
+	github.com/jonboulle/clockwork v0.2.2
 	github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/jstemmer/go-junit-report v0.9.1
diff --git a/go.sum b/go.sum
index 8662993d8120..964188473c5f 100644
--- a/go.sum
+++ b/go.sum
@@ -244,6 +244,7 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
+github.com/elastic/beats v7.6.2+incompatible h1:jHdLv83KURaqWUC6f55iMyVP6LYZrgElfeqxKWcskVE=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ecs v1.6.0 h1:8NmgfnsjmKXh9hVsK3H2tZtfUptepNc3msJOAynhtmc=
@@ -465,6 +466,8 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
+github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd h1:KikNiFwUO3QLyeKyN4k9yBH9Pcu/gU/yficWi61cJIw=
 github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd/go.mod h1:eJTEwMjXb7kZ633hO3Ln9mBUCOjX2+FlTljvpl9SYdE=
 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/rate_limit/algorithm/algorithm.go
index e84eef2efaf3..cbb995a8187e 100644
--- a/libbeat/processors/rate_limit/algorithm/algorithm.go
+++ b/libbeat/processors/rate_limit/algorithm/algorithm.go
@@ -20,10 +20,10 @@ package algorithm
 import (
 	"fmt"
 
+	"github.com/jonboulle/clockwork"
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/clock"
 )
 
 var registry = make(map[string]constructor, 0)
@@ -47,7 +47,7 @@ type Algorithm interface {
 	IsAllowed(uint64) bool
 
 	// SetClock allows test code to inject a fake clock.
-	SetClock(clock.Clock)
+	SetClock(clockwork.Clock)
 }
 
 type constructor func(Config) (Algorithm, error)
diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 7ce7659625e7..35909839ba96 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -21,10 +21,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/jonboulle/clockwork"
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common/atomic"
-	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/clock"
 )
 
 func init() {
@@ -50,7 +50,7 @@ type tokenBucket struct {
 		}
 	}
 
-	clock clock.Clock
+	clock clockwork.Clock
 }
 
 type tokenBucketGCConfig struct {
@@ -103,7 +103,7 @@ func newTokenBucket(config Config) (Algorithm, error) {
 				NumBuckets: cfg.GC.NumBuckets,
 			},
 		},
-		clock: clock.RealClock{},
+		clock: clockwork.NewRealClock(),
 	}, nil
 }
 
@@ -118,7 +118,7 @@ func (t *tokenBucket) IsAllowed(key uint64) bool {
 }
 
 // SetClock allows test code to inject a fake clock
-func (t *tokenBucket) SetClock(c clock.Clock) {
+func (t *tokenBucket) SetClock(c clockwork.Clock) {
 	t.clock = c
 }
 
@@ -146,7 +146,7 @@ func (b *bucket) withdraw() bool {
 	return true
 }
 
-func (b *bucket) replenish(rate Rate, clock clock.Clock) {
+func (b *bucket) replenish(rate Rate, clock clockwork.Clock) {
 	secsSinceLastReplenish := clock.Now().Sub(b.lastReplenish).Seconds()
 	tokensToReplenish := secsSinceLastReplenish * rate.valuePerSecond()
 
diff --git a/libbeat/processors/rate_limit/clock/clock.go b/libbeat/processors/rate_limit/clock/clock.go
deleted file mode 100644
index d40467105e3a..000000000000
--- a/libbeat/processors/rate_limit/clock/clock.go
+++ /dev/null
@@ -1,27 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package clock
-
-import (
-	"time"
-)
-
-type Clock interface {
-	Now() time.Time
-	Sleep(duration time.Duration)
-}
diff --git a/libbeat/processors/rate_limit/clock/fake_clock.go b/libbeat/processors/rate_limit/clock/fake_clock.go
deleted file mode 100644
index d64bf4c35e6f..000000000000
--- a/libbeat/processors/rate_limit/clock/fake_clock.go
+++ /dev/null
@@ -1,36 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package clock
-
-import "time"
-
-type FakeClock struct {
-	now time.Time
-}
-
-func (f *FakeClock) Now() time.Time {
-	return f.now
-}
-
-func (f *FakeClock) SetNow(now time.Time) {
-	f.now = now
-}
-
-func (f *FakeClock) Sleep(d time.Duration) {
-	f.now = f.now.Add(d)
-}
diff --git a/libbeat/processors/rate_limit/clock/real_clock.go b/libbeat/processors/rate_limit/clock/real_clock.go
deleted file mode 100644
index 81b7183ed7fb..000000000000
--- a/libbeat/processors/rate_limit/clock/real_clock.go
+++ /dev/null
@@ -1,30 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package clock
-
-import "time"
-
-type RealClock struct{}
-
-func (r RealClock) Now() time.Time {
-	return time.Now()
-}
-
-func (r RealClock) Sleep(d time.Duration) {
-	time.Sleep(d)
-}
diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index dc0cd775a3e7..836893fbe45a 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"sort"
 
+	"github.com/jonboulle/clockwork"
 	"github.com/mitchellh/hashstructure"
 	"github.com/pkg/errors"
 
@@ -29,7 +30,6 @@ import (
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/processors"
 	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/algorithm"
-	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/clock"
 	jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor"
 )
 
@@ -46,7 +46,7 @@ type rateLimit struct {
 	logger    *logp.Logger
 
 	// For testing
-	c clock.Clock
+	c clockwork.Clock
 }
 
 // New constructs a new rate limit processor.
@@ -75,7 +75,7 @@ func New(cfg *common.Config) (processors.Processor, error) {
 		logger:    logp.NewLogger("rate_limit"),
 	}
 
-	p.setClock(clock.RealClock{})
+	p.setClock(clockwork.NewRealClock())
 
 	return p, nil
 }
@@ -127,7 +127,7 @@ func (p *rateLimit) makeKey(event *beat.Event) (uint64, error) {
 }
 
 // setClock allows test code to inject a fake clock
-func (p *rateLimit) setClock(c clock.Clock) {
+func (p *rateLimit) setClock(c clockwork.Clock) {
 	p.c = c
 	p.algorithm.SetClock(c)
 }
diff --git a/libbeat/processors/rate_limit/rate_limit_test.go b/libbeat/processors/rate_limit/rate_limit_test.go
index bc386a333869..40bd971bdd5a 100644
--- a/libbeat/processors/rate_limit/rate_limit_test.go
+++ b/libbeat/processors/rate_limit/rate_limit_test.go
@@ -21,11 +21,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/jonboulle/clockwork"
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/clock"
 )
 
 func TestNew(t *testing.T) {
@@ -154,10 +154,9 @@ func TestRateLimit(t *testing.T) {
 			p, err := New(common.MustNewConfigFrom(test.config))
 			require.NoError(t, err)
 
-			fakeClock := clock.FakeClock{}
-			fakeClock.SetNow(time.Now())
+			fakeClock := clockwork.NewFakeClock()
 
-			p.(*rateLimit).setClock(&fakeClock)
+			p.(*rateLimit).setClock(fakeClock)
 
 			out := make([]beat.Event, 0)
 			for _, in := range test.inEvents {
@@ -166,7 +165,7 @@ func TestRateLimit(t *testing.T) {
 				if o != nil {
 					out = append(out, *o)
 				}
-				fakeClock.Sleep(test.delay)
+				fakeClock.Advance(test.delay)
 			}
 
 			require.Equal(t, test.outEvents, out)

From a220fbfb5ccbd45de9363eb811d1937379414981 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Tue, 8 Dec 2020 20:56:59 +0530
Subject: [PATCH 38/53] Running make update

---
 NOTICE.txt | 211 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 go.sum     |   1 -
 2 files changed, 211 insertions(+), 1 deletion(-)

diff --git a/NOTICE.txt b/NOTICE.txt
index 2d574adf53cf..2fa71582363a 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -11032,6 +11032,217 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/jonboulle/clockwork
+Version: v0.2.2
+Licence type (autodetected): Apache-2.0
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/jonboulle/clockwork@v0.2.2/LICENSE:
+
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/josephspurrier/goversioninfo
 Version: v0.0.0-20190209210621-63e6d1acd3dd
diff --git a/go.sum b/go.sum
index 964188473c5f..58fd46efade3 100644
--- a/go.sum
+++ b/go.sum
@@ -244,7 +244,6 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
-github.com/elastic/beats v7.6.2+incompatible h1:jHdLv83KURaqWUC6f55iMyVP6LYZrgElfeqxKWcskVE=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ecs v1.6.0 h1:8NmgfnsjmKXh9hVsK3H2tZtfUptepNc3msJOAynhtmc=

From 4b530546b72e03b9ce64d0ce64fc5e234bb21b09 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Wed, 9 Dec 2020 10:20:18 +0530
Subject: [PATCH 39/53] Add mutex to prevent only one GC thread from running at
 any time

---
 .../rate_limit/algorithm/token_bucket.go      | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 35909839ba96..9d87868eb1de 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -37,6 +37,8 @@ type bucket struct {
 }
 
 type tokenBucket struct {
+	mu sync.RWMutex
+
 	limit   Rate
 	depth   float64
 	buckets sync.Map
@@ -161,10 +163,21 @@ func (t *tokenBucket) runGC() {
 		return
 	}
 
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	// Don't run GC if thresholds haven't been crossed.
+	// Check again in case another GC thread has run while this GC thread
+	// was waiting to acquire the lock and the other GC thread reset the
+	// metrics.
+	if (t.gc.metrics.numBuckets.Load() < t.gc.thresholds.NumBuckets) &&
+		(t.gc.metrics.numCalls.Load() < t.gc.thresholds.NumCalls) {
+		return
+	}
+
 	// Add tokens to all buckets according to the rate limit
 	// and flag full buckets for deletion.
 	toDelete := make([]uint64, 0)
-	var numBuckets uint
 	t.buckets.Range(func(k, v interface{}) bool {
 		key := k.(uint64)
 		b := v.(*bucket)
@@ -175,18 +188,15 @@ func (t *tokenBucket) runGC() {
 			toDelete = append(toDelete, key)
 		}
 
-		numBuckets++
-
 		return true
 	})
 
 	// Cleanup full buckets to free up memory
 	for _, key := range toDelete {
 		t.buckets.Delete(key)
-		numBuckets--
 	}
 
 	// Reset GC metrics
 	t.gc.metrics.numCalls = atomic.MakeUint(0)
-	t.gc.metrics.numBuckets = atomic.MakeUint(numBuckets)
+	t.gc.metrics.numBuckets.Sub(uint(len(toDelete)))
 }

From c36b1867bc85ea6f70c34e3c487187f316eacba6 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 08:38:13 +0530
Subject: [PATCH 40/53] Adding logging

---
 .../rate_limit/algorithm/token_bucket.go        | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 9d87868eb1de..1aa94a43b6fd 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -25,6 +25,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common/atomic"
+	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
 func init() {
@@ -52,7 +53,8 @@ type tokenBucket struct {
 		}
 	}
 
-	clock clockwork.Clock
+	clock  clockwork.Clock
+	logger *logp.Logger
 }
 
 type tokenBucketGCConfig struct {
@@ -105,7 +107,8 @@ func newTokenBucket(config Config) (Algorithm, error) {
 				NumBuckets: cfg.GC.NumBuckets,
 			},
 		},
-		clock: clockwork.NewRealClock(),
+		clock:  clockwork.NewRealClock(),
+		logger: logp.NewLogger("token_bucket"),
 	}, nil
 }
 
@@ -175,9 +178,12 @@ func (t *tokenBucket) runGC() {
 		return
 	}
 
+	gcStartTime := time.Now()
+
 	// Add tokens to all buckets according to the rate limit
 	// and flag full buckets for deletion.
 	toDelete := make([]uint64, 0)
+	numBucketsBefore := 0
 	t.buckets.Range(func(k, v interface{}) bool {
 		key := k.(uint64)
 		b := v.(*bucket)
@@ -188,6 +194,7 @@ func (t *tokenBucket) runGC() {
 			toDelete = append(toDelete, key)
 		}
 
+		numBucketsBefore++
 		return true
 	})
 
@@ -199,4 +206,10 @@ func (t *tokenBucket) runGC() {
 	// Reset GC metrics
 	t.gc.metrics.numCalls = atomic.MakeUint(0)
 	t.gc.metrics.numBuckets.Sub(uint(len(toDelete)))
+
+	gcDuration := time.Now().Sub(gcStartTime)
+	numBucketsDeleted := len(toDelete)
+	numBucketsAfter := numBucketsBefore - numBucketsDeleted
+	t.logger.Debugf("gc duration: %v, buckets: (before: %v, deleted: %v, after: %v)",
+		gcDuration, numBucketsBefore, numBucketsDeleted, numBucketsAfter)
 }

From c89eccad7b11ffe68b14d29e022cee69bf4c8742 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 08:40:15 +0530
Subject: [PATCH 41/53] Remove NumBuckets GC threshold

---
 .../rate_limit/algorithm/token_bucket.go      | 25 +++++--------------
 1 file changed, 6 insertions(+), 19 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 1aa94a43b6fd..ccfc5500c545 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -48,8 +48,7 @@ type tokenBucket struct {
 	gc struct {
 		thresholds tokenBucketGCConfig
 		metrics    struct {
-			numCalls   atomic.Uint
-			numBuckets atomic.Uint
+			numCalls atomic.Uint
 		}
 	}
 
@@ -61,11 +60,6 @@ type tokenBucketGCConfig struct {
 	// NumCalls is the number of calls made to IsAllowed. When more than
 	// the specified number of calls are made, GC is performed.
 	NumCalls uint `config:"num_calls"`
-
-	// NumBuckets is the number of buckets being utilized by the token
-	// bucket algorithm. When more than the specified number are utilized,
-	// GC is performed.
-	NumBuckets uint `config:"num_buckets"`
 }
 
 type tokenBucketConfig struct {
@@ -82,8 +76,7 @@ func newTokenBucket(config Config) (Algorithm, error) {
 	cfg := tokenBucketConfig{
 		BurstMultiplier: 1.0,
 		GC: tokenBucketGCConfig{
-			NumCalls:   10000,
-			NumBuckets: 1000,
+			NumCalls: 10000,
 		},
 	}
 
@@ -98,13 +91,11 @@ func newTokenBucket(config Config) (Algorithm, error) {
 		gc: struct {
 			thresholds tokenBucketGCConfig
 			metrics    struct {
-				numCalls   atomic.Uint
-				numBuckets atomic.Uint
+				numCalls atomic.Uint
 			}
 		}{
 			thresholds: tokenBucketGCConfig{
-				NumCalls:   cfg.GC.NumCalls,
-				NumBuckets: cfg.GC.NumBuckets,
+				NumCalls: cfg.GC.NumCalls,
 			},
 		},
 		clock:  clockwork.NewRealClock(),
@@ -139,7 +130,6 @@ func (t *tokenBucket) getBucket(key uint64) *bucket {
 		return b
 	}
 
-	t.gc.metrics.numBuckets.Inc()
 	return b
 }
 
@@ -161,8 +151,7 @@ func (b *bucket) replenish(rate Rate, clock clockwork.Clock) {
 
 func (t *tokenBucket) runGC() {
 	// Don't run GC if thresholds haven't been crossed.
-	if (t.gc.metrics.numBuckets.Load() < t.gc.thresholds.NumBuckets) &&
-		(t.gc.metrics.numCalls.Load() < t.gc.thresholds.NumCalls) {
+	if t.gc.metrics.numCalls.Load() < t.gc.thresholds.NumCalls {
 		return
 	}
 
@@ -173,8 +162,7 @@ func (t *tokenBucket) runGC() {
 	// Check again in case another GC thread has run while this GC thread
 	// was waiting to acquire the lock and the other GC thread reset the
 	// metrics.
-	if (t.gc.metrics.numBuckets.Load() < t.gc.thresholds.NumBuckets) &&
-		(t.gc.metrics.numCalls.Load() < t.gc.thresholds.NumCalls) {
+	if t.gc.metrics.numCalls.Load() < t.gc.thresholds.NumCalls {
 		return
 	}
 
@@ -205,7 +193,6 @@ func (t *tokenBucket) runGC() {
 
 	// Reset GC metrics
 	t.gc.metrics.numCalls = atomic.MakeUint(0)
-	t.gc.metrics.numBuckets.Sub(uint(len(toDelete)))
 
 	gcDuration := time.Now().Sub(gcStartTime)
 	numBucketsDeleted := len(toDelete)

From 425b03ac602dd329db9961df744501a2b2039c1e Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 08:43:47 +0530
Subject: [PATCH 42/53] Use non-blocking mutex

---
 .../processors/rate_limit/algorithm/token_bucket.go | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index ccfc5500c545..f7b77d9d9205 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/elastic/go-concert/unison"
 	"github.com/jonboulle/clockwork"
 	"github.com/pkg/errors"
 
@@ -38,7 +39,7 @@ type bucket struct {
 }
 
 type tokenBucket struct {
-	mu sync.RWMutex
+	mu unison.Mutex
 
 	limit   Rate
 	depth   float64
@@ -155,16 +156,10 @@ func (t *tokenBucket) runGC() {
 		return
 	}
 
-	t.mu.Lock()
-	defer t.mu.Unlock()
-
-	// Don't run GC if thresholds haven't been crossed.
-	// Check again in case another GC thread has run while this GC thread
-	// was waiting to acquire the lock and the other GC thread reset the
-	// metrics.
-	if t.gc.metrics.numCalls.Load() < t.gc.thresholds.NumCalls {
+	if !t.mu.TryLock() {
 		return
 	}
+	defer t.mu.Unlock()
 
 	gcStartTime := time.Now()
 

From e10a73d0ca4739dc7a850e20bb44228f0594bb8a Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 08:44:21 +0530
Subject: [PATCH 43/53] Perform actual GC in own goroutine

---
 .../rate_limit/algorithm/token_bucket.go      | 56 ++++++++++---------
 1 file changed, 29 insertions(+), 27 deletions(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index f7b77d9d9205..cdce87666f4e 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -159,39 +159,41 @@ func (t *tokenBucket) runGC() {
 	if !t.mu.TryLock() {
 		return
 	}
-	defer t.mu.Unlock()
 
-	gcStartTime := time.Now()
+	go func() {
+		defer t.mu.Unlock()
+		gcStartTime := time.Now()
 
-	// Add tokens to all buckets according to the rate limit
-	// and flag full buckets for deletion.
-	toDelete := make([]uint64, 0)
-	numBucketsBefore := 0
-	t.buckets.Range(func(k, v interface{}) bool {
-		key := k.(uint64)
-		b := v.(*bucket)
+		// Add tokens to all buckets according to the rate limit
+		// and flag full buckets for deletion.
+		toDelete := make([]uint64, 0)
+		numBucketsBefore := 0
+		t.buckets.Range(func(k, v interface{}) bool {
+			key := k.(uint64)
+			b := v.(*bucket)
 
-		b.replenish(t.limit, t.clock)
+			b.replenish(t.limit, t.clock)
 
-		if b.tokens >= t.depth {
-			toDelete = append(toDelete, key)
-		}
+			if b.tokens >= t.depth {
+				toDelete = append(toDelete, key)
+			}
 
-		numBucketsBefore++
-		return true
-	})
+			numBucketsBefore++
+			return true
+		})
 
-	// Cleanup full buckets to free up memory
-	for _, key := range toDelete {
-		t.buckets.Delete(key)
-	}
+		// Cleanup full buckets to free up memory
+		for _, key := range toDelete {
+			t.buckets.Delete(key)
+		}
 
-	// Reset GC metrics
-	t.gc.metrics.numCalls = atomic.MakeUint(0)
+		// Reset GC metrics
+		t.gc.metrics.numCalls = atomic.MakeUint(0)
 
-	gcDuration := time.Now().Sub(gcStartTime)
-	numBucketsDeleted := len(toDelete)
-	numBucketsAfter := numBucketsBefore - numBucketsDeleted
-	t.logger.Debugf("gc duration: %v, buckets: (before: %v, deleted: %v, after: %v)",
-		gcDuration, numBucketsBefore, numBucketsDeleted, numBucketsAfter)
+		gcDuration := time.Now().Sub(gcStartTime)
+		numBucketsDeleted := len(toDelete)
+		numBucketsAfter := numBucketsBefore - numBucketsDeleted
+		t.logger.Debugf("gc duration: %v, buckets: (before: %v, deleted: %v, after: %v)",
+			gcDuration, numBucketsBefore, numBucketsDeleted, numBucketsAfter)
+	}()
 }

From 91493ba422226308dd2057da0f2da076e1433e25 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 11:30:23 +0530
Subject: [PATCH 44/53] Running mage fmt

---
 libbeat/processors/rate_limit/algorithm/token_bucket.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index cdce87666f4e..22f85ad0e80a 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -21,10 +21,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/elastic/go-concert/unison"
 	"github.com/jonboulle/clockwork"
 	"github.com/pkg/errors"
 
+	"github.com/elastic/go-concert/unison"
+
 	"github.com/elastic/beats/v7/libbeat/common/atomic"
 	"github.com/elastic/beats/v7/libbeat/logp"
 )

From c957161e8ce20561d76e4de351724ee396ed2653 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 13:11:05 +0530
Subject: [PATCH 45/53] Fixing processor name

---
 libbeat/processors/rate_limit/rate_limit.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index 836893fbe45a..a928054c19d5 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -35,7 +35,7 @@ import (
 
 func init() {
 	processors.RegisterPlugin("rate_limit", New)
-	jsprocessor.RegisterPlugin("Fingerprint", New)
+	jsprocessor.RegisterPlugin("RateLimit", New)
 }
 
 const processorName = "rate_limit"

From 3f4b8003e3c676d1283018cca898677ac0e1c8e8 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 13:11:10 +0530
Subject: [PATCH 46/53] Importing rate limit processor

---
 libbeat/cmd/instance/imports_common.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libbeat/cmd/instance/imports_common.go b/libbeat/cmd/instance/imports_common.go
index a2b2569d61c5..e1b97b259a5f 100644
--- a/libbeat/cmd/instance/imports_common.go
+++ b/libbeat/cmd/instance/imports_common.go
@@ -34,6 +34,7 @@ import (
 	_ "github.com/elastic/beats/v7/libbeat/processors/dns"
 	_ "github.com/elastic/beats/v7/libbeat/processors/extract_array"
 	_ "github.com/elastic/beats/v7/libbeat/processors/fingerprint"
+	_ "github.com/elastic/beats/v7/libbeat/processors/rate_limit"
 	_ "github.com/elastic/beats/v7/libbeat/processors/registered_domain"
 	_ "github.com/elastic/beats/v7/libbeat/processors/translate_sid"
 	_ "github.com/elastic/beats/v7/libbeat/processors/urldecode"

From 84c4f74ef13101a277a5ee341cb34e250a9e4eef Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 14:47:20 +0530
Subject: [PATCH 47/53] Initialize mutex

---
 libbeat/processors/rate_limit/algorithm/token_bucket.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/rate_limit/algorithm/token_bucket.go
index 22f85ad0e80a..715a4d0bedc7 100644
--- a/libbeat/processors/rate_limit/algorithm/token_bucket.go
+++ b/libbeat/processors/rate_limit/algorithm/token_bucket.go
@@ -102,6 +102,7 @@ func newTokenBucket(config Config) (Algorithm, error) {
 		},
 		clock:  clockwork.NewRealClock(),
 		logger: logp.NewLogger("token_bucket"),
+		mu:     unison.MakeMutex(),
 	}, nil
 }
 

From ac6343b2aed7d7977015e36d0dc8a4ad02630c40 Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 17:45:00 +0530
Subject: [PATCH 48/53] Do not register as a JS processor

---
 libbeat/processors/rate_limit/rate_limit.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index a928054c19d5..8ef839a5c445 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -30,12 +30,10 @@ import (
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/processors"
 	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/algorithm"
-	jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor"
 )
 
 func init() {
 	processors.RegisterPlugin("rate_limit", New)
-	jsprocessor.RegisterPlugin("RateLimit", New)
 }
 
 const processorName = "rate_limit"

From 00251a8abc3a0e37eeb2b52eb7ec84210e8c203e Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 17:45:49 +0530
Subject: [PATCH 49/53] Remove unused field

---
 libbeat/processors/rate_limit/rate_limit.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/rate_limit/rate_limit.go
index 8ef839a5c445..d2de71309700 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/rate_limit/rate_limit.go
@@ -42,9 +42,6 @@ type rateLimit struct {
 	config    Config
 	algorithm algorithm.Algorithm
 	logger    *logp.Logger
-
-	// For testing
-	c clockwork.Clock
 }
 
 // New constructs a new rate limit processor.
@@ -126,6 +123,5 @@ func (p *rateLimit) makeKey(event *beat.Event) (uint64, error) {
 
 // setClock allows test code to inject a fake clock
 func (p *rateLimit) setClock(c clockwork.Clock) {
-	p.c = c
 	p.algorithm.SetClock(c)
 }

From a3ce476d3e75c14c76ac5457fee1196360d9bbcd Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 17:46:31 +0530
Subject: [PATCH 50/53] Mark processor as beta

---
 libbeat/processors/rate_limit/docs/rate_limit.asciidoc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libbeat/processors/rate_limit/docs/rate_limit.asciidoc b/libbeat/processors/rate_limit/docs/rate_limit.asciidoc
index 5a63a6d54d0a..cdc33d14ab6e 100644
--- a/libbeat/processors/rate_limit/docs/rate_limit.asciidoc
+++ b/libbeat/processors/rate_limit/docs/rate_limit.asciidoc
@@ -1,5 +1,6 @@
 [[rate_limit]]
 === Rate limit the flow of events
+beta[]
 
 ++++
 <titleabbrev>rate_limit</titleabbrev>

From 51257481d41162648f50536d76ff430f0212b5be Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 17:49:38 +0530
Subject: [PATCH 51/53] Renaming package

---
 libbeat/cmd/instance/imports_common.go                      | 2 +-
 .../{rate_limit => ratelimit}/algorithm/algorithm.go        | 0
 .../processors/{rate_limit => ratelimit}/algorithm/limit.go | 0
 .../{rate_limit => ratelimit}/algorithm/token_bucket.go     | 0
 libbeat/processors/{rate_limit => ratelimit}/config.go      | 4 ++--
 .../{rate_limit => ratelimit}/docs/rate_limit.asciidoc      | 0
 libbeat/processors/{rate_limit => ratelimit}/rate_limit.go  | 6 +++---
 .../processors/{rate_limit => ratelimit}/rate_limit_test.go | 2 +-
 8 files changed, 7 insertions(+), 7 deletions(-)
 rename libbeat/processors/{rate_limit => ratelimit}/algorithm/algorithm.go (100%)
 rename libbeat/processors/{rate_limit => ratelimit}/algorithm/limit.go (100%)
 rename libbeat/processors/{rate_limit => ratelimit}/algorithm/token_bucket.go (100%)
 rename libbeat/processors/{rate_limit => ratelimit}/config.go (94%)
 rename libbeat/processors/{rate_limit => ratelimit}/docs/rate_limit.asciidoc (100%)
 rename libbeat/processors/{rate_limit => ratelimit}/rate_limit.go (96%)
 rename libbeat/processors/{rate_limit => ratelimit}/rate_limit_test.go (99%)

diff --git a/libbeat/cmd/instance/imports_common.go b/libbeat/cmd/instance/imports_common.go
index e1b97b259a5f..e47dbf937993 100644
--- a/libbeat/cmd/instance/imports_common.go
+++ b/libbeat/cmd/instance/imports_common.go
@@ -34,7 +34,7 @@ import (
 	_ "github.com/elastic/beats/v7/libbeat/processors/dns"
 	_ "github.com/elastic/beats/v7/libbeat/processors/extract_array"
 	_ "github.com/elastic/beats/v7/libbeat/processors/fingerprint"
-	_ "github.com/elastic/beats/v7/libbeat/processors/rate_limit"
+	_ "github.com/elastic/beats/v7/libbeat/processors/ratelimit"
 	_ "github.com/elastic/beats/v7/libbeat/processors/registered_domain"
 	_ "github.com/elastic/beats/v7/libbeat/processors/translate_sid"
 	_ "github.com/elastic/beats/v7/libbeat/processors/urldecode"
diff --git a/libbeat/processors/rate_limit/algorithm/algorithm.go b/libbeat/processors/ratelimit/algorithm/algorithm.go
similarity index 100%
rename from libbeat/processors/rate_limit/algorithm/algorithm.go
rename to libbeat/processors/ratelimit/algorithm/algorithm.go
diff --git a/libbeat/processors/rate_limit/algorithm/limit.go b/libbeat/processors/ratelimit/algorithm/limit.go
similarity index 100%
rename from libbeat/processors/rate_limit/algorithm/limit.go
rename to libbeat/processors/ratelimit/algorithm/limit.go
diff --git a/libbeat/processors/rate_limit/algorithm/token_bucket.go b/libbeat/processors/ratelimit/algorithm/token_bucket.go
similarity index 100%
rename from libbeat/processors/rate_limit/algorithm/token_bucket.go
rename to libbeat/processors/ratelimit/algorithm/token_bucket.go
diff --git a/libbeat/processors/rate_limit/config.go b/libbeat/processors/ratelimit/config.go
similarity index 94%
rename from libbeat/processors/rate_limit/config.go
rename to libbeat/processors/ratelimit/config.go
index 6a1231a23732..4fe78db0aab7 100644
--- a/libbeat/processors/rate_limit/config.go
+++ b/libbeat/processors/ratelimit/config.go
@@ -15,13 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package rate_limit
+package ratelimit
 
 import (
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/algorithm"
+	"github.com/elastic/beats/v7/libbeat/processors/ratelimit/algorithm"
 )
 
 // Config for rate limit processor.
diff --git a/libbeat/processors/rate_limit/docs/rate_limit.asciidoc b/libbeat/processors/ratelimit/docs/rate_limit.asciidoc
similarity index 100%
rename from libbeat/processors/rate_limit/docs/rate_limit.asciidoc
rename to libbeat/processors/ratelimit/docs/rate_limit.asciidoc
diff --git a/libbeat/processors/rate_limit/rate_limit.go b/libbeat/processors/ratelimit/rate_limit.go
similarity index 96%
rename from libbeat/processors/rate_limit/rate_limit.go
rename to libbeat/processors/ratelimit/rate_limit.go
index d2de71309700..29fa69543960 100644
--- a/libbeat/processors/rate_limit/rate_limit.go
+++ b/libbeat/processors/ratelimit/rate_limit.go
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package rate_limit
+package ratelimit
 
 import (
 	"fmt"
@@ -29,11 +29,11 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/processors"
-	"github.com/elastic/beats/v7/libbeat/processors/rate_limit/algorithm"
+	"github.com/elastic/beats/v7/libbeat/processors/ratelimit/algorithm"
 )
 
 func init() {
-	processors.RegisterPlugin("rate_limit", New)
+	processors.RegisterPlugin(processorName, New)
 }
 
 const processorName = "rate_limit"
diff --git a/libbeat/processors/rate_limit/rate_limit_test.go b/libbeat/processors/ratelimit/rate_limit_test.go
similarity index 99%
rename from libbeat/processors/rate_limit/rate_limit_test.go
rename to libbeat/processors/ratelimit/rate_limit_test.go
index 40bd971bdd5a..d8a70297951e 100644
--- a/libbeat/processors/rate_limit/rate_limit_test.go
+++ b/libbeat/processors/ratelimit/rate_limit_test.go
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package rate_limit
+package ratelimit
 
 import (
 	"testing"

From 0ffcd7df6568a2c0c94ca77ab9b8af9f073bc14d Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 18:02:29 +0530
Subject: [PATCH 52/53] Flatenning package hierarchy

---
 .../ratelimit/{algorithm => }/algorithm.go    | 24 +++++++++----------
 libbeat/processors/ratelimit/config.go        |  9 ++++---
 .../ratelimit/{algorithm => }/limit.go        | 10 ++++----
 libbeat/processors/ratelimit/rate_limit.go    | 23 +++++++++---------
 .../processors/ratelimit/rate_limit_test.go   |  4 ++--
 .../ratelimit/{algorithm => }/token_bucket.go | 14 +++++------
 6 files changed, 41 insertions(+), 43 deletions(-)
 rename libbeat/processors/ratelimit/{algorithm => }/algorithm.go (77%)
 rename libbeat/processors/ratelimit/{algorithm => }/limit.go (92%)
 rename libbeat/processors/ratelimit/{algorithm => }/token_bucket.go (94%)

diff --git a/libbeat/processors/ratelimit/algorithm/algorithm.go b/libbeat/processors/ratelimit/algorithm.go
similarity index 77%
rename from libbeat/processors/ratelimit/algorithm/algorithm.go
rename to libbeat/processors/ratelimit/algorithm.go
index cbb995a8187e..41431d5c4733 100644
--- a/libbeat/processors/ratelimit/algorithm/algorithm.go
+++ b/libbeat/processors/ratelimit/algorithm.go
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package algorithm
+package ratelimit
 
 import (
 	"fmt"
@@ -28,18 +28,18 @@ import (
 
 var registry = make(map[string]constructor, 0)
 
-// Config for rate limit algorithm.
-type Config struct {
-	// Limit is the rate limit to be enforced by the algorithm.
-	Limit Rate
+// algoConfig for rate limit algorithm.
+type algoConfig struct {
+	// limit is the rate limit to be enforced by the algorithm.
+	limit rate
 
-	// Config is any algorithm-specific additional configuration.
-	Config common.Config
+	// config is any algorithm-specific additional configuration.
+	config common.Config
 }
 
-// Algorithm is the interface that all rate limiting algorithms must
+// algorithm is the interface that all rate limiting algorithms must
 // conform to.
-type Algorithm interface {
+type algorithm interface {
 	// IsAllowed accepts a key and returns whether that key is allowed
 	// (true) or not (false). If a key is allowed, it means it is NOT
 	// rate limited. If a key is not allowed, it means it is being rate
@@ -50,15 +50,15 @@ type Algorithm interface {
 	SetClock(clockwork.Clock)
 }
 
-type constructor func(Config) (Algorithm, error)
+type constructor func(algoConfig) (algorithm, error)
 
 func register(id string, ctor constructor) {
 	registry[id] = ctor
 }
 
-// Factory returns the requested rate limiting algorithm, if one is found. If not found,
+// factory returns the requested rate limiting algorithm, if one is found. If not found,
 // an error is returned.
-func Factory(id string, config Config) (Algorithm, error) {
+func factory(id string, config algoConfig) (algorithm, error) {
 	var ctor constructor
 	var found bool
 	if ctor, found = registry[id]; !found {
diff --git a/libbeat/processors/ratelimit/config.go b/libbeat/processors/ratelimit/config.go
index 4fe78db0aab7..79fb5a499325 100644
--- a/libbeat/processors/ratelimit/config.go
+++ b/libbeat/processors/ratelimit/config.go
@@ -21,17 +21,16 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/processors/ratelimit/algorithm"
 )
 
-// Config for rate limit processor.
-type Config struct {
-	Limit     algorithm.Rate         `config:"limit" validate:"required"`
+// config for rate limit processor.
+type config struct {
+	Limit     rate                   `config:"limit" validate:"required"`
 	Fields    []string               `config:"fields"`
 	Algorithm common.ConfigNamespace `config:"algorithm"`
 }
 
-func (c *Config) SetDefaults() error {
+func (c *config) setDefaults() error {
 	if c.Algorithm.Name() == "" {
 		cfg, err := common.NewConfigFrom(map[string]interface{}{
 			"token_bucket": map[string]interface{}{},
diff --git a/libbeat/processors/ratelimit/algorithm/limit.go b/libbeat/processors/ratelimit/limit.go
similarity index 92%
rename from libbeat/processors/ratelimit/algorithm/limit.go
rename to libbeat/processors/ratelimit/limit.go
index d93eb12b1065..8b1f1e0517c5 100644
--- a/libbeat/processors/ratelimit/algorithm/limit.go
+++ b/libbeat/processors/ratelimit/limit.go
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package algorithm
+package ratelimit
 
 import (
 	"fmt"
@@ -31,13 +31,13 @@ const (
 	unitPerHour   unit = "h"
 )
 
-type Rate struct {
+type rate struct {
 	value float64
 	unit  unit
 }
 
-// Unpack creates a Rate from the given string
-func (l *Rate) Unpack(str string) error {
+// Unpack creates a rate from the given string
+func (l *rate) Unpack(str string) error {
 	parts := strings.Split(str, "/")
 	if len(parts) != 2 {
 		return fmt.Errorf(`rate in invalid format: %v. Must be specified as "number/unit"`, str)
@@ -68,7 +68,7 @@ func (l *Rate) Unpack(str string) error {
 	return nil
 }
 
-func (l *Rate) valuePerSecond() float64 {
+func (l *rate) valuePerSecond() float64 {
 	switch l.unit {
 	case unitPerSecond:
 		return l.value
diff --git a/libbeat/processors/ratelimit/rate_limit.go b/libbeat/processors/ratelimit/rate_limit.go
index 29fa69543960..d6c16c98633e 100644
--- a/libbeat/processors/ratelimit/rate_limit.go
+++ b/libbeat/processors/ratelimit/rate_limit.go
@@ -29,37 +29,36 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/processors"
-	"github.com/elastic/beats/v7/libbeat/processors/ratelimit/algorithm"
 )
 
 func init() {
-	processors.RegisterPlugin(processorName, New)
+	processors.RegisterPlugin(processorName, new)
 }
 
 const processorName = "rate_limit"
 
 type rateLimit struct {
-	config    Config
-	algorithm algorithm.Algorithm
+	config    config
+	algorithm algorithm
 	logger    *logp.Logger
 }
 
-// New constructs a new rate limit processor.
-func New(cfg *common.Config) (processors.Processor, error) {
-	var config Config
+// new constructs a new rate limit processor.
+func new(cfg *common.Config) (processors.Processor, error) {
+	var config config
 	if err := cfg.Unpack(&config); err != nil {
 		return nil, errors.Wrap(err, "could not unpack processor configuration")
 	}
 
-	if err := config.SetDefaults(); err != nil {
+	if err := config.setDefaults(); err != nil {
 		return nil, errors.Wrap(err, "could not set default configuration")
 	}
 
-	algoConfig := algorithm.Config{
-		Limit:  config.Limit,
-		Config: *config.Algorithm.Config(),
+	algoConfig := algoConfig{
+		limit:  config.Limit,
+		config: *config.Algorithm.Config(),
 	}
-	algo, err := algorithm.Factory(config.Algorithm.Name(), algoConfig)
+	algo, err := factory(config.Algorithm.Name(), algoConfig)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not construct rate limiting algorithm")
 	}
diff --git a/libbeat/processors/ratelimit/rate_limit_test.go b/libbeat/processors/ratelimit/rate_limit_test.go
index d8a70297951e..99941ebfd0f7 100644
--- a/libbeat/processors/ratelimit/rate_limit_test.go
+++ b/libbeat/processors/ratelimit/rate_limit_test.go
@@ -50,7 +50,7 @@ func TestNew(t *testing.T) {
 	for name, test := range cases {
 		t.Run(name, func(t *testing.T) {
 			config := common.MustNewConfigFrom(test.config)
-			_, err := New(config)
+			_, err := new(config)
 			if test.err == "" {
 				require.NoError(t, err)
 			} else {
@@ -151,7 +151,7 @@ func TestRateLimit(t *testing.T) {
 
 	for name, test := range cases {
 		t.Run(name, func(t *testing.T) {
-			p, err := New(common.MustNewConfigFrom(test.config))
+			p, err := new(common.MustNewConfigFrom(test.config))
 			require.NoError(t, err)
 
 			fakeClock := clockwork.NewFakeClock()
diff --git a/libbeat/processors/ratelimit/algorithm/token_bucket.go b/libbeat/processors/ratelimit/token_bucket.go
similarity index 94%
rename from libbeat/processors/ratelimit/algorithm/token_bucket.go
rename to libbeat/processors/ratelimit/token_bucket.go
index 715a4d0bedc7..49e598bcd533 100644
--- a/libbeat/processors/ratelimit/algorithm/token_bucket.go
+++ b/libbeat/processors/ratelimit/token_bucket.go
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package algorithm
+package ratelimit
 
 import (
 	"sync"
@@ -42,7 +42,7 @@ type bucket struct {
 type tokenBucket struct {
 	mu unison.Mutex
 
-	limit   Rate
+	limit   rate
 	depth   float64
 	buckets sync.Map
 
@@ -74,7 +74,7 @@ type tokenBucketConfig struct {
 	GC tokenBucketGCConfig `config:"gc"`
 }
 
-func newTokenBucket(config Config) (Algorithm, error) {
+func newTokenBucket(config algoConfig) (algorithm, error) {
 	cfg := tokenBucketConfig{
 		BurstMultiplier: 1.0,
 		GC: tokenBucketGCConfig{
@@ -82,13 +82,13 @@ func newTokenBucket(config Config) (Algorithm, error) {
 		},
 	}
 
-	if err := config.Config.Unpack(&cfg); err != nil {
+	if err := config.config.Unpack(&cfg); err != nil {
 		return nil, errors.Wrap(err, "could not unpack token_bucket algorithm configuration")
 	}
 
 	return &tokenBucket{
-		limit:   config.Limit,
-		depth:   config.Limit.value * cfg.BurstMultiplier,
+		limit:   config.limit,
+		depth:   config.limit.value * cfg.BurstMultiplier,
 		buckets: sync.Map{},
 		gc: struct {
 			thresholds tokenBucketGCConfig
@@ -144,7 +144,7 @@ func (b *bucket) withdraw() bool {
 	return true
 }
 
-func (b *bucket) replenish(rate Rate, clock clockwork.Clock) {
+func (b *bucket) replenish(rate rate, clock clockwork.Clock) {
 	secsSinceLastReplenish := clock.Now().Sub(b.lastReplenish).Seconds()
 	tokensToReplenish := secsSinceLastReplenish * rate.valuePerSecond()
 

From 2b92955f7385db4f41ef944605cc674bd6828ecc Mon Sep 17 00:00:00 2001
From: Shaunak Kashyap <ycombinator@gmail.com>
Date: Thu, 10 Dec 2020 18:30:31 +0530
Subject: [PATCH 53/53] Remove SetClock from algorithm interface

---
 libbeat/processors/ratelimit/algorithm.go    | 4 ----
 libbeat/processors/ratelimit/rate_limit.go   | 5 ++++-
 libbeat/processors/ratelimit/token_bucket.go | 4 ++--
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/libbeat/processors/ratelimit/algorithm.go b/libbeat/processors/ratelimit/algorithm.go
index 41431d5c4733..2d937d269315 100644
--- a/libbeat/processors/ratelimit/algorithm.go
+++ b/libbeat/processors/ratelimit/algorithm.go
@@ -20,7 +20,6 @@ package ratelimit
 import (
 	"fmt"
 
-	"github.com/jonboulle/clockwork"
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
@@ -45,9 +44,6 @@ type algorithm interface {
 	// rate limited. If a key is not allowed, it means it is being rate
 	// limited.
 	IsAllowed(uint64) bool
-
-	// SetClock allows test code to inject a fake clock.
-	SetClock(clockwork.Clock)
 }
 
 type constructor func(algoConfig) (algorithm, error)
diff --git a/libbeat/processors/ratelimit/rate_limit.go b/libbeat/processors/ratelimit/rate_limit.go
index d6c16c98633e..210ac5a5912e 100644
--- a/libbeat/processors/ratelimit/rate_limit.go
+++ b/libbeat/processors/ratelimit/rate_limit.go
@@ -121,6 +121,9 @@ func (p *rateLimit) makeKey(event *beat.Event) (uint64, error) {
 }
 
 // setClock allows test code to inject a fake clock
+// TODO: remove this method and move tests that use it to algorithm level.
 func (p *rateLimit) setClock(c clockwork.Clock) {
-	p.algorithm.SetClock(c)
+	if a, ok := p.algorithm.(interface{ setClock(clock clockwork.Clock) }); ok {
+		a.setClock(c)
+	}
 }
diff --git a/libbeat/processors/ratelimit/token_bucket.go b/libbeat/processors/ratelimit/token_bucket.go
index 49e598bcd533..d7460a37b068 100644
--- a/libbeat/processors/ratelimit/token_bucket.go
+++ b/libbeat/processors/ratelimit/token_bucket.go
@@ -116,8 +116,8 @@ func (t *tokenBucket) IsAllowed(key uint64) bool {
 	return allowed
 }
 
-// SetClock allows test code to inject a fake clock
-func (t *tokenBucket) SetClock(c clockwork.Clock) {
+// setClock allows test code to inject a fake clock
+func (t *tokenBucket) setClock(c clockwork.Clock) {
 	t.clock = c
 }