From 356ff125a97a354ea540db3fadb25d9becbb724e Mon Sep 17 00:00:00 2001 From: Ryan Miller Date: Mon, 5 Aug 2019 11:49:36 -0700 Subject: [PATCH 1/3] Bump versions of plugins used in Dockerfile --- deploy/docker/Dockerfile | 15 +++++++-------- deploy/kubernetes/fluentd-sumologic.yaml.tmpl | 1 - .../lib/sumologic/kubernetes/reader.rb | 2 +- .../lib/fluent/plugin/in_events.rb | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 9e1539a7e7..e6f8f6db41 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM fluent/fluentd:v1.6.2-1.0 +FROM fluent/fluentd:v1.6.3-1.0 # Use root account to use apk USER root @@ -17,14 +17,13 @@ RUN apk add --no-cache --update --virtual .build-deps sudo build-base ruby-dev \ && gem install snappy # FluentD plugins from RubyGems -RUN gem install fluent-plugin-s3 -v 1.1.4 \ - && gem install fluent-plugin-systemd -v 0.3.1 \ - && gem install fluent-plugin-record-modifier \ - && gem install fluent-plugin-kubernetes_metadata_filter -v 1.0.2 \ +RUN gem install fluent-plugin-systemd -v 1.0.2 \ + && gem install fluent-plugin-record-modifier -v 2.0.1 \ + && gem install fluent-plugin-kubernetes_metadata_filter -v 2.2.0 \ && gem install fluent-plugin-sumologic_output -v 1.5.0 \ - && gem install fluent-plugin-concat -v 2.3.0 \ - && gem install fluent-plugin-rewrite-tag-filter -v 2.1.0 \ - && gem install fluent-plugin-prometheus -v 1.4.0 \ + && gem install fluent-plugin-concat -v 2.4.0 \ + && gem install fluent-plugin-rewrite-tag-filter -v 2.2.0 \ + && gem install fluent-plugin-prometheus -v 1.5.0 \ && gem install fluent-plugin-kubernetes_sumologic -v 2.4.2 # FluentD plugins from this repository diff --git a/deploy/kubernetes/fluentd-sumologic.yaml.tmpl b/deploy/kubernetes/fluentd-sumologic.yaml.tmpl index 163458b56d..03aed4f340 100644 --- a/deploy/kubernetes/fluentd-sumologic.yaml.tmpl +++ b/deploy/kubernetes/fluentd-sumologic.yaml.tmpl @@ -179,7 +179,6 @@ data: cache_size "#{ENV['K8S_METADATA_FILTER_BEARER_CACHE_SIZE']}" cache_ttl "#{ENV['K8S_METADATA_FILTER_BEARER_CACHE_TTL']}" tag_to_kubernetes_name_regexp '.+?\.containers\.(?[^_]+)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$' - merge_json_log false @type enhance_k8s_metadata diff --git a/fluent-plugin-enhance-k8s-metadata/lib/sumologic/kubernetes/reader.rb b/fluent-plugin-enhance-k8s-metadata/lib/sumologic/kubernetes/reader.rb index d01760f1fe..6a61b653aa 100644 --- a/fluent-plugin-enhance-k8s-metadata/lib/sumologic/kubernetes/reader.rb +++ b/fluent-plugin-enhance-k8s-metadata/lib/sumologic/kubernetes/reader.rb @@ -113,7 +113,7 @@ def fetch_resource(type, name, namespace, api_version = 'v1') nil end rescue Kubeclient::ResourceNotFoundError => e - log.error e + log.warn e nil end end diff --git a/fluent-plugin-events/lib/fluent/plugin/in_events.rb b/fluent-plugin-events/lib/fluent/plugin/in_events.rb index 47b09ebd91..cec6d651ac 100644 --- a/fluent-plugin-events/lib/fluent/plugin/in_events.rb +++ b/fluent-plugin-events/lib/fluent/plugin/in_events.rb @@ -111,7 +111,7 @@ def start_watcher_thread end if (!rv) - log.error "Resource version #{rv} expired, waiting for stream to be recreated with more recent version." + log.warn "Resource version #{rv} expired, waiting for stream to be recreated with more recent version." break end end From 3933edc2e301e503fc914f69b048f2c0e33ac50c Mon Sep 17 00:00:00 2001 From: Ryan Miller Date: Mon, 5 Aug 2019 13:29:23 -0700 Subject: [PATCH 2/3] Add fluentd plugins to allow customers to forward data if needed --- deploy/docker/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index e6f8f6db41..b380b866f9 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -16,6 +16,11 @@ RUN apk add --no-cache --update --virtual .build-deps sudo build-base ruby-dev \ && gem install lru_redux \ && gem install snappy +# FluentD plugins to allow customers to forward data if needed to various cloud platforms +RUN gem install fluent-plugin-s3 \ + && gem install fluent-plugin-google-cloud \ + && gem install fluent-plugin-azure-storage-append-blob + # FluentD plugins from RubyGems RUN gem install fluent-plugin-systemd -v 1.0.2 \ && gem install fluent-plugin-record-modifier -v 2.0.1 \ From 548968e67dfe57ad86c9e74a50152c0be800ab10 Mon Sep 17 00:00:00 2001 From: Ryan Miller Date: Mon, 5 Aug 2019 15:00:32 -0700 Subject: [PATCH 3/3] Retain s3 fluentd plugin for now; Add TODO to add more later --- deploy/docker/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index b380b866f9..8d56d05076 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -16,10 +16,11 @@ RUN apk add --no-cache --update --virtual .build-deps sudo build-base ruby-dev \ && gem install lru_redux \ && gem install snappy -# FluentD plugins to allow customers to forward data if needed to various cloud platforms -RUN gem install fluent-plugin-s3 \ - && gem install fluent-plugin-google-cloud \ - && gem install fluent-plugin-azure-storage-append-blob +# FluentD plugins to allow customers to forward data if needed to various cloud providers +RUN gem install fluent-plugin-s3 + # TODO: Support additional cloud providers + # && gem install fluent-plugin-google-cloud \ + # && gem install fluent-plugin-azure-storage-append-blob # FluentD plugins from RubyGems RUN gem install fluent-plugin-systemd -v 1.0.2 \