fix(enhance_k8s_metadata): remove nonexistent pods on cache refresh

There's no reason to keep keys for pods which don't exist anymore and unnecessarily spam the API Server.

Move the cache refresh logic to the CacheStrategy, which is the more
natural place for it, and makes it easier to test.

fluent-plugin-enhance-k8s-metadata/lib/fluent/plugin/filter_enhance_k8s_metadata.rb

@@ -137,21 +137,7 @@ def start_cache_timer
         cache_refresh_with_variation = apply_variation(@cache_refresh, @cache_refresh_variation)
         log.info "Will refresh cache every #{format_time(cache_refresh_with_variation)}"
         timer_execute(:"cache_refresher", cache_refresh_with_variation) {
-          entries = @cache.to_a
-          log.info "Refreshing metadata for #{entries.count} entries"
-
-          entries.each { |entry|
-            begin
-              log.debug "Refreshing metadata for key #{entry[0]}"
-              split = entry[0].split("::")
-              namespace_name = split[0]
-              pod_name = split[1]
-              metadata = fetch_pod_metadata(namespace_name, pod_name)
-              @cache[entry[0]] = metadata unless metadata.empty?
-            rescue => e
-              log.error "Cannot refresh metadata for entry #{entry}: #{e}"
-            end
-          }
+            refresh_cache
         }
       end
 

fluent-plugin-enhance-k8s-metadata/lib/sumologic/kubernetes/cache_strategy.rb

@@ -18,6 +18,33 @@ def get_pod_metadata(namespace_name, pod_name)
         end
         metadata
       end
+
+      def refresh_cache
+        # Refresh the cache by re-fetching all the pod metadata.
+        entries = @cache.to_a
+        log.info "Refreshing metadata for #{entries.count} entries"
+
+        entries.each { |key, _|
+          begin
+            refresh_cache_entry(key)
+          rescue => e
+            log.error "Cannot refresh metadata for key #{key}: #{e}"
+          end
+        }
+      end
+
+      def refresh_cache_entry(cache_key)
+        log.debug "Refreshing metadata for key #{cache_key}"
+        namespace_name, pod_name = cache_key.split("::")
+        metadata = fetch_pod_metadata(namespace_name, pod_name)
+        if metadata.empty?
+            # if the pod doesn't exist anymore, remove its key from the cache
+            @cache.delete(cache_key)
+        else
+            @cache[cache_key] = metadata
+        end
+      end
+
     end
   end
 end

fluent-plugin-enhance-k8s-metadata/test/sumologic/kubernetes/test_cache_strategy.rb

@@ -51,4 +51,14 @@ def log
     assert_not_nil metadata
     assert_equal 0, metadata.size
   end
+
+  test 'refreshing cache entry deletes it if no metadata' do
+    stub_request(:get, %r{/api/v1/namespaces/namespace/pods/non_existent})
+        .to_raise(Kubeclient::ResourceNotFoundError.new(404, nil, nil))
+    key = 'namespace::non_existent'
+    @cache[key] = {}
+    refresh_cache_entry(key)
+    assert_false @cache.key?(key)
+  end
+
 end