fix(enhance_k8s_metadata): remove nonexistent pods on cache refresh

fluent-plugin-enhance-k8s-metadata/lib/fluent/plugin/filter_enhance_k8s_metadata.rb

@@ -137,21 +137,7 @@ def start_cache_timer
         cache_refresh_with_variation = apply_variation(@cache_refresh, @cache_refresh_variation)
         log.info "Will refresh cache every #{format_time(cache_refresh_with_variation)}"
         timer_execute(:"cache_refresher", cache_refresh_with_variation) {
-          entries = @cache.to_a
-          log.info "Refreshing metadata for #{entries.count} entries"
-
-          entries.each { |entry|
-            begin
-              log.debug "Refreshing metadata for key #{entry[0]}"
-              split = entry[0].split("::")
-              namespace_name = split[0]
-              pod_name = split[1]
-              metadata = fetch_pod_metadata(namespace_name, pod_name)
-              @cache[entry[0]] = metadata unless metadata.empty?
-            rescue => e
-              log.error "Cannot refresh metadata for entry #{entry}: #{e}"
-            end
-          }
+            refresh_cache
         }
       end
 

fluent-plugin-enhance-k8s-metadata/lib/sumologic/kubernetes/cache_strategy.rb

@@ -18,6 +18,33 @@ def get_pod_metadata(namespace_name, pod_name)
         end
         metadata
       end
+
+      def refresh_cache
+        # Refresh the cache by re-fetching all the pod metadata.
+        entries = @cache.to_a
+        log.info "Refreshing metadata for #{entries.count} entries"
+
+        entries.each { |key, _|
+          begin
+            refresh_cache_entry(key)
+          rescue => e
+            log.error "Cannot refresh metadata for key #{key}: #{e}"
+          end
+        }
+      end
+
+      def refresh_cache_entry(cache_key)
+        log.debug "Refreshing metadata for key #{cache_key}"
+        namespace_name, pod_name = cache_key.split("::")
+        metadata = fetch_pod_metadata(namespace_name, pod_name)
+        if metadata.empty?
+            # if the pod doesn't exist anymore, remove its key from the cache
+            @cache.delete(cache_key)
+        else
+            @cache[cache_key] = metadata
+        end
+      end
+
     end
   end
 end

fluent-plugin-enhance-k8s-metadata/test/sumologic/kubernetes/test_cache_strategy.rb

@@ -51,4 +51,14 @@ def log
     assert_not_nil metadata
     assert_equal 0, metadata.size
   end
+
+  test 'refreshing cache entry deletes it if no metadata' do
+    stub_request(:get, %r{/api/v1/namespaces/namespace/pods/non_existent})
+        .to_raise(Kubeclient::ResourceNotFoundError.new(404, nil, nil))
+    key = 'namespace::non_existent'
+    @cache[key] = {}
+    refresh_cache_entry(key)
+    assert_false @cache.key?(key)
+  end
+
 end