diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/StringSetData.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/StringSetData.java
index 93dfb8e3ebc8..466d4ad46eb6 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/StringSetData.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/StringSetData.java
@@ -19,7 +19,6 @@
 
 import com.google.auto.value.AutoValue;
 import java.io.Serializable;
-import java.util.HashSet;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
@@ -50,12 +49,16 @@ public static StringSetData empty() {
    * Combines this {@link StringSetData} with other, both original StringSetData are left intact.
    */
   public StringSetData combine(StringSetData other) {
-    // do not merge other on this as this StringSetData might hold an immutable set like in case
-    // of  EmptyStringSetData
-    Set<String> combined = new HashSet<>();
-    combined.addAll(this.stringSet());
-    combined.addAll(other.stringSet());
-    return StringSetData.create(combined);
+    if (this.stringSet().isEmpty()) {
+      return other;
+    } else if (other.stringSet().isEmpty()) {
+      return this;
+    } else {
+      ImmutableSet.Builder<String> combined = ImmutableSet.builder();
+      combined.addAll(this.stringSet());
+      combined.addAll(other.stringSet());
+      return StringSetData.create(combined.build());
+    }
   }
 
   /**