diff --git a/holonote/annotate/display.py b/holonote/annotate/display.py
index e941e00f..6e9fe395 100644
--- a/holonote/annotate/display.py
+++ b/holonote/annotate/display.py
@@ -549,6 +549,9 @@ def static_indicators(self, **events):
         highlighters = {opt: self._selected_dim_expr(v[0], v[1]) for opt, v in highlight.items()}
         indicator = indicator.opts(*self.style.indicator(**highlighters))
 
+        if len(indicator.data) == 0:
+            return hv.NdOverlay({0: self._make_empty_element()})
+
         return indicator.overlay() if self.annotator.groupby else hv.NdOverlay({0: indicator})
 
     def _selected_dim_expr(self, selected_value, non_selected_value) -> hv.dim:
diff --git a/holonote/tests/test_annotators_element.py b/holonote/tests/test_annotators_element.py
index e30f0225..32fd2443 100644
--- a/holonote/tests/test_annotators_element.py
+++ b/holonote/tests/test_annotators_element.py
@@ -251,6 +251,25 @@ def test_groupby_visible(cat_annotator):
         next(iter_indicator)
 
 
+def test_groupby_with_overlay_from_empty_annotator(annotator_range2d, capsys):
+    # Test for https://github.com/holoviz/holonote/issues/119
+    annotator = annotator_range2d
+    annotator.groupby = "description"
+    bounds = (-1, -1, 1, 1)
+    data = np.array([[0, 1], [1, 0]])
+    img = hv.Image(data, kdims=["x", "y"], bounds=bounds)
+
+    plot = annotator * img
+    hv.render(plot)
+
+    annotator.set_regions(x=(-0.15, 0.15), y=(-0.25, 0.25))
+    annotator.add_annotation(description="Test")
+
+    captured = capsys.readouterr()
+    bad_output = "AssertionError: DynamicMap must only contain one type of object, not both Overlay and NdOverlay."
+    assert bad_output not in captured.out
+
+
 def test_multiply_overlay(annotator_range1d):
     el1 = hv.Curve([], kdims=["TIME"])
     el2 = hv.Curve([], kdims=["TIME"])