diff --git a/panel/template/base.py b/panel/template/base.py
index f95972ccff..1f365e66e2 100644
--- a/panel/template/base.py
+++ b/panel/template/base.py
@@ -85,6 +85,8 @@ def __repr__(self):
             cls=cls, objs=('%s' % spacer).join(objs), spacer=spacer)
 
     def _apply_modifiers(self, viewable, mref):
+        if mref not in viewable._models:
+            return
         model, _ = viewable._models[mref]
         modifiers = self._modifiers.get(type(viewable), {})
         viewable.param.set_param(**{k: v for k, v in modifiers.items()
@@ -102,7 +104,10 @@ def _init_doc(self, doc=None, comm=None, title=None, notebook=False, location=Tr
             mref = model.ref['id']
             doc.on_session_destroyed(obj._server_destroy)
             for sub in obj.select(Viewable):
-                sub._models[ref] = sub._models.get(mref)
+                submodel = sub._models.get(mref)
+                if submodel is None:
+                    continue
+                sub._models[ref] = submodel
                 if isinstance(sub, HoloViews) and mref in sub._plots:
                     sub._plots[ref] = sub._plots.get(mref)
             col.objects.append(obj)