--- a/layout/generic/nsSubDocumentFrame.cpp
+++ b/layout/generic/nsSubDocumentFrame.cpp
@@ -442,36 +442,31 @@ nsSubDocumentFrame::BuildDisplayList(nsD
       haveDisplayPort ||
       presContext->IsRootContentDocument() ||
       (sf && sf->IsScrollingActive(aBuilder)))
   {
     needsOwnLayer = true;
   }
 
   if (aBuilder->IsRetainingDisplayList()) {
-    // The value of needsOwnLayer can change between builds without
-    // an invalidation recorded for this frame (like if the root
-    // scrollframe becomes active). If this happens,
-    // then we need to notify the builder so that merging can
-    // happen correctly.
-    if (!mPreviouslyNeededLayer ||
-        mPreviouslyNeededLayer.value() != needsOwnLayer) {
-      dirty = visible;
-      aBuilder->MarkCurrentFrameModifiedDuringBuilding();
-    }
-    mPreviouslyNeededLayer = Some(needsOwnLayer);
-
     // Caret frame changed, rebuild the entire subdoc.
     // We could just invalidate the old and new frame
     // areas and save some work here. RetainedDisplayListBuilder
     // does this, so we could teach it to find and check all
     // subdocs in advance.
     if (mPreviousCaret != aBuilder->GetCaretFrame()) {
       dirty = visible;
-      aBuilder->MarkCurrentFrameModifiedDuringBuilding();
+      aBuilder->RebuildAllItemsInCurrentSubtree();
+      // Mark the old caret frame as invalid so that we remove the
+      // old nsDisplayCaret. We don't mark the current frame as invalid
+      // since we want the nsDisplaySubdocument to retain it's place
+      // in the retained display list.
+      if (mPreviousCaret) {
+        aBuilder->MarkFrameModifiedDuringBuilding(mPreviousCaret);
+      }
     }
     mPreviousCaret = aBuilder->GetCaretFrame();
   }
 
   nsDisplayList childItems;
 
   {
     DisplayListClipState::AutoSaveRestore nestedClipState(aBuilder);

--- a/layout/generic/nsSubDocumentFrame.h
+++ b/layout/generic/nsSubDocumentFrame.h
@@ -163,17 +163,16 @@ protected:
   nsIFrame* ObtainIntrinsicSizeFrame();
 
   nsView* GetViewInternal() const override { return mOuterView; }
   void SetViewInternal(nsView* aView) override { mOuterView = aView; }
 
   RefPtr<nsFrameLoader> mFrameLoader;
   nsView* mOuterView;
   nsView* mInnerView;
-  Maybe<bool> mPreviouslyNeededLayer;
   bool mIsInline;
   bool mPostedReflowCallback;
   bool mDidCreateDoc;
   bool mCallingShow;
   WeakFrame mPreviousCaret;
 };
 
 #endif /* NSSUBDOCUMENTFRAME_H_ */

--- a/layout/painting/nsDisplayList.h
+++ b/layout/painting/nsDisplayList.h
@@ -1746,16 +1746,22 @@ public:
     if (MarkFrameModifiedDuringBuilding(const_cast<nsIFrame*>(mCurrentFrame))) {
       mInInvalidSubtree = true;
       mDirtyRect = mVisibleRect;
       return true;
     }
     return false;
   }
 
+  void RebuildAllItemsInCurrentSubtree()
+  {
+    mInInvalidSubtree = true;
+    mDirtyRect = mVisibleRect;
+  }
+
   /**
    * This is a convenience function to ease the transition until AGRs and ASRs
    * are unified.
    */
   AnimatedGeometryRoot* AnimatedGeometryRootForASR(const ActiveScrolledRoot* aASR);
 
   bool HitTestIsForVisibility() const {
     return mHitTestIsForVisibility;