--- a/browser/components/downloads/content/downloads.js
+++ b/browser/components/downloads/content/downloads.js
@@ -1485,16 +1485,20 @@ const DownloadsFooter = {
    */
   set showingSummary(aValue) {
     if (this._footerNode) {
       if (aValue) {
         this._footerNode.setAttribute("showingsummary", "true");
       } else {
         this._footerNode.removeAttribute("showingsummary");
       }
+      if (!aValue && this._showingSummary) {
+        // Make sure the panel's height shrinks when the summary is hidden.
+        DownloadsBlockedSubview.view.setHeightToFit();
+      }
       this._showingSummary = aValue;
     }
     return aValue;
   },
 
   /**
    * Element corresponding to the footer of the downloads panel.
    */