--- a/intl/l10n/DOMLocalization.jsm
+++ b/intl/l10n/DOMLocalization.jsm
@@ -653,17 +653,20 @@ class DOMLocalization extends Localizati
           for (let i = 0; i < overlayTranslations.length; i++) {
             if (overlayTranslations[i] !== undefined &&
                 untranslatedElements[i] !== undefined) {
               translateElement(untranslatedElements[i], overlayTranslations[i]);
             }
           }
           this.resumeObserving();
         })
-        .catch(() => this.resumeObserving());
+        .catch((e) => {
+          this.resumeObserving();
+          throw e;
+        });
     }
     return this.translateElements(this.getTranslatables(frag));
   }
 
   /**
    * Translate a list of DOM elements asynchronously using this
    * `DOMLocalization` object.
    *

--- a/intl/l10n/Localization.jsm
+++ b/intl/l10n/Localization.jsm
@@ -151,20 +151,24 @@ class Localization {
         ctx = await ctx;
       }
       const missingIds = keysFromContext(method, ctx, keys, translations);
 
       if (missingIds.size === 0) {
         break;
       }
 
-      if (AppConstants.NIGHTLY_BUILD) {
+      if (AppConstants.NIGHTLY_BUILD || Cu.isInAutomation) {
         const locale = ctx.locales[0];
         const ids = Array.from(missingIds).join(", ");
-        console.warn(`Missing translations in ${locale}: ${ids}`);
+        if (Cu.isInAutomation) {
+          throw new Error(`Missing translations in ${locale}: ${ids}`);
+        } else {
+          console.warn(`Missing translations in ${locale}: ${ids}`);
+        }
       }
     }
 
     return translations;
   }
 
   /**
    * Format translations into {value, attrs} objects.