new file mode 100644
--- /dev/null
+++ b/layout/reftests/svg/nested-mask-mode.svg
@@ -0,0 +1,13 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="200" height="200">
+  <title>Testcase for mask-mode in a nested mask.</title>
+  <mask id="m1" width="1" height="1">
+    <!-- Set mask-mode as alpha. The fill color of the rect in m2 should
+         not effect mask result. -->
+    <rect x="0" y="0" width="100" height="100" mask="url(#m2)" style="fill: #ffffff; mask-mode:alpha;"/>
+  </mask>
+  <mask id="m2" width="1" height="1">
+    <rect x="0" y="0" width="100" height="50" style="fill:#000000"/>
+  </mask>
+
+  <rect width="100" height="100" fill="blue" mask="url(#m1)" />
+</svg>

--- a/layout/reftests/svg/reftest.list
+++ b/layout/reftests/svg/reftest.list
@@ -453,8 +453,10 @@ fuzzy(1,32400) == clipPath-on-outflowEle
 fuzzy(1,32400) == clipPath-on-outflowElement-02b.html clipPath-on-outflowElement-02-ref.html
 default-preferences
 
 == mask-on-outflowElement-01a.html clipPath-on-outflowElement-01-ref.html
 == mask-on-outflowElement-01b.html clipPath-on-outflowElement-01-ref.html
 
 == clipPath-and-mask-on-outflowElement-01a.html clipPath-on-outflowElement-01-ref.html
 == clipPath-and-mask-on-outflowElement-01b.html clipPath-on-outflowElement-01-ref.html
+
+!= nested-mask-mode.svg about:blank
\ No newline at end of file

--- a/layout/svg/nsSVGIntegrationUtils.cpp
+++ b/layout/svg/nsSVGIntegrationUtils.cpp
@@ -454,54 +454,60 @@ PaintMaskSurface(const PaintFramesParams
   RefPtr<gfxContext> maskContext = gfxContext::CreateOrNull(aMaskDT);
   MOZ_ASSERT(maskContext);
   maskContext->SetMatrix(aMaskSurfaceMatrix);
 
   // Multiple SVG masks interleave with image mask. Paint each layer onto
   // aMaskDT one at a time.
   for (int i = aMaskFrames.Length() - 1; i >= 0 ; i--) {
     nsSVGMaskFrame *maskFrame = aMaskFrames[i];
-
+    DrawResult result = DrawResult::SUCCESS;
     CompositionOp compositionOp = (i == int(aMaskFrames.Length() - 1))
       ? CompositionOp::OP_OVER
       : nsCSSRendering::GetGFXCompositeMode(svgReset->mMask.mLayers[i].mComposite);
 
     // maskFrame != nullptr means we get a SVG mask.
     // maskFrame == nullptr means we get an image mask.
     if (maskFrame) {
       Matrix svgMaskMatrix;
-      RefPtr<SourceSurface> svgMask =
-        maskFrame->GetMaskForMaskedFrame(maskContext, aParams.frame,
-                                         cssPxToDevPxMatrix,
-                                         aOpacity,
-                                         &svgMaskMatrix,
-                                         svgReset->mMask.mLayers[i].mMaskMode);
+      nsSVGMaskFrame::MaskParams params(maskContext, aParams.frame,
+                                                  cssPxToDevPxMatrix,
+                                                  aOpacity, &svgMaskMatrix,
+                                                  svgReset->mMask.mLayers[i].mMaskMode);
+      RefPtr<SourceSurface> svgMask;
+      Tie(result, svgMask) = maskFrame->GetMaskForMaskedFrame(params);
+
       if (svgMask) {
+        MOZ_ASSERT(result == DrawResult::SUCCESS);
         gfxContextMatrixAutoSaveRestore matRestore(maskContext);
 
         maskContext->Multiply(ThebesMatrix(svgMaskMatrix));
         aMaskDT->MaskSurface(ColorPattern(Color(0.0, 0.0, 0.0, 1.0)), svgMask,
                              Point(0, 0),
                              DrawOptions(1.0, compositionOp));
       }
+
+      if (result != DrawResult::SUCCESS) {
+        return result;
+      }
     } else {
       gfxContextMatrixAutoSaveRestore matRestore(maskContext);
 
       maskContext->Multiply(gfxMatrix::Translation(-devPixelOffsetToUserSpace));
       nsRenderingContext rc(maskContext);
       nsCSSRendering::PaintBGParams  params =
         nsCSSRendering::PaintBGParams::ForSingleLayer(*presContext,
                                                       rc, aParams.dirtyRect,
                                                       aParams.borderArea,
                                                       aParams.frame,
                                                       aParams.builder->GetBackgroundPaintFlags() |
                                                       nsCSSRendering::PAINTBG_MASK_IMAGE,
                                                       i, compositionOp);
 
-      DrawResult result =
+      result =
         nsCSSRendering::PaintBackgroundWithSC(params, aSC,
                                               *aParams.frame->StyleBorder());
       if (result != DrawResult::SUCCESS) {
         return result;
       }
     }
   }
 
@@ -516,29 +522,29 @@ CreateAndPaintMaskSurface(const PaintFra
                           Matrix& aOutMaskTransform,
                           RefPtr<SourceSurface>& aOutMaskSurface,
                           bool& aOpacityApplied)
 {
   const nsStyleSVGReset *svgReset = aSC->StyleSVGReset();
   MOZ_ASSERT(aMaskFrames.Length() > 0);
 
   gfxContext& ctx = aParams.ctx;
+  DrawResult result = DrawResult::SUCCESS;
 
   // There is only one SVG mask.
   if (((aMaskFrames.Length() == 1) && aMaskFrames[0])) {
     gfxMatrix cssPxToDevPxMatrix =
     nsSVGIntegrationUtils::GetCSSPxToDevPxMatrix(aParams.frame);
-
+    nsSVGMaskFrame::MaskParams params(&ctx, aParams.frame, cssPxToDevPxMatrix,
+                                      aOpacity, &aOutMaskTransform,
+                                      svgReset->mMask.mLayers[0].mMaskMode);
     aOpacityApplied = true;
-    aOutMaskSurface =
-      aMaskFrames[0]->GetMaskForMaskedFrame(&ctx, aParams.frame,
-                                            cssPxToDevPxMatrix, aOpacity,
-                                            &aOutMaskTransform,
-                                            svgReset->mMask.mLayers[0].mMaskMode);
-    return DrawResult::SUCCESS;
+    Tie(result, aOutMaskSurface) =
+      aMaskFrames[0]->GetMaskForMaskedFrame(params);
+    return result;
   }
 
   const IntRect& maskSurfaceRect = aParams.maskRect;
   if (maskSurfaceRect.IsEmpty()) {
     return DrawResult::SUCCESS;
   }
 
   RefPtr<DrawTarget> maskDT =
@@ -553,20 +559,19 @@ CreateAndPaintMaskSurface(const PaintFra
   // caller does not need to apply it again.
   aOpacityApplied = !HasNonSVGMask(aMaskFrames);
 
   // Set context's matrix on maskContext, offset by the maskSurfaceRect's
   // position. This makes sure that we combine the masks in device space.
   gfxMatrix maskSurfaceMatrix =
     ctx.CurrentMatrix() * gfxMatrix::Translation(-aParams.maskRect.TopLeft());
 
-  DrawResult result = PaintMaskSurface(aParams, maskDT,
-                                       aOpacityApplied ? aOpacity : 1.0,
-                                       aSC, aMaskFrames, maskSurfaceMatrix,
-                                       aOffsetToUserSpace);
+  result = PaintMaskSurface(aParams, maskDT, aOpacityApplied ? aOpacity : 1.0,
+                            aSC, aMaskFrames, maskSurfaceMatrix,
+                            aOffsetToUserSpace);
   if (result != DrawResult::SUCCESS) {
     return result;
   }
 
   aOutMaskTransform = ToMatrix(maskSurfaceMatrix);
   if (!aOutMaskTransform.Invert()) {
     return DrawResult::SUCCESS;
   }

--- a/layout/svg/nsSVGMaskFrame.cpp
+++ b/layout/svg/nsSVGMaskFrame.cpp
@@ -195,144 +195,143 @@ ComputeAlphaMask(const uint8_t *aSourceD
 nsIFrame*
 NS_NewSVGMaskFrame(nsIPresShell* aPresShell, nsStyleContext* aContext)
 {
   return new (aPresShell) nsSVGMaskFrame(aContext);
 }
 
 NS_IMPL_FRAMEARENA_HELPERS(nsSVGMaskFrame)
 
-already_AddRefed<SourceSurface>
-nsSVGMaskFrame::GetMaskForMaskedFrame(gfxContext* aContext,
-                                      nsIFrame* aMaskedFrame,
-                                      const gfxMatrix &aMatrix,
-                                      float aOpacity,
-                                      Matrix* aMaskTransform,
-                                      uint8_t aMaskOp)
+mozilla::Pair<DrawResult, RefPtr<SourceSurface>>
+nsSVGMaskFrame::GetMaskForMaskedFrame(MaskParams& aParams)
 {
   // If the flag is set when we get here, it means this mask frame
   // has already been used painting the current mask, and the document
   // has a mask reference loop.
   if (mInUse) {
     NS_WARNING("Mask loop detected!");
-    return nullptr;
+    return MakePair(DrawResult::SUCCESS, RefPtr<SourceSurface>());
   }
   AutoMaskReferencer maskRef(this);
 
-  gfxRect maskArea = GetMaskArea(aMaskedFrame);
-
+  gfxRect maskArea = GetMaskArea(aParams.maskedFrame);
+  gfxContext* context = aParams.ctx;
   // Get the clip extents in device space:
   // Minimizing the mask surface extents (using both the current clip extents
   // and maskArea) is important for performance.
-  aContext->Save();
-  nsSVGUtils::SetClipRect(aContext, aMatrix, maskArea);
-  aContext->SetMatrix(gfxMatrix());
-  gfxRect maskSurfaceRect = aContext->GetClipExtents();
+  context->Save();
+  nsSVGUtils::SetClipRect(context, aParams.toUserSpace, maskArea);
+  context->SetMatrix(gfxMatrix());
+  gfxRect maskSurfaceRect = context->GetClipExtents();
   maskSurfaceRect.RoundOut();
-  aContext->Restore();
+  context->Restore();
 
   bool resultOverflows;
   IntSize maskSurfaceSize =
     nsSVGUtils::ConvertToSurfaceSize(maskSurfaceRect.Size(), &resultOverflows);
 
   if (resultOverflows || maskSurfaceSize.IsEmpty()) {
-    // XXXjwatt we should return an empty surface so we don't paint aMaskedFrame!
-    return nullptr;
+    // XXXjwatt we should return an empty surface so we don't paint
+    // aParams.maskedFrame!
+    return MakePair(DrawResult::TEMPORARY_ERROR, RefPtr<SourceSurface>());
   }
 
   RefPtr<DrawTarget> maskDT =
     Factory::CreateDrawTarget(BackendType::CAIRO, maskSurfaceSize,
                               SurfaceFormat::B8G8R8A8);
   if (!maskDT || !maskDT->IsValid()) {
-    return nullptr;
+    return MakePair(DrawResult::TEMPORARY_ERROR, RefPtr<SourceSurface>());
   }
 
   gfxMatrix maskSurfaceMatrix =
-    aContext->CurrentMatrix() * gfxMatrix::Translation(-maskSurfaceRect.TopLeft());
+    context->CurrentMatrix() * gfxMatrix::Translation(-maskSurfaceRect.TopLeft());
 
   RefPtr<gfxContext> tmpCtx = gfxContext::CreateOrNull(maskDT);
   MOZ_ASSERT(tmpCtx); // already checked the draw target above
   tmpCtx->SetMatrix(maskSurfaceMatrix);
 
-  mMatrixForChildren = GetMaskTransform(aMaskedFrame) * aMatrix;
+  mMatrixForChildren = GetMaskTransform(aParams.maskedFrame) *
+                       aParams.toUserSpace;
+  DrawResult result;
 
   for (nsIFrame* kid = mFrames.FirstChild(); kid;
        kid = kid->GetNextSibling()) {
     // The CTM of each frame referencing us can be different
     nsISVGChildFrame* SVGFrame = do_QueryFrame(kid);
     if (SVGFrame) {
       SVGFrame->NotifySVGChanged(nsISVGChildFrame::TRANSFORM_CHANGED);
     }
     gfxMatrix m = mMatrixForChildren;
     if (kid->GetContent()->IsSVGElement()) {
       m = static_cast<nsSVGElement*>(kid->GetContent())->
             PrependLocalTransformsTo(m);
     }
-    DrawResult result = nsSVGUtils::PaintFrameWithEffects(kid, *tmpCtx, m);
+    result = nsSVGUtils::PaintFrameWithEffects(kid, *tmpCtx, m);
     if (result != DrawResult::SUCCESS) {
-      return nullptr;
+      return MakePair(result, RefPtr<SourceSurface>());
     }
   }
 
   RefPtr<SourceSurface> maskSnapshot = maskDT->Snapshot();
   if (!maskSnapshot) {
-    return nullptr;
+    return MakePair(DrawResult::TEMPORARY_ERROR, RefPtr<SourceSurface>());
   }
   RefPtr<DataSourceSurface> maskSurface = maskSnapshot->GetDataSurface();
   DataSourceSurface::MappedSurface map;
   if (!maskSurface->Map(DataSourceSurface::MapType::READ, &map)) {
-    return nullptr;
+    return MakePair(DrawResult::TEMPORARY_ERROR, RefPtr<SourceSurface>());
   }
 
   // Create alpha channel mask for output
   RefPtr<DataSourceSurface> destMaskSurface =
     Factory::CreateDataSourceSurface(maskSurfaceSize, SurfaceFormat::A8);
   if (!destMaskSurface) {
-    return nullptr;
+    return MakePair(DrawResult::TEMPORARY_ERROR, RefPtr<SourceSurface>());
   }
   DataSourceSurface::MappedSurface destMap;
   if (!destMaskSurface->Map(DataSourceSurface::MapType::WRITE, &destMap)) {
-    return nullptr;
+    return MakePair(DrawResult::TEMPORARY_ERROR, RefPtr<SourceSurface>());
   }
 
   uint8_t maskType;
-  if (aMaskOp == NS_STYLE_MASK_MODE_MATCH_SOURCE) {
+  if (aParams.maskMode == NS_STYLE_MASK_MODE_MATCH_SOURCE) {
     maskType = StyleSVGReset()->mMaskType;
   } else {
-    maskType = aMaskOp == NS_STYLE_MASK_MODE_LUMINANCE ?
-                 NS_STYLE_MASK_TYPE_LUMINANCE : NS_STYLE_MASK_TYPE_ALPHA;
+    maskType = aParams.maskMode == NS_STYLE_MASK_MODE_LUMINANCE
+               ? NS_STYLE_MASK_TYPE_LUMINANCE : NS_STYLE_MASK_TYPE_ALPHA;
   }
 
   if (maskType == NS_STYLE_MASK_TYPE_LUMINANCE) {
     if (StyleSVG()->mColorInterpolation ==
         NS_STYLE_COLOR_INTERPOLATION_LINEARRGB) {
       ComputeLinearRGBLuminanceMask(map.mData, map.mStride,
                                     destMap.mData, destMap.mStride,
-                                    maskSurfaceSize, aOpacity);
+                                    maskSurfaceSize, aParams.opacity);
     } else {
       ComputesRGBLuminanceMask(map.mData, map.mStride,
                                destMap.mData, destMap.mStride,
-                               maskSurfaceSize, aOpacity);
+                               maskSurfaceSize, aParams.opacity);
     }
   } else {
-      ComputeAlphaMask(map.mData, map.mStride,
-                       destMap.mData, destMap.mStride,
-                       maskSurfaceSize, aOpacity);
+    ComputeAlphaMask(map.mData, map.mStride,
+                     destMap.mData, destMap.mStride,
+                     maskSurfaceSize, aParams.opacity);
   }
 
   maskSurface->Unmap();
   destMaskSurface->Unmap();
 
   // Moz2D transforms in the opposite direction to Thebes
   if (!maskSurfaceMatrix.Invert()) {
-    return nullptr;
+    return MakePair(DrawResult::TEMPORARY_ERROR, RefPtr<SourceSurface>());
   }
 
-  *aMaskTransform = ToMatrix(maskSurfaceMatrix);
-  return destMaskSurface.forget();
+  *aParams.maskTransform = ToMatrix(maskSurfaceMatrix);
+  RefPtr<SourceSurface> surface = destMaskSurface.forget();
+  return MakePair(DrawResult::SUCCESS, Move(surface));
 }
 
 gfxRect
 nsSVGMaskFrame::GetMaskArea(nsIFrame* aMaskedFrame)
 {
   SVGMaskElement *maskElem = static_cast<SVGMaskElement*>(mContent);
 
   uint16_t units =

--- a/layout/svg/nsSVGMaskFrame.h
+++ b/layout/svg/nsSVGMaskFrame.h
@@ -4,16 +4,17 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef __NS_SVGMASKFRAME_H__
 #define __NS_SVGMASKFRAME_H__
 
 #include "mozilla/Attributes.h"
 #include "mozilla/gfx/2D.h"
 #include "mozilla/RefPtr.h"
+#include "mozilla/Pair.h"
 #include "gfxPattern.h"
 #include "gfxMatrix.h"
 #include "nsSVGContainerFrame.h"
 #include "nsSVGUtils.h"
 
 class gfxContext;
 
 /**
@@ -33,36 +34,48 @@ class gfxContext;
 
 class nsSVGMaskFrame final : public nsSVGContainerFrame
 {
   friend nsIFrame*
   NS_NewSVGMaskFrame(nsIPresShell* aPresShell, nsStyleContext* aContext);
 
   typedef mozilla::gfx::Matrix Matrix;
   typedef mozilla::gfx::SourceSurface SourceSurface;
+  typedef mozilla::image::DrawResult DrawResult;
 
 protected:
   explicit nsSVGMaskFrame(nsStyleContext* aContext)
     : nsSVGContainerFrame(aContext)
     , mInUse(false)
   {
     AddStateBits(NS_FRAME_IS_NONDISPLAY);
   }
 
 public:
   NS_DECL_FRAMEARENA_HELPERS
 
+  struct MaskParams {
+    gfxContext* ctx;
+    nsIFrame* maskedFrame;
+    const gfxMatrix& toUserSpace;
+    float opacity;
+    Matrix* maskTransform;
+    uint8_t maskMode;
+
+    explicit MaskParams(gfxContext* aCtx, nsIFrame* aMaskedFrame,
+                        const gfxMatrix& aToUserSpace, float aOpacity,
+                        Matrix* aMaskTransform, uint8_t aMaskMode)
+    : ctx(aCtx), maskedFrame(aMaskedFrame), toUserSpace(aToUserSpace),
+      opacity(aOpacity), maskTransform(aMaskTransform), maskMode(aMaskMode)
+    { }
+  };
+
   // nsSVGMaskFrame method:
-  already_AddRefed<SourceSurface>
-  GetMaskForMaskedFrame(gfxContext* aContext,
-                        nsIFrame* aMaskedFrame,
-                        const gfxMatrix &aMatrix,
-                        float aOpacity,
-                        Matrix* aMaskTransform,
-                        uint8_t aMaskOp = NS_STYLE_MASK_MODE_MATCH_SOURCE);
+  mozilla::Pair<DrawResult, RefPtr<SourceSurface>>
+  GetMaskForMaskedFrame(MaskParams& aParams);
 
   gfxRect
   GetMaskArea(nsIFrame* aMaskedFrame);
 
   virtual nsresult AttributeChanged(int32_t         aNameSpaceID,
                                     nsIAtom*        aAttribute,
                                     int32_t         aModType) override;
 

--- a/layout/svg/nsSVGUtils.cpp
+++ b/layout/svg/nsSVGUtils.cpp
@@ -736,48 +736,58 @@ nsSVGUtils::PaintFrameWithEffects(nsIFra
       ? RefPtr<gfxContext>(&aContext).forget()
       : CreateBlendTarget(&aContext, targetOffset);
   aContext.Restore();
 
   if (!target) {
     return DrawResult::TEMPORARY_ERROR;
   }
 
+  DrawResult result = DrawResult::SUCCESS;
+
   /* Check if we need to do additional operations on this child's
    * rendering, which necessitates rendering into another surface. */
   bool shouldGenerateMask = (maskUsage.opacity != 1.0f ||
                              maskUsage.shouldGenerateClipMaskLayer ||
                              maskUsage.shouldGenerateMaskLayer ||
                              aFrame->StyleEffects()->mMixBlendMode != NS_STYLE_BLEND_NORMAL);
 
   if (shouldGenerateMask) {
     Matrix maskTransform;
     RefPtr<SourceSurface> maskSurface;
 
     if (maskUsage.shouldGenerateMaskLayer) {
-      maskSurface =
-        maskFrame->GetMaskForMaskedFrame(&aContext, aFrame, aTransform,
-                                         maskUsage.opacity, &maskTransform);
+      uint8_t maskMode =
+        aFrame->StyleSVGReset()->mMask.mLayers[0].mMaskMode;
+      nsSVGMaskFrame::MaskParams params(&aContext, aFrame, aTransform,
+                                        maskUsage.opacity, &maskTransform,
+                                        maskMode);
+      Tie(result, maskSurface) = maskFrame->GetMaskForMaskedFrame(params);
 
       if (!maskSurface) {
-        // Entire surface is clipped out.
-        return DrawResult::SUCCESS;
+        // Either entire surface is clipped out, or gfx buffer allocation
+        // failure in nsSVGMaskFrame::GetMaskForMaskedFrame.
+        return result;
       }
     }
 
     if (maskUsage.shouldGenerateClipMaskLayer) {
       Matrix clippedMaskTransform;
       RefPtr<SourceSurface> clipMaskSurface =
         clipPathFrame->GetClipMask(aContext, aFrame, aTransform,
                                    &clippedMaskTransform, maskSurface,
-                                   maskTransform);
+                                   maskTransform, &result);
 
       if (clipMaskSurface) {
         maskSurface = clipMaskSurface;
         maskTransform = clippedMaskTransform;
+      } else {
+        // Either entire surface is clipped out, or gfx buffer allocation
+        // failure in nsSVGClipPathFrame::GetClipMask.
+        return result;
       }
     }
 
     // SVG mask multiply opacity into maskSurface already, so we do not bother
     // to apply opacity again.
     float opacity = maskFrame ? 1.0 : maskUsage.opacity;
     target->PushGroupForBlendBack(gfxContentType::COLOR_ALPHA, opacity,
                                   maskSurface, maskTransform);
@@ -789,18 +799,16 @@ nsSVGUtils::PaintFrameWithEffects(nsIFra
   if (maskUsage.shouldApplyClipPath || maskUsage.shouldApplyBasicShape) {
     if (maskUsage.shouldApplyClipPath) {
       clipPathFrame->ApplyClipPath(aContext, aFrame, aTransform);
     } else {
       nsCSSClipPathInstance::ApplyBasicShapeClip(aContext, aFrame);
     }
   }
 
-  DrawResult result = DrawResult::SUCCESS;
-
   /* Paint the child */
   if (effectProperties.HasValidFilter()) {
     nsRegion* dirtyRegion = nullptr;
     nsRegion tmpDirtyRegion;
     if (aDirtyRect) {
       // aDirtyRect is in outer-<svg> device pixels, but the filter code needs
       // it in frame space.
       gfxMatrix userToDeviceSpace = GetUserToCanvasTM(aFrame);