Bug 1444563 - Update stencil front/back mismatch validation. - r=kvark draft
authorJeff Gilbert <jgilbert@mozilla.com>
Fri, 09 Mar 2018 16:48:51 -0800
changeset 765681 eeb2b87bf242099fa8b05c8aa907ebc0aa913067
parent 765547 415e9b18ca2a1532086d5e2d5d21343cd004b5fd
push id102141
push userbmo:jgilbert@mozilla.com
push dateSat, 10 Mar 2018 00:47:29 +0000
reviewerskvark
bugs1444563
milestone60.0a1
Bug 1444563 - Update stencil front/back mismatch validation. - r=kvark MozReview-Commit-ID: GyCpJ0a1F9H
dom/canvas/WebGLContext.h
dom/canvas/WebGLContextDraw.cpp
dom/canvas/WebGLContextValidate.cpp
--- a/dom/canvas/WebGLContext.h
+++ b/dom/canvas/WebGLContext.h
@@ -1632,17 +1632,17 @@ protected:
     bool ValidateFaceEnum(GLenum face, const char* info);
     bool ValidateTexInputData(GLenum type, js::Scalar::Type jsArrayType,
                               WebGLTexImageFunc func, WebGLTexDimensions dims);
     bool ValidateDrawModeEnum(GLenum mode, const char* info);
     bool ValidateAttribIndex(GLuint index, const char* info);
     bool ValidateAttribPointer(bool integerMode, GLuint index, GLint size, GLenum type,
                                WebGLboolean normalized, GLsizei stride,
                                WebGLintptr byteOffset, const char* info);
-    bool ValidateStencilParamsForDrawCall();
+    bool ValidateStencilParamsForDrawCall(const char* funcName) const;
 
     bool ValidateCopyTexImage(TexInternalFormat srcFormat, TexInternalFormat dstformat,
                               WebGLTexImageFunc func, WebGLTexDimensions dims);
 
     bool ValidateTexImage(TexImageTarget texImageTarget,
                           GLint level, GLenum internalFormat,
                           GLint xoffset, GLint yoffset, GLint zoffset,
                           GLint width, GLint height, GLint depth,
--- a/dom/canvas/WebGLContextDraw.cpp
+++ b/dom/canvas/WebGLContextDraw.cpp
@@ -214,16 +214,59 @@ WebGLContext::BindFakeBlack(uint32_t tex
     gl->fActiveTexture(LOCAL_GL_TEXTURE0 + texUnit);
     gl->fBindTexture(target.get(), fakeBlackTex->mGLName);
     gl->fActiveTexture(LOCAL_GL_TEXTURE0 + mActiveTexture);
     return true;
 }
 
 ////////////////////////////////////////
 
+bool
+WebGLContext::ValidateStencilParamsForDrawCall(const char* const funcName) const
+{
+    const auto stencilBits = [&]() -> uint8_t {
+        if (!mStencilTestEnabled)
+            return 0;
+
+        if (!mBoundDrawFramebuffer)
+            return mOptions.stencil ? 8 : 0;
+
+        if (mBoundDrawFramebuffer->StencilAttachment().IsDefined())
+            return 8;
+
+        if (mBoundDrawFramebuffer->DepthStencilAttachment().IsDefined())
+            return 8;
+
+        return 0;
+    }();
+    const uint32_t stencilMax = (1 << stencilBits) - 1;
+
+    const auto fnMask = [&](const uint32_t x) { return x & stencilMax; };
+    const auto fnClamp = [&](const int32_t x) {
+        return std::max(0, std::min(x, (int32_t)stencilMax));
+    };
+
+    bool ok = true;
+    ok &= (fnMask(mStencilWriteMaskFront) == fnMask(mStencilWriteMaskBack));
+    ok &= (fnMask(mStencilValueMaskFront) == fnMask(mStencilValueMaskBack));
+    ok &= (fnClamp(mStencilRefFront) == fnClamp(mStencilRefBack));
+
+    if (!ok) {
+        ErrorInvalidOperation("%s: Stencil front/back state must effectively match."
+                              " (before front/back comparison, WRITEMASK and VALUE_MASK"
+                              " are masked with (2^s)-1, and REF is clamped to"
+                              " [0, (2^s)-1], where `s` is the number of enabled stencil"
+                              " bits in the draw framebuffer)",
+                              funcName);
+    }
+    return ok;
+}
+
+////////////////////////////////////////
+
 template<typename T>
 static bool
 DoSetsIntersect(const std::set<T>& a, const std::set<T>& b)
 {
     std::vector<T> intersection;
     std::set_intersection(a.begin(), a.end(), b.begin(), b.end(),
                           std::back_inserter(intersection));
     return bool(intersection.size());
@@ -248,17 +291,17 @@ public:
             return;
         }
 
         if (!mWebGL->ValidateDrawModeEnum(mode, funcName)) {
             *out_error = true;
             return;
         }
 
-        if (!mWebGL->ValidateStencilParamsForDrawCall()) {
+        if (!mWebGL->ValidateStencilParamsForDrawCall(funcName)) {
             *out_error = true;
             return;
         }
 
         if (!mWebGL->mActiveProgramLinkInfo) {
             mWebGL->ErrorInvalidOperation("%s: The current program is not linked.", funcName);
             *out_error = true;
             return;
--- a/dom/canvas/WebGLContextValidate.cpp
+++ b/dom/canvas/WebGLContextValidate.cpp
@@ -361,40 +361,16 @@ WebGLContext::ValidateAttribIndex(GLuint
                               " MAX_VERTEX_ATTRIBS.", info);
         }
     }
 
     return valid;
 }
 
 bool
-WebGLContext::ValidateStencilParamsForDrawCall()
-{
-    const char msg[] = "%s set different front and back stencil %s. Drawing in"
-                       " this configuration is not allowed.";
-
-    if (mStencilRefFront != mStencilRefBack) {
-        ErrorInvalidOperation(msg, "stencilFuncSeparate", "reference values");
-        return false;
-    }
-
-    if (mStencilValueMaskFront != mStencilValueMaskBack) {
-        ErrorInvalidOperation(msg, "stencilFuncSeparate", "value masks");
-        return false;
-    }
-
-    if (mStencilWriteMaskFront != mStencilWriteMaskBack) {
-        ErrorInvalidOperation(msg, "stencilMaskSeparate", "write masks");
-        return false;
-    }
-
-    return true;
-}
-
-bool
 WebGLContext::InitAndValidateGL(FailureReason* const out_failReason)
 {
     MOZ_RELEASE_ASSERT(gl, "GFX: GL not initialized");
 
     // Unconditionally create a new format usage authority. This is
     // important when restoring contexts and extensions need to add
     // formats back into the authority.
     mFormatUsage = CreateFormatUsage(gl);