Bug 1180684 - Part 1. Analyze MOV with GS. r?aklotz draft
authorMakoto Kato <m_kato@ga2.so-net.ne.jp>
Wed, 24 Aug 2016 16:39:31 +0900
changeset 404827 16bc22579d530b52d177987ee4a1044bfbf01e15
parent 404651 bd7645928990649c84609d3f531e803c2d41f269
child 404828 deb94e05c6aa9f6fc6110919b6227aa2ba2fbf70
push id27317
push userm_kato@ga2.so-net.ne.jp
push dateWed, 24 Aug 2016 07:41:37 +0000
reviewersaklotz
bugs1180684
milestone51.0a1
Bug 1180684 - Part 1. Analyze MOV with GS. r?aklotz Allow MOV with GS prefix on x64. Windows 10 uses this opcode on GetKeyState(). MozReview-Commit-ID: CqDSxmWdkiH
toolkit/xre/test/win/TestDllInterceptor.cpp
xpcom/build/nsWindowsDllInterceptor.h
--- a/toolkit/xre/test/win/TestDllInterceptor.cpp
+++ b/toolkit/xre/test/win/TestDllInterceptor.cpp
@@ -157,15 +157,18 @@ int main()
       TestHook("kernel32.dll", "VirtualAlloc") &&
       TestHook("kernel32.dll", "MapViewOfFile") &&
       TestHook("gdi32.dll", "CreateDIBSection") &&
       TestHook("kernel32.dll", "CreateFileW") &&
 #endif
       TestHook("imm32.dll", "ImmGetContext") &&
       TestHook("imm32.dll", "ImmGetCompositionStringW") &&
       TestHook("imm32.dll", "ImmSetCandidateWindow") &&
+#ifdef _M_X64
+      TestHook("user32.dll", "GetKeyState") &&
+#endif
       TestDetour("ntdll.dll", "LdrLoadDll")) {
     printf("TEST-PASS | WindowsDllInterceptor | all checks passed\n");
     return 0;
   }
 
   return 1;
 }
--- a/xpcom/build/nsWindowsDllInterceptor.h
+++ b/xpcom/build/nsWindowsDllInterceptor.h
@@ -1,16 +1,17 @@
 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef NS_WINDOWS_DLL_INTERCEPTOR_H_
 #define NS_WINDOWS_DLL_INTERCEPTOR_H_
+
 #include <windows.h>
 #include <winternl.h>
 
 /*
  * Simple function interception.
  *
  * We have two separate mechanisms for intercepting a function: We can use the
  * built-in nop space, if it exists, or we can create a detour.
@@ -61,16 +62,67 @@
  *
  */
 
 #include <stdint.h>
 
 namespace mozilla {
 namespace internal {
 
+// Get op length of '/r'
+static size_t
+GetOpLengthByModRM(const uint8_t* aOp)
+{
+  uint8_t mod = *aOp >> 6;
+  uint8_t rm = *aOp & 0x7;
+
+  switch (mod) {
+  case 0:
+    if (rm == 4) {
+      // SIB
+      if ((*(aOp + 1) & 0xf) == 5) {
+        // disp32
+        return 6;
+      }
+      return 2;
+    } else if (rm == 5) {
+      // [RIP/EIP + disp32]
+      // Since we don't modify relative offset, we should mark as improssible
+      // code.
+      return 0;
+    }
+    // [r/m]
+    return 1;
+
+  case 1:
+    if (rm == 4) {
+      // [SIB + imm8]
+      return 3;
+    }
+    // [r/m + imm8]
+    return 2;
+
+  case 2:
+    if (rm == 4) {
+      // [SIB + imm32]
+      return 6;
+    }
+    // [r/m + imm32]
+    return 5;
+
+  case 3:
+    // r/w
+    return 1;
+
+  default:
+     break;
+  }
+  return 0;
+}
+
 class AutoVirtualProtect
 {
 public:
   AutoVirtualProtect(void* aFunc, size_t aSize, DWORD aProtect)
     : mFunc(aFunc), mSize(aSize), mNewProtect(aProtect), mOldProtect(0),
       mSuccess(false)
   {}
 
@@ -639,16 +691,30 @@ protected:
           }
         } else {
           // not support yet!
           return;
         }
       } else if ((origBytes[nBytes] & 0xf0) == 0x50) {
         // 1-byte push/pop
         nBytes++;
+      } else if (origBytes[nBytes] == 0x65) {
+        // GS prefix
+        if (origBytes[nBytes + 1] == 0x48 &&
+            (origBytes[nBytes + 2] >= 0x88 && origBytes[nBytes + 2] <= 0x8b)) {
+          nBytes += 3;
+          size_t len = GetOpLengthByModRM(origBytes + nBytes);
+          if (!len) {
+            // no way to support this yet.
+            return;
+          }
+          nBytes += len;
+        } else {
+          return;
+        }
       } else if (origBytes[nBytes] == 0x90) {
         // nop
         nBytes++;
       } else if (origBytes[nBytes] == 0xb8) {
         // MOV 0xB8: http://ref.x86asm.net/coder32.html#xB8
         nBytes += 5;
       } else if (origBytes[nBytes] == 0xc3) {
         // ret