Bug 1264566 - Part 1: Refactor FileDescriptor to fix leaks in content processes. r?valentin,baku draft
authorWei-Cheng Pan <wpan@mozilla.com>
Fri, 27 May 2016 15:58:51 +0800
changeset 387515 8aca966c40695a08d14af38cf2d08b36e59e4813
parent 386893 04821a70c739a00d12e12df651c0989441e22728
child 387516 26ca0d28089b54975aa13d71cd21a15fef45830e
push id22974
push userbmo:wpan@mozilla.com
push dateThu, 14 Jul 2016 06:19:47 +0000
reviewersvalentin, baku
bugs1264566
milestone50.0a1
Bug 1264566 - Part 1: Refactor FileDescriptor to fix leaks in content processes. r?valentin,baku Now FileDescriptor takes the ownership of the platform handle, in both parent and content processes. Copying will duplicate the underlying handle. It also comes with a move constructor to avoid duplicating if possible. The getter will duplicate a new handle, so a UniquePtr is needed to hold the duplicated handle. MozReview-Commit-ID: DgvjmTI4tpf
ipc/glue/FileDescriptor.cpp
ipc/glue/FileDescriptor.h
--- a/ipc/glue/FileDescriptor.cpp
+++ b/ipc/glue/FileDescriptor.cpp
@@ -2,16 +2,17 @@
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "FileDescriptor.h"
 
 #include "mozilla/Assertions.h"
+#include "mozilla/Move.h"
 #include "nsDebug.h"
 
 #ifdef XP_WIN
 
 #include <windows.h>
 #include "ProtocolUtils.h"
 #define INVALID_HANDLE INVALID_HANDLE_VALUE
 
@@ -26,75 +27,91 @@
 #include "base/eintr_wrapper.h"
 #define INVALID_HANDLE -1
 
 #endif // XP_WIN
 
 using mozilla::ipc::FileDescriptor;
 
 FileDescriptor::FileDescriptor()
-  : mHandle(INVALID_HANDLE), mHandleCreatedByOtherProcess(false)
-#ifdef DEBUG
-  , mHandleCreatedByOtherProcessWasUsed(false)
-#endif
-{ }
+  : mHandle(INVALID_HANDLE)
+{
+}
+
+FileDescriptor::FileDescriptor(const FileDescriptor& aOther)
+  : mHandle(INVALID_HANDLE)
+{
+  Assign(aOther);
+}
+
+FileDescriptor::FileDescriptor(FileDescriptor&& aOther)
+  : mHandle(INVALID_HANDLE)
+{
+  *this = mozilla::Move(aOther);
+}
 
 FileDescriptor::FileDescriptor(PlatformHandleType aHandle)
-  : mHandle(INVALID_HANDLE), mHandleCreatedByOtherProcess(false)
-#ifdef DEBUG
-  , mHandleCreatedByOtherProcessWasUsed(false)
+  : mHandle(INVALID_HANDLE)
+{
+  mHandle = Clone(aHandle);
+}
+
+FileDescriptor::FileDescriptor(const IPDLPrivate&, const PickleType& aPickle)
+  : mHandle(INVALID_HANDLE)
+{
+#ifdef XP_WIN
+  mHandle = aPickle;
+#else
+  mHandle = aPickle.fd;
 #endif
+}
+
+FileDescriptor::~FileDescriptor()
 {
-  DuplicateInCurrentProcess(aHandle);
+  Close();
+}
+
+FileDescriptor&
+FileDescriptor::operator=(const FileDescriptor& aOther)
+{
+  if (this != &aOther) {
+    Assign(aOther);
+  }
+  return *this;
+}
+
+FileDescriptor&
+FileDescriptor::operator=(FileDescriptor&& aOther)
+{
+  if (this != &aOther) {
+    Close();
+    mHandle = aOther.mHandle;
+    aOther.mHandle = INVALID_HANDLE;
+  }
+  return *this;
+}
+
+bool
+FileDescriptor::IsValid() const
+{
+  return IsValid(mHandle);
 }
 
 void
-FileDescriptor::DuplicateInCurrentProcess(PlatformHandleType aHandle)
+FileDescriptor::Assign(const FileDescriptor& aOther)
 {
-  MOZ_ASSERT_IF(mHandleCreatedByOtherProcess && IsValid(),
-                mHandleCreatedByOtherProcessWasUsed);
-
-  if (IsValid(aHandle)) {
-    PlatformHandleType newHandle;
-#ifdef XP_WIN
-    if (::DuplicateHandle(GetCurrentProcess(), aHandle, GetCurrentProcess(),
-                          &newHandle, 0, FALSE, DUPLICATE_SAME_ACCESS)) {
-#else // XP_WIN
-    if ((newHandle = dup(aHandle)) != INVALID_HANDLE) {
-#endif
-      mHandle = newHandle;
-      return;
-    }
-    NS_WARNING("Failed to duplicate file handle for current process!");
-  }
-
-  mHandle = INVALID_HANDLE;
+  Close();
+  mHandle = Clone(aOther.mHandle);
 }
 
 void
-FileDescriptor::CloseCurrentProcessHandle()
+FileDescriptor::Close()
 {
-  MOZ_ASSERT_IF(mHandleCreatedByOtherProcess && IsValid(),
-                mHandleCreatedByOtherProcessWasUsed);
-
-  // Don't actually close handles created by another process.
-  if (mHandleCreatedByOtherProcess) {
-    return;
-  }
-
-  if (IsValid()) {
-#ifdef XP_WIN
-    if (!CloseHandle(mHandle)) {
-      NS_WARNING("Failed to close file handle for current process!");
-    }
-#else // XP_WIN
-    HANDLE_EINTR(close(mHandle));
-#endif
-    mHandle = INVALID_HANDLE;
-  }
+  Close(mHandle);
+  mHandle = INVALID_HANDLE;
 }
 
 FileDescriptor::PickleType
 FileDescriptor::ShareTo(const FileDescriptor::IPDLPrivate&,
                         FileDescriptor::ProcessId aTargetPid) const
 {
   PlatformHandleType newHandle;
 #ifdef XP_WIN
@@ -115,14 +132,95 @@ FileDescriptor::ShareTo(const FileDescri
     NS_WARNING("Failed to duplicate file handle for other process!");
   }
   return base::FileDescriptor();
 #endif
 
   MOZ_CRASH("Must not get here!");
 }
 
+FileDescriptor::UniquePlatformHandle
+FileDescriptor::ClonePlatformHandle() const
+{
+  return UniquePlatformHandle(Clone(mHandle));
+}
+
+bool
+FileDescriptor::operator==(const FileDescriptor& aOther) const
+{
+  return mHandle == aOther.mHandle;
+}
+
 // static
 bool
 FileDescriptor::IsValid(PlatformHandleType aHandle)
 {
   return aHandle != INVALID_HANDLE;
 }
+
+// static
+FileDescriptor::PlatformHandleType
+FileDescriptor::Clone(PlatformHandleType aHandle)
+{
+  if (!IsValid(aHandle)) {
+    return INVALID_HANDLE;
+  }
+  FileDescriptor::PlatformHandleType newHandle;
+#ifdef XP_WIN
+  if (::DuplicateHandle(GetCurrentProcess(), aHandle, GetCurrentProcess(),
+                        &newHandle, 0, FALSE, DUPLICATE_SAME_ACCESS)) {
+#else // XP_WIN
+  if ((newHandle = dup(aHandle)) != INVALID_HANDLE) {
+#endif
+        return newHandle;
+  }
+  NS_WARNING("Failed to duplicate file handle for current process!");
+  return INVALID_HANDLE;
+}
+
+// static
+void
+FileDescriptor::Close(PlatformHandleType aHandle)
+{
+  if (IsValid(aHandle)) {
+#ifdef XP_WIN
+    if (!CloseHandle(aHandle)) {
+      NS_WARNING("Failed to close file handle for current process!");
+    }
+#else // XP_WIN
+    HANDLE_EINTR(close(aHandle));
+#endif
+  }
+}
+
+FileDescriptor::PlatformHandleHelper::PlatformHandleHelper(FileDescriptor::PlatformHandleType aHandle)
+  :mHandle(aHandle)
+{
+}
+
+FileDescriptor::PlatformHandleHelper::PlatformHandleHelper(std::nullptr_t)
+  :mHandle(INVALID_HANDLE)
+{
+}
+
+bool
+FileDescriptor::PlatformHandleHelper::operator!=(std::nullptr_t) const
+{
+  return mHandle != INVALID_HANDLE;
+}
+
+FileDescriptor::PlatformHandleHelper::operator PlatformHandleType () const
+{
+  return mHandle;
+}
+
+#ifdef XP_WIN
+FileDescriptor::PlatformHandleHelper::operator std::intptr_t () const
+{
+  return reinterpret_cast<std::intptr_t>(mHandle);
+}
+#endif
+
+void
+FileDescriptor::PlatformHandleDeleter::operator()(FileDescriptor::PlatformHandleHelper aHelper)
+{
+  FileDescriptor::Close(aHelper);
+}
--- a/ipc/glue/FileDescriptor.h
+++ b/ipc/glue/FileDescriptor.h
@@ -4,18 +4,17 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef mozilla_ipc_FileDescriptor_h
 #define mozilla_ipc_FileDescriptor_h
 
 #include "base/basictypes.h"
 #include "base/process.h"
-#include "mozilla/DebugOnly.h"
-#include "nscore.h"
+#include "mozilla/UniquePtr.h"
 
 #ifdef XP_WIN
 // Need the HANDLE typedef.
 #include <winnt.h>
 #else
 #include "base/file_descriptor_posix.h"
 #endif
 
@@ -42,131 +41,100 @@ public:
 #ifdef XP_WIN
   typedef HANDLE PlatformHandleType;
   typedef HANDLE PickleType;
 #else
   typedef int PlatformHandleType;
   typedef base::FileDescriptor PickleType;
 #endif
 
+  struct PlatformHandleHelper
+  {
+    MOZ_IMPLICIT PlatformHandleHelper(PlatformHandleType aHandle);
+    MOZ_IMPLICIT PlatformHandleHelper(std::nullptr_t);
+    bool operator != (std::nullptr_t) const;
+    operator PlatformHandleType () const;
+#ifdef XP_WIN
+    operator std::intptr_t () const;
+#endif
+  private:
+    PlatformHandleType mHandle;
+  };
+  struct PlatformHandleDeleter
+  {
+    typedef PlatformHandleHelper pointer;
+    void operator () (PlatformHandleHelper aHelper);
+  };
+  typedef UniquePtr<PlatformHandleType, PlatformHandleDeleter> UniquePlatformHandle;
+
   // This should only ever be created by IPDL.
   struct IPDLPrivate
   {};
 
+  // Represents an invalid handle.
   FileDescriptor();
 
-  FileDescriptor(const FileDescriptor& aOther)
-    : mHandleCreatedByOtherProcess(false)
-#ifdef DEBUG
-    , mHandleCreatedByOtherProcessWasUsed(false)
-#endif
-  {
-    // Don't use operator= here because that will call
-    // CloseCurrentProcessHandle() on this (uninitialized) object.
-    Assign(aOther);
-  }
+  // Copy constructor will duplicate a new handle.
+  FileDescriptor(const FileDescriptor& aOther);
 
+  FileDescriptor(FileDescriptor&& aOther);
+
+  // This constructor will duplicate a new handle.
+  // The caller still have to close aHandle.
   explicit FileDescriptor(PlatformHandleType aHandle);
 
-  FileDescriptor(const IPDLPrivate&, const PickleType& aPickle)
-#ifdef XP_WIN
-  : mHandle(aPickle)
-#else
-  : mHandle(aPickle.fd)
-#endif
-  , mHandleCreatedByOtherProcess(true)
-#ifdef DEBUG
-  , mHandleCreatedByOtherProcessWasUsed(false)
-#endif
-  { }
+  // This constructor WILL NOT duplicate the handle.
+  // FileDescriptor takes the ownership from IPC message.
+  FileDescriptor(const IPDLPrivate&, const PickleType& aPickle);
 
-  ~FileDescriptor()
-  {
-    CloseCurrentProcessHandle();
-  }
+  ~FileDescriptor();
 
   FileDescriptor&
-  operator=(const FileDescriptor& aOther)
-  {
-    CloseCurrentProcessHandle();
-    Assign(aOther);
-    return *this;
-  }
+  operator=(const FileDescriptor& aOther);
+
+  FileDescriptor&
+  operator=(FileDescriptor&& aOther);
 
   // Performs platform-specific actions to duplicate mHandle in the other
   // process (e.g. dup() on POSIX, DuplicateHandle() on Windows). Returns a
   // pickled value that can be passed to the other process via IPC.
   PickleType
   ShareTo(const IPDLPrivate&, ProcessId aTargetPid) const;
 
   // Tests mHandle against a well-known invalid platform-specific file handle
   // (e.g. -1 on POSIX, INVALID_HANDLE_VALUE on Windows).
   bool
-  IsValid() const
-  {
-    return IsValid(mHandle);
-  }
+  IsValid() const;
 
-  PlatformHandleType
-  PlatformHandle() const
-  {
-#ifdef DEBUG
-    if (mHandleCreatedByOtherProcess) {
-      mHandleCreatedByOtherProcessWasUsed = true;
-    }
-#endif
-    return mHandle;
-  }
+  // Returns a duplicated handle, it is caller's responsibility to close the
+  // handle.
+  UniquePlatformHandle
+  ClonePlatformHandle() const;
 
+  // Only used in nsTArray.
   bool
-  operator==(const FileDescriptor& aOther) const
-  {
-    return mHandle == aOther.mHandle;
-  }
+  operator==(const FileDescriptor& aOther) const;
 
 private:
+  friend struct PlatformHandleTrait;
+
   void
-  Assign(const FileDescriptor& aOther)
-  {
-    if (aOther.mHandleCreatedByOtherProcess) {
-      mHandleCreatedByOtherProcess = true;
-#ifdef DEBUG
-      mHandleCreatedByOtherProcessWasUsed =
-        aOther.mHandleCreatedByOtherProcessWasUsed;
-#endif
-      mHandle = aOther.PlatformHandle();
-    } else {
-      DuplicateInCurrentProcess(aOther.PlatformHandle());
-      mHandleCreatedByOtherProcess = false;
-#ifdef DEBUG
-      mHandleCreatedByOtherProcessWasUsed = false;
-#endif
-    }
-  }
+  Assign(const FileDescriptor& aOther);
+
+  void
+  Close();
 
   static bool
   IsValid(PlatformHandleType aHandle);
 
-  void
-  DuplicateInCurrentProcess(PlatformHandleType aHandle);
+  static PlatformHandleType
+  Clone(PlatformHandleType aHandle);
 
-  void
-  CloseCurrentProcessHandle();
+  static void
+  Close(PlatformHandleType aHandle);
 
   PlatformHandleType mHandle;
-
-  // If this is true then this instance is created by IPDL to ferry a handle to
-  // its eventual consumer and we never close the handle. If this is false then
-  // we are a RAII wrapper around the handle and we close the handle on
-  // destruction.
-  bool mHandleCreatedByOtherProcess;
-
-#ifdef DEBUG
-  // This is to ensure that we don't leak the handle (which is only possible
-  // when we're in the receiving process).
-  mutable bool mHandleCreatedByOtherProcessWasUsed;
-#endif
 };
 
 } // namespace ipc
 } // namespace mozilla
 
 #endif // mozilla_ipc_FileDescriptor_h