Bug 1475899: Part 3 - Add helper for parsing memory mappings on Linux. r?erahm,jld draft
authorKris Maglione <maglione.k@gmail.com>
Sat, 14 Jul 2018 02:20:55 -0700
changeset 819055 42c69cdbb25b62681d35a1bbf9658f0f675093f4
parent 819054 a2395487770e1362b5bb96ff47d64391d4caf121
child 819056 3032865f9499efc656e8a4c9a879033d17a3a042
child 819390 4a8cb980ca2dac89ac6198117af0d0b93b1f81cb
push id116423
push usermaglione.k@gmail.com
push dateTue, 17 Jul 2018 01:54:16 +0000
reviewerserahm, jld
bugs1475899
milestone63.0a1
Bug 1475899: Part 3 - Add helper for parsing memory mappings on Linux. r?erahm,jld The only real way to get memory mapping information on Linux is to read and parse /proc/self/maps, and infer information about a specific pointer based on its contents. This patch adds a helper which parses the entire contents of smaps, and returns an array of objects describing each region. It also updates resident unique reporter to use that helper. Later patches use it to map stack base addresses to VM regions, and report their usage data. MozReview-Commit-ID: 8VUu1kMT77L
xpcom/base/MemoryMapping.cpp
xpcom/base/MemoryMapping.h
xpcom/base/moz.build
xpcom/base/nsMemoryReporterManager.cpp
new file mode 100644
--- /dev/null
+++ b/xpcom/base/MemoryMapping.cpp
@@ -0,0 +1,211 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/MemoryMapping.h"
+
+#include "mozilla/BinarySearch.h"
+#include "mozilla/FileUtils.h"
+#include "mozilla/Scoped.h"
+
+#include <stdio.h>
+#include <string.h>
+
+namespace mozilla {
+
+MOZ_TYPE_SPECIFIC_SCOPED_POINTER_TEMPLATE(ScopedCharArray, char, free);
+
+namespace {
+struct VMFlagString
+{
+  const char* mName;
+  const char* mPrettyName;
+  VMFlag mFlag;
+};
+
+static const VMFlagString sVMFlagStrings[] = {
+  {"ac", "Accountable",   VMFlag::Accountable},
+  {"ar", "ArchSpecific",  VMFlag::ArchSpecific},
+  {"dc", "NoFork",        VMFlag::NoFork},
+  {"dd", "NoCore",        VMFlag::NoCore},
+  {"de", "NoExpand",      VMFlag::NoExpand},
+  {"dw", "DisabledWrite", VMFlag::DisabledWrite},
+  {"ex", "Executable",    VMFlag::Executable},
+  {"gd", "GrowsDown",     VMFlag::GrowsDown},
+  {"hg", "HugePage",      VMFlag::HugePage},
+  {"ht", "HugeTLB",       VMFlag::HugeTLB},
+  {"io", "IO",            VMFlag::IO},
+  {"lo", "Locked",        VMFlag::Locked},
+  {"me", "MayExecute",    VMFlag::MayExecute},
+  {"mg", "Mergeable",     VMFlag::Mergeable},
+  {"mm", "MixedMap",      VMFlag::MixedMap},
+  {"mr", "MayRead",       VMFlag::MayRead},
+  {"ms", "MayShare",      VMFlag::MayShare},
+  {"mw", "MayWrite",      VMFlag::MayWrite},
+  {"nh", "NoHugePage",    VMFlag::NoHugePage},
+  {"nl", "NonLinear",     VMFlag::NonLinear},
+  {"nr", "NotReserved",   VMFlag::NotReserved},
+  {"pf", "PurePFN",       VMFlag::PurePFN},
+  {"rd", "Readable",      VMFlag::Readable},
+  {"rr", "Random",        VMFlag::Random},
+  {"sd", "SoftDirty",     VMFlag::SoftDirty},
+  {"sh", "Shared",        VMFlag::Shared},
+  {"sr", "Sequential",    VMFlag::Sequential},
+  {"wr", "Writable",      VMFlag::Writable},
+};
+} // anonymous namespace
+
+constexpr size_t kVMFlags = size_t(-1);
+
+// An array of known field names which may be present in an smaps file, and the
+// offsets of the corresponding fields in a MemoryMapping class.
+const MemoryMapping::Field MemoryMapping::sFields[] = {
+  {"AnonHugePages",   offsetof(MemoryMapping, mAnonHugePages)},
+  {"Anonymous",       offsetof(MemoryMapping, mAnonymous)},
+  {"KernelPageSize",  offsetof(MemoryMapping, mKernelPageSize)},
+  {"LazyFree",        offsetof(MemoryMapping, mLazyFree)},
+  {"Locked",          offsetof(MemoryMapping, mLocked)},
+  {"MMUPageSize",     offsetof(MemoryMapping, mMMUPageSize)},
+  {"Private_Clean",   offsetof(MemoryMapping, mPrivate_Clean)},
+  {"Private_Dirty",   offsetof(MemoryMapping, mPrivate_Dirty)},
+  {"Private_Hugetlb", offsetof(MemoryMapping, mPrivate_Hugetlb)},
+  {"Pss",             offsetof(MemoryMapping, mPss)},
+  {"Referenced",      offsetof(MemoryMapping, mReferenced)},
+  {"Rss",             offsetof(MemoryMapping, mRss)},
+  {"Shared_Clean",    offsetof(MemoryMapping, mShared_Clean)},
+  {"Shared_Dirty",    offsetof(MemoryMapping, mShared_Dirty)},
+  {"Shared_Hugetlb",  offsetof(MemoryMapping, mShared_Hugetlb)},
+  {"ShmemPmdMapped",  offsetof(MemoryMapping, mShmemPmdMapped)},
+  {"Size",            offsetof(MemoryMapping, mSize)},
+  {"Swap",            offsetof(MemoryMapping, mSwap)},
+  {"SwapPss",         offsetof(MemoryMapping, mSwapPss)},
+  // VmFlags is a special case. It contains an array of flag strings, which
+  // describe attributes of the mapping, rather than a mapping size. We include
+  // it in this array to aid in parsing, but give it a separate sentinel value,
+  // and treat it specially.
+  {"VmFlags",         kVMFlags},
+};
+
+template <typename T, int n>
+const T*
+FindEntry(const char* aName, const T (&aEntries)[n])
+{
+  size_t index;
+  if (BinarySearchIf(aEntries, 0, n,
+                     [&] (const T& aEntry) {
+                       return strcmp(aName, aEntry.mName);
+                     },
+                     &index)) {
+    return &aEntries[index];
+  }
+  return nullptr;
+}
+
+using Perm = MemoryMapping::Perm;
+using PermSet = MemoryMapping::PermSet;
+
+nsresult
+GetMemoryMappings(nsTArray<MemoryMapping>& aMappings)
+{
+  ScopedCloseFile file(fopen("/proc/self/smaps", "r"));
+  if (NS_WARN_IF(!file)) {
+    return NS_ERROR_UNEXPECTED;
+  }
+
+  // Note: We unfortunately can't just use UniquePtr<char[]> for this, because
+  // getline() will reallocate the buffer if it isn't large enough, and
+  // replace the value in our pointer. In the end, we need to free whatever its
+  // final value is. UniquePtr does not give us a way to handle this. Scoped
+  // does.
+  size_t bufferSize = 1024;
+  ScopedCharArray buffer(new char[bufferSize]);
+
+  MemoryMapping* current = nullptr;
+
+  while (getline(&buffer.rwget(), &bufferSize, file) > 0) {
+    size_t start, end, offset;
+    char flags[4] = "---";
+    char name[512];
+
+    name[0] = 0;
+
+    if (sscanf(buffer, "%zx-%zx %4c %zx %*u:%*u %*u %511s",
+               &start, &end, flags, &offset, name) >= 4) {
+      PermSet perms;
+      if (flags[0] == 'r') {
+        perms += Perm::Read;
+      }
+      if (flags[1] == 'w') {
+        perms += Perm::Write;
+      }
+      if (flags[2] == 'x') {
+        perms += Perm::Execute;
+      }
+      if (flags[3] == 'p') {
+        perms += Perm::Private;
+      } else if (flags[3] == 's') {
+        perms += Perm::Shared;
+      }
+
+      current = aMappings.AppendElement(MemoryMapping{start, end, perms, offset, name});
+      continue;
+    }
+    if (!current) {
+      continue;
+    }
+
+    char* line = buffer;
+    char* savePtr;
+    char* fieldName = strtok_r(line, ":", &savePtr);
+    if (!fieldName) {
+      continue;
+    }
+    auto* field = FindEntry(fieldName, MemoryMapping::sFields);
+    if (!field) {
+      continue;
+    }
+
+    if (field->mOffset == kVMFlags) {
+      while (char* flagName = strtok_r(nullptr, " \n", &savePtr)) {
+        if (auto* flag = FindEntry(flagName, sVMFlagStrings)) {
+          current->mFlags += flag->mFlag;
+        }
+      }
+      continue;
+    }
+
+    line = strtok_r(nullptr, "\n", &savePtr);
+    size_t value;
+    if (sscanf(line, "%zd kB", &value) > 0) {
+      current->ValueForField(*field) = value * 1024;
+    }
+  }
+
+  return NS_OK;
+}
+
+void
+MemoryMapping::Dump(nsACString& aOut) const
+{
+  aOut.AppendPrintf("%zx-%zx Size: %zu Offset: %zx %s\n",
+                    mStart, mEnd,
+                    mEnd - mStart,
+                    mOffset, mName.get());
+
+  for (auto& field : MemoryMapping::sFields) {
+    if (field.mOffset < sizeof(*this)) {
+      aOut.AppendPrintf("  %s: %zd\n", field.mName, ValueForField(field));
+    }
+  }
+
+  aOut.AppendPrintf("  Flags: %x\n", mFlags.serialize());
+  for (auto& flag : sVMFlagStrings) {
+    if (mFlags.contains(flag.mFlag)) {
+      aOut.AppendPrintf("       : %s %s\n", flag.mName, flag.mPrettyName);
+    }
+  }
+}
+
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/xpcom/base/MemoryMapping.h
@@ -0,0 +1,190 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_MemoryMapping_h
+#define mozilla_MemoryMapping_h
+
+#include "mozilla/EnumSet.h"
+#include "nsString.h"
+#include "nsTArray.h"
+
+/**
+ * MemoryMapping is a helper class which describes an entry in the Linux
+ * /proc/<pid>/smaps file. See procfs(5) for details on the entry format.
+ *
+ * The GetMemoryMappings() function returns an array of such entries, sorted by
+ * start address, one for each entry in the current process's address space.
+ */
+
+namespace mozilla {
+
+enum class VMFlag : uint8_t
+{
+  Readable,      // rd  - readable
+  Writable,      // wr  - writable
+  Executable,    // ex  - executable
+  Shared,        // sh  - shared
+  MayRead,       // mr  - may read
+  MayWrite,      // mw  - may write
+  MayExecute,    // me  - may execute
+  MayShare,      // ms  - may share
+  GrowsDown,     // gd  - stack segment grows down
+  PurePFN,       // pf  - pure PFN range
+  DisabledWrite, // dw  - disabled write to the mapped file
+  Locked,        // lo  - pages are locked in memory
+  IO,            // io  - memory mapped I/O area
+  Sequential,    // sr  - sequential read advise provided
+  Random,        // rr  - random read advise provided
+  NoFork,        // dc  - do not copy area on fork
+  NoExpand,      // de  - do not expand area on remapping
+  Accountable,   // ac  - area is accountable
+  NotReserved,   // nr  - swap space is not reserved for the area
+  HugeTLB,       // ht  - area uses huge tlb pages
+  NonLinear,     // nl  - non-linear mapping
+  ArchSpecific,  // ar  - architecture specific flag
+  NoCore,        // dd  - do not include area into core dump
+  SoftDirty,     // sd  - soft-dirty flag
+  MixedMap,      // mm  - mixed map area
+  HugePage,      // hg  - huge page advise flag
+  NoHugePage,    // nh  - no-huge page advise flag
+  Mergeable,     // mg  - mergeable advise flag
+};
+
+using VMFlagSet = EnumSet<VMFlag>;
+
+class MemoryMapping final
+{
+public:
+  enum class Perm : uint8_t
+  {
+    Read,
+    Write,
+    Execute,
+    Shared,
+    Private,
+  };
+
+  using PermSet = EnumSet<Perm>;
+
+  MemoryMapping(uintptr_t aStart, uintptr_t aEnd,
+                PermSet aPerms, size_t aOffset,
+                const char* aName)
+    : mStart(aStart)
+    , mEnd(aEnd)
+    , mOffset(aOffset)
+    , mName(aName)
+    , mPerms(aPerms)
+  {}
+
+  const nsCString& Name() const { return mName; }
+
+  uintptr_t Start() const { return mStart; }
+  uintptr_t End() const { return mEnd; }
+
+  bool Includes(const void* aPtr) const
+  {
+    auto ptr = uintptr_t(aPtr);
+    return ptr >= mStart && ptr < mEnd;
+  }
+
+  PermSet Perms() const { return mPerms; }
+  VMFlagSet VMFlags() const { return mFlags; }
+
+  // For file mappings, the offset in the mapped file which corresponds to the
+  // start of the mapped region.
+  size_t Offset() const { return mOffset; }
+
+  size_t AnonHugePages() const { return mAnonHugePages; }
+  size_t Anonymous() const { return mAnonymous; }
+  size_t KernelPageSize() const { return mKernelPageSize; }
+  size_t LazyFree() const { return mLazyFree; }
+  size_t Locked() const { return mLocked; }
+  size_t MMUPageSize() const { return mMMUPageSize; }
+  size_t Private_Clean() const { return mPrivate_Clean; }
+  size_t Private_Dirty() const { return mPrivate_Dirty; }
+  size_t Private_Hugetlb() const { return mPrivate_Hugetlb; }
+  size_t Pss() const { return mPss; }
+  size_t Referenced() const { return mReferenced; }
+  size_t Rss() const { return mRss; }
+  size_t Shared_Clean() const { return mShared_Clean; }
+  size_t Shared_Dirty() const { return mShared_Dirty; }
+  size_t Shared_Hugetlb() const { return mShared_Hugetlb; }
+  size_t ShmemPmdMapped() const { return mShmemPmdMapped; }
+  size_t Size() const { return mSize; }
+  size_t Swap() const { return mSwap; }
+  size_t SwapPss() const { return mSwapPss; }
+
+  // Dumps a string representation of the entry, similar to its format in the
+  // smaps file, to the given string. Mainly useful for debugging.
+  void Dump(nsACString& aOut) const;
+
+  // These comparison operators are used for binary searching sorted arrays of
+  // MemoryMapping entries to find the one which contains a given pointer.
+  bool operator==(const void* aPtr) const { return Includes(aPtr); }
+  bool operator<(const void* aPtr) const { return mStart < uintptr_t(aPtr); }
+
+private:
+  friend nsresult GetMemoryMappings(nsTArray<MemoryMapping>& aMappings);
+
+  uintptr_t mStart = 0;
+  uintptr_t mEnd = 0;
+
+  size_t mOffset = 0;
+
+  nsCString mName;
+
+  // Members for size fields in the smaps file. Please keep these in sync with
+  // the sFields array.
+  size_t mAnonHugePages = 0;
+  size_t mAnonymous = 0;
+  size_t mKernelPageSize = 0;
+  size_t mLazyFree = 0;
+  size_t mLocked = 0;
+  size_t mMMUPageSize = 0;
+  size_t mPrivate_Clean = 0;
+  size_t mPrivate_Dirty = 0;
+  size_t mPrivate_Hugetlb = 0;
+  size_t mPss = 0;
+  size_t mReferenced = 0;
+  size_t mRss = 0;
+  size_t mShared_Clean = 0;
+  size_t mShared_Dirty = 0;
+  size_t mShared_Hugetlb = 0;
+  size_t mShmemPmdMapped = 0;
+  size_t mSize = 0;
+  size_t mSwap = 0;
+  size_t mSwapPss = 0;
+
+  PermSet mPerms{};
+  VMFlagSet mFlags{};
+
+  // Contains the name and offset of one of the above size_t fields, for use in
+  // parsing in dumping. The below helpers contain a list of the fields, and map
+  // Field entries to the appropriate member in a class instance.
+  struct Field
+  {
+    const char* mName;
+    size_t mOffset;
+  };
+
+  static const Field sFields[20];
+
+  size_t& ValueForField(const Field& aField)
+  {
+    char* fieldPtr = reinterpret_cast<char*>(this) + aField.mOffset;
+    return reinterpret_cast<size_t*>(fieldPtr)[0];
+  }
+  size_t ValueForField(const Field& aField) const
+  {
+    return const_cast<MemoryMapping*>(this)->ValueForField(aField);
+  }
+};
+
+nsresult GetMemoryMappings(nsTArray<MemoryMapping>& aMappings);
+
+} // namespace mozilla
+
+#endif // mozilla_MemoryMapping_h
--- a/xpcom/base/moz.build
+++ b/xpcom/base/moz.build
@@ -107,16 +107,17 @@ EXPORTS.mozilla += [
     'DebuggerOnGCRunnable.h',
     'DeferredFinalize.h',
     'EnumeratedArrayCycleCollection.h',
     'ErrorNames.h',
     'HoldDropJSObjects.h',
     'IntentionalCrash.h',
     'JSObjectHolder.h',
     'Logging.h',
+    'MemoryMapping.h',
     'MemoryReportingProcess.h',
     'nsMemoryInfoDumper.h',
     'NSPRLogModulesParser.h',
     'OwningNonNull.h',
     'SizeOfState.h',
     'StaticMutex.h',
     'StaticPtr.h',
 ]
@@ -168,16 +169,21 @@ UNIFIED_SOURCES += [
     'nsSystemInfo.cpp',
     'nsTraceRefcnt.cpp',
     'nsUUIDGenerator.cpp',
     'nsVersionComparator.cpp',
     'nsVersionComparatorImpl.cpp',
     'nsWeakReference.cpp',
 ]
 
+if CONFIG['OS_TARGET'] == 'Linux':
+    UNIFIED_SOURCES += [
+        'MemoryMapping.cpp',
+    ]
+
 GENERATED_FILES += [
     "error_list.rs",
     "ErrorList.h",
     "ErrorNamesInternal.h",
 ]
 
 GENERATED_FILES["ErrorList.h"].script = "ErrorList.py:error_list_h"
 GENERATED_FILES["ErrorNamesInternal.h"].script = "ErrorList.py:error_names_internal_h"
--- a/xpcom/base/nsMemoryReporterManager.cpp
+++ b/xpcom/base/nsMemoryReporterManager.cpp
@@ -84,58 +84,25 @@ static MOZ_MUST_USE nsresult
 GetProcSelfSmapsPrivate(int64_t* aN)
 {
   // You might be tempted to calculate USS by subtracting the "shared" value
   // from the "resident" value in /proc/<pid>/statm. But at least on Linux,
   // statm's "shared" value actually counts pages backed by files, which has
   // little to do with whether the pages are actually shared. /proc/self/smaps
   // on the other hand appears to give us the correct information.
 
-  FILE* f = fopen("/proc/self/smaps", "r");
-  if (NS_WARN_IF(!f)) {
-    return NS_ERROR_UNEXPECTED;
-  }
-
-  // We carry over the end of the buffer to the beginning to make sure we only
-  // interpret complete lines.
-  static const uint32_t carryOver = 32;
-  static const uint32_t readSize = 4096;
+  nsTArray<MemoryMapping> mappings(1024);
+  MOZ_TRY(GetMemoryMappings(mappings));
 
   int64_t amount = 0;
-  char buffer[carryOver + readSize + 1];
-
-  // Fill the beginning of the buffer with spaces, as a sentinel for the first
-  // iteration.
-  memset(buffer, ' ', carryOver);
-
-  for (;;) {
-    size_t bytes = fread(buffer + carryOver, sizeof(*buffer), readSize, f);
-    char* end = buffer + bytes;
-    char* ptr = buffer;
-    end[carryOver] = '\0';
-    // We are looking for lines like "Private_{Clean,Dirty}: 4 kB".
-    while ((ptr = strstr(ptr, "Private"))) {
-      if (ptr >= end) {
-        break;
-      }
-      ptr += sizeof("Private_Xxxxx:");
-      amount += strtol(ptr, nullptr, 10);
-    }
-    if (bytes < readSize) {
-      // We do not expect any match within the end of the buffer.
-      MOZ_ASSERT(!strstr(end, "Private"));
-      break;
-    }
-    // Carry the end of the buffer over to the beginning.
-    memcpy(buffer, end, carryOver);
+  for (auto& mapping : mappings) {
+    amount += mapping.Private_Clean();
+    amount += mapping.Private_Dirty();
   }
-
-  fclose(f);
-  // Convert from kB to bytes.
-  *aN = amount * 1024;
+  *aN = amount;
   return NS_OK;
 }
 
 #define HAVE_VSIZE_AND_RESIDENT_REPORTERS 1
 static MOZ_MUST_USE nsresult
 VsizeDistinguishedAmount(int64_t* aN)
 {
   return GetProcSelfStatmField(0, aN);