Bug 1262102 - Part 4. Load ICU data from APK on demand. r?glandium draft
authorMakoto Kato <m_kato@ga2.so-net.ne.jp>
Tue, 05 Jul 2016 17:34:20 +0900
changeset 387013 d73bd703f36dcf439968c97c97e439af2df855b6
parent 387012 517e1645db17314c79ea5ce05eaeca9f904090a1
child 525281 a81a38730f3ff23ab22a9631bb9d6391ffbab4c5
push id22886
push userbmo:m_kato@ga2.so-net.ne.jp
push dateWed, 13 Jul 2016 09:46:55 +0000
reviewersglandium
bugs1262102
milestone50.0a1
Bug 1262102 - Part 4. Load ICU data from APK on demand. r?glandium Add new class to load compressed asset that is ICU data file. But, when running on unit test such as xpcshell, it will run on local executable binary instead of Android app. If no APK, use GRE path for ICU data directory. MozReview-Commit-ID: GPQM76ZnfHY
mozglue/android/APKOpen.cpp
mozglue/linker/AssetLoader.cpp
mozglue/linker/AssetLoader.h
mozglue/linker/ElfLoader.cpp
mozglue/linker/moz.build
xpcom/build/XPCOMInit.cpp
xpcom/build/moz.build
--- a/mozglue/android/APKOpen.cpp
+++ b/mozglue/android/APKOpen.cpp
@@ -26,16 +26,17 @@
 #include "APKOpen.h"
 #include <sys/time.h>
 #include <sys/resource.h>
 #include <sys/prctl.h>
 #include "sqlite3.h"
 #include "SQLiteBridge.h"
 #include "NSSBridge.h"
 #include "ElfLoader.h"
+#include "AssetLoader.h"
 #include "application.ini.h"
 
 #include "mozilla/TimeStamp.h"
 #include "mozilla/UniquePtr.h"
 #include "XREChildData.h"
 
 /* Android headers don't define RUSAGE_THREAD */
 #ifndef RUSAGE_THREAD
@@ -240,16 +241,19 @@ loadGeckoLibs(const char *apkName)
                       RUSAGE_TIMEDIFF(usage1, usage2, utime),
                       RUSAGE_TIMEDIFF(usage1_thread, usage2_thread, stime),
                       RUSAGE_TIMEDIFF(usage1, usage2, stime),
                       usage2_thread.ru_majflt - usage1_thread.ru_majflt,
                       usage2.ru_majflt - usage1.ru_majflt);
 
   XRE_StartupTimelineRecord(LINKER_INITIALIZED, t0);
   XRE_StartupTimelineRecord(LIBRARIES_LOADED, t1);
+
+  AssetLoader::Singleton.SetApkFile(apkName);
+
   return SUCCESS;
 }
 
 static mozglueresult loadNSSLibs(const char *apkName);
 
 static mozglueresult
 loadSQLiteLibs(const char *apkName)
 {
new file mode 100644
--- /dev/null
+++ b/mozglue/linker/AssetLoader.cpp
@@ -0,0 +1,130 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "AssetLoader.h"
+#include "ElfLoader.h"
+#include "Mappable.h"
+#include "Logging.h"
+#include "Zip.h"
+
+AssetLoader AssetLoader::Singleton;
+
+void*
+OpenAsset(const char* aPath)
+{
+  return AssetLoader::Singleton.Load(aPath);
+}
+
+void*
+AssetLoader::Load(const char* aPath)
+{
+  Mappable* mappable = GetMappableFromPath(aPath);
+  if (!mappable) {
+    return nullptr;
+  }
+  MemoryRange range(mappable->mmap(nullptr, mappable->GetLength(), PROT_READ,
+                                   MAP_PRIVATE, 0),
+                    mappable->GetLength());
+  mappable->finalize();
+
+  DEBUG_LOG("AssetLoader::Load(\"%s\") maps %p length: %u",
+            aPath, range.get(), range.GetLength());
+
+  AssetHandle handle(mappable, range);
+  mHandles.push_back(handle);
+
+  // load 1st page that asset will have a header
+  mappable->ensure(range.get());
+
+  const char *ondemand = getenv("MOZ_LINKER_ONDEMAND");
+
+  if (!ElfLoader::Singleton.hasRegisteredHandler() ||
+      (ondemand && !strncmp(ondemand, "0", 2 /* Including '\0' */))) {
+    for (uintptr_t off = PageSize(); off < range.GetLength();
+         off += PageSize()) {
+      mappable->ensure(reinterpret_cast<char *>(range.get()) + off);
+    }
+  }
+
+  return range.get();
+}
+
+void
+AssetLoader::Unload(void* aBaseAddress)
+{
+  for (auto iter = mHandles.begin(); iter < mHandles.end(); ++iter) {
+    if (iter->GetBaseAddress() == aBaseAddress) {
+      mHandles.erase(iter);
+      return;
+    }
+  }
+}
+
+AssetHandle*
+AssetLoader::GetAssetHandleByPtr(void* aAddr)
+{
+  DEBUG_LOG("AssetLoader::GetAssetHandleByPtr(%p)", aAddr);
+
+  for (auto iter = mHandles.begin(); iter < mHandles.end(); ++iter) {
+    if (iter->Contains(aAddr)) {
+      return &(*iter);
+    }
+  }
+  return nullptr;
+}
+
+/**
+ * Returns the part after the last '/' for the given path
+ */
+static const char *
+LeafName(const char *path)
+{
+  const char *lastSlash = strrchr(path, '/');
+  if (lastSlash) {
+    return lastSlash + 1;
+  }
+  return path;
+}
+
+Mappable*
+AssetLoader::GetMappableFromPath(const char* aPath)
+{
+  if (!mApkPath) {
+    // On unit test, APK isn't used
+    DEBUG_LOG("No APK filename");
+    return nullptr;
+  }
+
+  Mappable *mappable = nullptr;
+  RefPtr<Zip> zip;
+  zip = ZipCollection::GetZip(mApkPath);
+  if (!zip) {
+    ERROR("Cannot open %s", mApkPath);
+    return nullptr;
+  }
+  Zip::Stream s;
+  if (!zip->GetStream(aPath, &s)) {
+    ERROR("Cannot find %s from %s", aPath, mApkPath);
+    return nullptr;
+  }
+
+  const char *name = LeafName(aPath);
+  const char *extract = getenv("MOZ_LINKER_EXTRACT");
+  if (extract && !strncmp(extract, "1", 2 /* Including '\0' */)) {
+    mappable = MappableExtractFile::Create(name, zip, &s);
+  }
+
+  if (!mappable) {
+    if (s.GetType() == Zip::Stream::DEFLATE) {
+      mappable = MappableDeflate::Create(name, zip, &s);
+    } else if (s.GetType() == Zip::Stream::STORE) {
+      mappable = MappableSeekableZStream::Create(name, zip, &s);
+    }
+  }
+
+  DEBUG_LOG("AssetLoader::GetMappableFromPath(\"%s\") from \"%s\" returns %p",
+      aPath, mApkPath, mappable);
+
+  return mappable;
+}
new file mode 100644
--- /dev/null
+++ b/mozglue/linker/AssetLoader.h
@@ -0,0 +1,82 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef AssetLoader_h
+#define AssetLoader_h
+
+#include <vector>
+#include <string.h>
+
+#include "mozilla/Types.h"
+#include "mozilla/RefPtr.h"
+#include "Mappable.h"
+#include "Utils.h"
+
+extern "C" MFBT_API void*
+OpenAsset(const char* aPath);
+
+class AssetHandle final
+{
+public:
+  AssetHandle(Mappable* aMappable, const MemoryRange& aBase)
+    : mMappable(aMappable), mBase(aBase)
+  {
+  }
+
+  bool Contains(void* aAddr) const {
+    return mBase.Contains(aAddr);
+  }
+
+  Mappable* GetMappable() const {
+    return mMappable.get();
+  }
+
+  void* GetBaseAddress() const {
+    return mBase.get();
+  }
+
+private:
+  RefPtr<Mappable> mMappable;
+  MemoryRange mBase;
+};
+
+
+/*
+ * AssetLoader can extract compressed assst data into apk to memroy.
+ * AssetManager into Android SDK cannot support compressed asset.
+ * If asset is compressed by szip, it will be extracted by signal handler.
+ */
+class AssetLoader final
+{
+public:
+   static AssetLoader Singleton;
+
+   AssetLoader() : mApkPath(nullptr)
+   {
+   }
+
+   void* Load(const char* aPath);
+   void Unload(void* aBaseAddress);
+   AssetHandle* GetAssetHandleByPtr(void* aAddr);
+
+   void SetApkFile(const char* aApkPath)
+   {
+     mApkPath = strdup(aApkPath);
+   }
+
+private:
+   ~AssetLoader()
+   {
+     if (mApkPath) {
+       free(mApkPath);
+     }
+   }
+
+   Mappable* GetMappableFromPath(const char* aPath);
+
+   std::vector<AssetHandle> mHandles;
+   char* mApkPath;
+};
+
+#endif
--- a/mozglue/linker/ElfLoader.cpp
+++ b/mozglue/linker/ElfLoader.cpp
@@ -9,16 +9,17 @@
 #include <dlfcn.h>
 #include <unistd.h>
 #include <algorithm>
 #include <fcntl.h>
 #include "ElfLoader.h"
 #include "BaseElf.h"
 #include "CustomElf.h"
 #include "Mappable.h"
+#include "AssetLoader.h"
 #include "Logging.h"
 #include <inttypes.h>
 
 #if defined(ANDROID)
 #include <sys/syscall.h>
 
 #include <android/api-level.h>
 #if __ANDROID_API__ < 8
@@ -1213,16 +1214,24 @@ void SEGVHandler::handler(int signum, si
       ElfLoader::Singleton.GetHandleByPtr(info->si_addr);
     BaseElf *elf;
     if (handle && (elf = handle->AsBaseElf())) {
       DEBUG_LOG("Within the address space of %s", handle->GetPath());
       if (elf->mappable && elf->mappable->ensure(info->si_addr)) {
         return;
       }
     }
+    AssetHandle* assetHandle =
+      AssetLoader::Singleton.GetAssetHandleByPtr(info->si_addr);
+    if (assetHandle) {
+      DEBUG_LOG("Within the address space for asset");
+      if (assetHandle->GetMappable()->ensure(info->si_addr)) {
+        return;
+      }
+    }
   }
 
   /* Redispatch to the registered handler */
   SEGVHandler &that = ElfLoader::Singleton;
   if (that.action.sa_flags & SA_SIGINFO) {
     DEBUG_LOG("Redispatching to registered handler @%p",
               FunctionPtr(that.action.sa_sigaction));
     that.action.sa_sigaction(signum, info, context);
--- a/mozglue/linker/moz.build
+++ b/mozglue/linker/moz.build
@@ -1,15 +1,16 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 SOURCES += [
+    'AssetLoader.cpp',
     'BaseElf.cpp',
     'CustomElf.cpp',
     'ElfLoader.cpp',
     'Mappable.cpp',
     'SeekableZStream.cpp',
     'Zip.cpp',
 ]
 
--- a/xpcom/build/XPCOMInit.cpp
+++ b/xpcom/build/XPCOMInit.cpp
@@ -155,17 +155,23 @@ extern nsresult nsStringInputStreamConst
 
 #include "jsapi.h"
 #include "js/Initialization.h"
 
 #include "gfxPlatform.h"
 
 #if EXPOSE_INTL_API
 #include "unicode/putil.h"
+#if defined(MOZ_WIDGET_ANDROID)
+#include "unicode/udata.h"
+
+extern "C" MFBT_API void* OpenAsset(const char* aPath);
 #endif
+#endif
+
 
 using namespace mozilla;
 using base::AtExitManager;
 using mozilla::ipc::BrowserProcessSubThread;
 
 namespace {
 
 static AtExitManager* sExitManager;
@@ -689,24 +695,33 @@ NS_InitXPCOM2(nsIServiceManager** aResul
   // And for libnestegg.
   // libnestegg expects that its realloc implementation will free
   // the pointer argument when a size of 0 is passed in, so we need
   // the special version of the counting realloc.
   nestegg_set_halloc_func(NesteggReporter::CountingFreeingRealloc);
 #endif
 
 #if EXPOSE_INTL_API && defined(MOZ_ICU_DATA_ARCHIVE)
-  nsCOMPtr<nsIFile> greDir;
-  nsDirectoryService::gService->Get(NS_GRE_DIR,
-                                    NS_GET_IID(nsIFile),
-                                    getter_AddRefs(greDir));
-  MOZ_ASSERT(greDir);
-  nsAutoCString nativeGREPath;
-  greDir->GetNativePath(nativeGREPath);
-  u_setDataDirectory(nativeGREPath.get());
+#if defined(MOZ_WIDGET_ANDROID)
+  void* assetData = OpenAsset("assets/" ANDROID_CPU_ARCH "/" ICU_DATA_FILE);
+  if (assetData) {
+    UErrorCode err = U_ZERO_ERROR;
+    udata_setCommonData(assetData, &err);
+  } else
+#endif // MOZ_WIDGET_ANDROID
+  {
+    nsCOMPtr<nsIFile> greDir;
+    nsDirectoryService::gService->Get(NS_GRE_DIR,
+                                      NS_GET_IID(nsIFile),
+                                      getter_AddRefs(greDir));
+    MOZ_ASSERT(greDir);
+    nsAutoCString nativeGREPath;
+    greDir->GetNativePath(nativeGREPath);
+    u_setDataDirectory(nativeGREPath.get());
+  }
 #endif
 
   // Initialize the JS engine.
   const char* jsInitFailureReason = JS_InitWithFailureDiagnostic();
   if (jsInitFailureReason) {
     NS_RUNTIMEABORT(jsInitFailureReason);
   }
 
--- a/xpcom/build/moz.build
+++ b/xpcom/build/moz.build
@@ -96,10 +96,18 @@ LOCAL_INCLUDES += [
 if CONFIG['MOZ_VPX']:
     LOCAL_INCLUDES += [
         '/media/libvpx',
     ]
 
 if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'cocoa':
     CXXFLAGS += CONFIG['TK_CFLAGS']
 
+if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'android':
+    if CONFIG['ICU_DATA_FILE']:
+        DEFINES['ICU_DATA_FILE'] = '"%s"' % CONFIG['ICU_DATA_FILE']
+        DEFINES['ANDROID_CPU_ARCH'] = '"%s"' % CONFIG['ANDROID_CPU_ARCH']
+    LOCAL_INCLUDES += [
+        '/mozglue/linker',
+    ]
+
 CXXFLAGS += CONFIG['MOZ_ICU_CFLAGS']
 LOCAL_INCLUDES += CONFIG['MOZ_ICU_INCLUDES']