Bug 1259355 - List sites using quota storage or appcache in Settings of Site Data, r?Fischer draft
authorFischer.json <fischer.json@gmail.com>
Tue, 11 Apr 2017 22:47:48 +0800
changeset 574617 f49b5e3b7581d262f731291edc04c3f2dc55715b
parent 571131 2e7c10a9b86e30691f67855f6c8f98d984508d7c
child 627656 395961974a56d8e24a0b33f15c971ea220634915
push id57771
push userbmo:fliu@mozilla.com
push dateTue, 09 May 2017 05:42:17 +0000
reviewersFischer
bugs1259355
milestone55.0a1
Bug 1259355 - List sites using quota storage or appcache in Settings of Site Data, r?Fischer MozReview-Commit-ID: 29zZTzOsC7c
browser/components/preferences/SiteDataManager.jsm
browser/components/preferences/siteDataSettings.js
browser/components/preferences/siteDataSettings.xul
--- a/browser/components/preferences/SiteDataManager.jsm
+++ b/browser/components/preferences/SiteDataManager.jsm
@@ -1,264 +1,257 @@
 "use strict";
 
 const { classes: Cc, interfaces: Ci, utils: Cu } = Components;
 
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
-Cu.import("resource://gre/modules/NetUtil.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "OfflineAppCacheHelper",
                                   "resource:///modules/offlineAppCache.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "ContextualIdentityService",
                                   "resource://gre/modules/ContextualIdentityService.jsm");
 
 this.EXPORTED_SYMBOLS = [
   "SiteDataManager"
 ];
 
 this.SiteDataManager = {
 
   _qms: Services.qms,
 
-  _diskCache: Services.cache2.diskCacheStorage(Services.loadContextInfo.default, false),
-
   _appCache: Cc["@mozilla.org/network/application-cache-service;1"].getService(Ci.nsIApplicationCacheService),
 
-  // A Map of sites using the persistent-storage API (have requested persistent-storage permission)
-  // Key is site's origin.
+  // A Map of sites and their disk usage according to Quota Manager and appcache
+  // Key is host (group sites based on host across scheme, port, origin atttributes).
   // Value is one object holding:
-  //   - perm: persistent-storage permision; instance of nsIPermission
-  //   - status: the permission granted/rejected status
+  //   - principals: instances of nsIPrincipal.
+  //   - persisted: the persistent-storage status.
   //   - quotaUsage: the usage of indexedDB and localStorage.
   //   - appCacheList: an array of app cache; instances of nsIApplicationCache
-  //   - diskCacheList: an array. Each element is object holding metadata of http cache:
-  //       - uri: the uri of that http cache
-  //       - dataSize: that http cache size
-  //       - idEnhance: the id extension of that http cache
   _sites: new Map(),
 
-  _updateQuotaPromise: null,
+  _getQuotaUsagePromise: null,
 
-  _updateDiskCachePromise: null,
-
-  _quotaUsageRequests: null,
+  _quotaUsageRequest: null,
 
   updateSites() {
     Services.obs.notifyObservers(null, "sitedatamanager:updating-sites");
 
     // Clear old data and requests first
     this._sites.clear();
-    this._cancelQuotaUpdate();
+    this._cancelGetQuotaUsage();
 
-    // Collect sites granted/rejected with the persistent-storage permission
-    let perm = null;
-    let status = null;
-    let e = Services.perms.enumerator;
-    while (e.hasMoreElements()) {
-      perm = e.getNext();
-      status = Services.perms.testExactPermissionFromPrincipal(perm.principal, "persistent-storage");
-      if (status === Ci.nsIPermissionManager.ALLOW_ACTION ||
-          status === Ci.nsIPermissionManager.DENY_ACTION) {
-        this._sites.set(perm.principal.URI.spec, {
-          perm,
-          status,
-          quotaUsage: 0,
-          appCacheList: [],
-          diskCacheList: []
+    this._getQuotaUsage()
+        .then(results => {
+          for (let result of results) {
+            let principal =
+              Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(result.origin);
+            let uri = principal.URI;
+            if (uri.scheme == "http" || uri.scheme == "https") {
+              let site = this._sites.get(uri.host);
+              if (!site) {
+                site = {
+                  persisted: false,
+                  quotaUsage: 0,
+                  principals: [],
+                  appCacheList: [],
+                };
+              }
+              // Assume 3 sites:
+              //   - Site A (not persisted): https://www.foo.com
+              //   - Site B (not persisted): https://www.foo.com^userContextId=2
+              //   - Site C (persisted):     https://www.foo.com:1234
+              // Although only C is persisted, grouping by host, as a result,
+              // we still mark as persisted here under this host group.
+              if (result.persisted) {
+                site.persisted = true;
+              }
+              site.principals.push(principal);
+              site.quotaUsage += result.usage;
+              this._sites.set(uri.host, site);
+            }
+          }
+          this._updateAppCache();
+          Services.obs.notifyObservers(null, "sitedatamanager:sites-updated", null);
         });
-      }
-    }
-
-    this._updateQuota();
-    this._updateAppCache();
-    this._updateDiskCache();
-
-    Promise.all([this._updateQuotaPromise, this._updateDiskCachePromise])
-           .then(() => {
-             Services.obs.notifyObservers(null, "sitedatamanager:sites-updated");
-           });
   },
 
-  _updateQuota() {
-    this._quotaUsageRequests = [];
-    let promises = [];
-    for (let site of this._sites.values()) {
-      promises.push(new Promise(resolve => {
-        let callback = {
-          onUsageResult(request) {
-            site.quotaUsage = request.result.usage;
-            resolve();
-          }
-        };
-        // XXX: The work of integrating localStorage into Quota Manager is in progress.
-        //      After the bug 742822 and 1286798 landed, localStorage usage will be included.
-        //      So currently only get indexedDB usage.
-        this._quotaUsageRequests.push(
-          this._qms.getUsageForPrincipal(site.perm.principal, callback));
-      }));
-    }
-    this._updateQuotaPromise = Promise.all(promises);
+  _getQuotaUsage() {
+    this._getQuotaUsagePromise = new Promise(resolve => {
+      let callback = {
+        onUsageResult(request) {
+          resolve(request.result);
+        }
+      };
+      // XXX: The work of integrating localStorage into Quota Manager is in progress.
+      //      After the bug 742822 and 1286798 landed, localStorage usage will be included.
+      //      So currently only get indexedDB usage.
+      this._quotaUsageRequest = this._qms.getUsage(callback);
+    });
+    return this._getQuotaUsagePromise;
   },
 
-  _cancelQuotaUpdate() {
-    if (this._quotaUsageRequests) {
-      for (let request of this._quotaUsageRequests) {
-        request.cancel();
-      }
-      this._quotaUsageRequests = null;
+  _cancelGetQuotaUsage() {
+    if (this._quotaUsageRequest) {
+      this._quotaUsageRequest.cancel();
+      this._quotaUsageRequest = null;
     }
   },
 
   _updateAppCache() {
-    let groups = null;
-    try {
-      groups =  this._appCache.getGroups();
-    } catch (e) {
-      return;
-    }
-
-    for (let site of this._sites.values()) {
-      for (let group of groups) {
-        let uri = Services.io.newURI(group);
-        if (site.perm.matchesURI(uri, true)) {
-          let cache = this._appCache.getActiveCache(group);
-          site.appCacheList.push(cache);
-        }
+    let groups = this._appCache.getGroups();
+    for (let group of groups) {
+      let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(group);
+      let uri = principal.URI;
+      let site = this._sites.get(uri.host);
+      if (!site) {
+        site = {
+          persisted: false,
+          quotaUsage: 0,
+          principals: [ principal ],
+          appCacheList: [],
+        };
+        this._sites.set(uri.host, site);
+      } else if (!site.principals.some(p => p.origin == principal.origin)) {
+        site.principals.push(principal);
       }
+      let cache = this._appCache.getActiveCache(group);
+      site.appCacheList.push(cache);
     }
   },
 
-  _updateDiskCache() {
-    this._updateDiskCachePromise = new Promise(resolve => {
-      if (this._sites.size) {
-        let sites = this._sites;
-        let visitor = {
-          onCacheEntryInfo(uri, idEnhance, dataSize) {
-            for (let site of sites.values()) {
-              if (site.perm.matchesURI(uri, true)) {
-                site.diskCacheList.push({
-                  uri,
-                  dataSize,
-                  idEnhance
-                });
-                break;
-              }
-            }
-          },
-          onCacheEntryVisitCompleted() {
-            resolve();
-          }
-        };
-        this._diskCache.asyncVisitStorage(visitor, true);
-      } else {
-        resolve();
-      }
-    });
-  },
-
   getTotalUsage() {
-    return Promise.all([this._updateQuotaPromise, this._updateDiskCachePromise])
-                  .then(() => {
-                    let usage = 0;
-                    for (let site of this._sites.values()) {
-                      let cache = null;
-                      for (cache of site.appCacheList) {
-                        usage += cache.usage;
-                      }
-                      for (cache of site.diskCacheList) {
-                        usage += cache.dataSize;
-                      }
-                      usage += site.quotaUsage;
-                    }
-                    return usage;
-                  });
+    return this._getQuotaUsagePromise.then(() => {
+      let usage = 0;
+      for (let site of this._sites.values()) {
+        for (let cache of site.appCacheList) {
+          usage += cache.usage;
+        }
+        usage += site.quotaUsage;
+      }
+      return usage;
+    });
   },
 
   getSites() {
-    return Promise.all([this._updateQuotaPromise, this._updateDiskCachePromise])
-                  .then(() => {
-                    let list = [];
-                    for (let [origin, site] of this._sites) {
-                      let cache = null;
-                      let usage = site.quotaUsage;
-                      for (cache of site.appCacheList) {
-                        usage += cache.usage;
-                      }
-                      for (cache of site.diskCacheList) {
-                        usage += cache.dataSize;
-                      }
-                      list.push({
-                        usage,
-                        status: site.status,
-                        uri: NetUtil.newURI(origin)
-                      });
-                    }
-                    return list;
-                  });
+    return this._getQuotaUsagePromise.then(() => {
+      let list = [];
+      for (let [host, site] of this._sites) {
+        let usage = site.quotaUsage;
+        for (let cache of site.appCacheList) {
+          usage += cache.usage;
+        }
+        list.push({
+          host,
+          usage,
+          persisted: site.persisted
+        });
+      }
+      return list;
+    });
   },
 
   _removePermission(site) {
-    Services.perms.removePermission(site.perm);
+    let removals = new Set();
+    for (let principal of site.principals) {
+      let { originNoSuffix } = principal;
+      if (removals.has(originNoSuffix)) {
+        // In case of encountering
+        //   - https://www.foo.com
+        //   - https://www.foo.com^userContextId=2
+        // because setting/removing permission is across OAs already so skip the same origin without suffix
+        continue;
+      }
+      removals.add(originNoSuffix);
+      Services.perms.removeFromPrincipal(principal, "persistent-storage");
+    }
   },
 
   _removeQuotaUsage(site) {
-    this._qms.clearStoragesForPrincipal(site.perm.principal, null, true);
-  },
-
-  _removeDiskCache(site) {
-    for (let cache of site.diskCacheList) {
-      this._diskCache.asyncDoomURI(cache.uri, cache.idEnhance, null);
+    let promises = [];
+    let removals = new Set();
+    for (let principal of site.principals) {
+      let { originNoSuffix } = principal;
+      if (removals.has(originNoSuffix)) {
+        // In case of encountering
+        //   - https://www.foo.com
+        //   - https://www.foo.com^userContextId=2
+        // below we have already removed across OAs so skip the same origin without suffix
+        continue;
+      }
+      removals.add(originNoSuffix);
+      promises.push(new Promise(resolve => {
+        // We are clearing *All* across OAs so need to ensure a principal without suffix here,
+        // or the call of `clearStoragesForPrincipal` would fail.
+        principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(originNoSuffix);
+        let request = this._qms.clearStoragesForPrincipal(principal, null, true);
+        request.callback = resolve;
+      }));
     }
+    return Promise.all(promises);
   },
 
   _removeAppCache(site) {
     for (let cache of site.appCacheList) {
       cache.discard();
     }
   },
 
   _removeCookie(site) {
-    let host = site.perm.principal.URI.host;
-    let e = Services.cookies.getCookiesFromHost(host, {});
-    while (e.hasMoreElements()) {
-      let cookie = e.getNext();
-      if (cookie instanceof Components.interfaces.nsICookie) {
-        if (this.isPrivateCookie(cookie)) {
-          continue;
+    for (let principal of site.principals) {
+      // Although `getCookiesFromHost` can get cookies across hosts under the same base domain, OAs matter.
+      // We still need OAs here.
+      let e = Services.cookies.getCookiesFromHost(principal.URI.host, principal.originAttributes);
+      while (e.hasMoreElements()) {
+        let cookie = e.getNext();
+        if (cookie instanceof Components.interfaces.nsICookie) {
+          if (this.isPrivateCookie(cookie)) {
+            continue;
+          }
+          Services.cookies.remove(
+            cookie.host, cookie.name, cookie.path, false, cookie.originAttributes);
         }
-        Services.cookies.remove(
-          cookie.host, cookie.name, cookie.path, false, cookie.originAttributes);
       }
     }
   },
 
-  remove(uris) {
-    for (let uri of uris) {
-      let site = this._sites.get(uri.spec);
+  remove(hosts) {
+    let promises = [];
+    let unknownHost = "";
+    for (let host of hosts) {
+      let site = this._sites.get(host);
       if (site) {
         this._removePermission(site);
-        this._removeQuotaUsage(site);
-        this._removeDiskCache(site);
         this._removeAppCache(site);
         this._removeCookie(site);
+        promises.push(this._removeQuotaUsage(site));
+      } else {
+        unknownHost = host;
+        break;
       }
     }
-    this.updateSites();
+    if (promises.length > 0) {
+      Promise.all(promises).then(() => this.updateSites());
+    }
+    if (unknownHost) {
+      throw `SiteDataManager: removing unknown site of ${unknownHost}`;
+    }
   },
 
   removeAll() {
+    let promises = [];
     for (let site of this._sites.values()) {
       this._removePermission(site);
-      this._removeQuotaUsage(site);
+      promises.push(this._removeQuotaUsage(site));
     }
     Services.cache2.clear();
     Services.cookies.removeAll();
     OfflineAppCacheHelper.clear();
-    this.updateSites();
+    Promise.all(promises).then(() => this.updateSites());
   },
 
   isPrivateCookie(cookie) {
     let { userContextId } = cookie.originAttributes;
     // A private cookie is when its userContextId points to a private identity.
     return userContextId && !ContextualIdentityService.getPublicIdentityFromId(userContextId);
   }
 };
--- a/browser/components/preferences/siteDataSettings.js
+++ b/browser/components/preferences/siteDataSettings.js
@@ -11,17 +11,17 @@ XPCOMUtils.defineLazyModuleGetter(this, 
                                   "resource:///modules/SiteDataManager.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "DownloadUtils",
                                   "resource://gre/modules/DownloadUtils.jsm");
 
 "use strict";
 
 let gSiteDataSettings = {
 
-  // Array of meatdata of sites. Each array element is object holding:
+  // Array of metadata of sites. Each array element is object holding:
   // - uri: uri of site; instance of nsIURI
   // - status: persistent-storage permission status
   // - usage: disk usage which site uses
   // - userAction: "remove" or "update-permission"; the action user wants to take.
   //               If not specified, means no action to take
   _sites: null,
 
   _list: null,
@@ -34,17 +34,17 @@ let gSiteDataSettings = {
               .addEventListener(eventType, callback.bind(gSiteDataSettings));
     }
 
     this._list = document.getElementById("sitesList");
     this._searchBox = document.getElementById("searchBox");
     this._prefStrBundle = document.getElementById("bundlePreferences");
     SiteDataManager.getSites().then(sites => {
       this._sites = sites;
-      let sortCol = document.getElementById("hostCol");
+      let sortCol = document.querySelector("treecol[data-isCurrentSortCol=true]");
       this._sortSites(this._sites, sortCol);
       this._buildSitesList(this._sites);
       Services.obs.notifyObservers(null, "sitedata-settings-init");
     });
 
     let brandShortName = document.getElementById("bundle_brand").getString("brandShortName");
     let settingsDescription = document.getElementById("settingsDescription");
     settingsDescription.textContent = this._prefStrBundle.getFormattedString("siteDataSettings.description", [brandShortName]);
@@ -87,24 +87,27 @@ let gSiteDataSettings = {
       // Sort on the current column, flip the sorting direction
       sortDirection = sortDirection === "ascending" ? "descending" : "ascending";
     }
 
     let sortFunc = null;
     switch (col.id) {
       case "hostCol":
         sortFunc = (a, b) => {
-          let aHost = a.uri.host.toLowerCase();
-          let bHost = b.uri.host.toLowerCase();
+          let aHost = a.host.toLowerCase();
+          let bHost = b.host.toLowerCase();
           return aHost.localeCompare(bHost);
         }
         break;
 
       case "statusCol":
-        sortFunc = (a, b) => a.status - b.status;
+        sortFunc = (a, b) => {
+          return a.persisted && !b.persisted ? 1 :
+                 !a.persisted && b.persisted ? -1 : 0;
+        };
         break;
 
       case "usageCol":
         sortFunc = (a, b) => a.usage - b.usage;
         break;
     }
     if (sortDirection === "descending") {
       sites.sort((a, b) => sortFunc(b, a));
@@ -128,62 +131,59 @@ let gSiteDataSettings = {
   _buildSitesList(sites) {
     // Clear old entries.
     let oldItems = this._list.querySelectorAll("richlistitem");
     for (let item of oldItems) {
       item.remove();
     }
 
     let keyword = this._searchBox.value.toLowerCase().trim();
-    for (let data of sites) {
-      let host = data.uri.host;
+    for (let site of sites) {
+      let host = site.host;
       if (keyword && !host.includes(keyword)) {
         continue;
       }
 
-      if (data.userAction === "remove") {
+      if (site.userAction === "remove") {
         continue;
       }
 
-      let size = DownloadUtils.convertByteUnits(data.usage);
+      let size = DownloadUtils.convertByteUnits(site.usage);
       let item = document.createElement("richlistitem");
-      item.setAttribute("data-origin", data.uri.spec);
       item.setAttribute("host", host);
       item.setAttribute("usage", this._prefStrBundle.getFormattedString("siteUsage", size));
-      if (data.status === Ci.nsIPermissionManager.ALLOW_ACTION ) {
+      if (site.persisted) {
         item.setAttribute("status", this._prefStrBundle.getString("persistent"));
       }
       this._list.appendChild(item);
     }
     this._updateButtonsState();
   },
 
   _removeSiteItems(items) {
     for (let i = items.length - 1; i >= 0; --i) {
       let item = items[i];
-      let origin = item.getAttribute("data-origin");
-      for (let site of this._sites) {
-        if (site.uri.spec === origin) {
-          site.userAction = "remove";
-          break;
-        }
+      let host = item.getAttribute("host");
+      let siteForHost = this._sites.find(site => site.host == host);
+      if (siteForHost) {
+        siteForHost.userAction = "remove";
       }
       item.remove();
     }
     this._updateButtonsState();
   },
 
   saveChanges() {
     let allowed = true;
 
     // Confirm user really wants to remove site data starts
     let removals = [];
     this._sites = this._sites.filter(site => {
       if (site.userAction === "remove") {
-        removals.push(site.uri);
+        removals.push(site.host);
         return false;
       }
       return true;
     });
 
     if (removals.length > 0) {
       if (this._sites.length == 0) {
         // User selects all sites so equivalent to clearing all data
@@ -199,46 +199,53 @@ let gSiteDataSettings = {
         allowed = result == 0;
         if (allowed) {
           SiteDataManager.removeAll();
         }
       } else {
         // User only removes partial sites.
         // We will remove cookies based on base domain, say, user selects "news.foo.com" to remove.
         // The cookies under "music.foo.com" will be removed together.
-        // We have to prmopt user about this action.
+        // We have to prompt user about this action.
         let hostsTable = new Map();
         // Group removed sites by base domain
-        for (let uri of removals) {
-          let baseDomain = Services.eTLD.getBaseDomain(uri);
+        for (let host of removals) {
+          let baseDomain = Services.eTLD.getBaseDomainFromHost(host);
           let hosts = hostsTable.get(baseDomain);
           if (!hosts) {
             hosts = [];
             hostsTable.set(baseDomain, hosts);
           }
-          hosts.push(uri.host);
+          hosts.push(host);
         }
         // Pick out sites with the same base domain as removed sites
         for (let site of this._sites) {
-          let baseDomain = Services.eTLD.getBaseDomain(site.uri);
+          let baseDomain = Services.eTLD.getBaseDomainFromHost(site.host);
           let hosts = hostsTable.get(baseDomain);
           if (hosts) {
-            hosts.push(site.uri.host);
+            hosts.push(site.host);
           }
         }
 
         let args = {
           hostsTable,
           allowed: false
         };
         let features = "centerscreen,chrome,modal,resizable=no";
         window.openDialog("chrome://browser/content/preferences/siteDataRemoveSelected.xul", "", features, args);
         allowed = args.allowed;
         if (allowed) {
-          SiteDataManager.remove(removals);
+          try {
+            SiteDataManager.remove(removals);
+          } catch (e) {
+            // Hit error, maybe remove unknown site.
+            // Let's print out the error, then proceed to close this settings dialog.
+            // When we next open again we will once more get sites from the SiteDataManager and refresh the list.
+            Cu.reportError(e);
+          }
         }
       }
     }
     // Confirm user really wants to remove site data ends
 
     this.close();
   },
 
--- a/browser/components/preferences/siteDataSettings.xul
+++ b/browser/components/preferences/siteDataSettings.xul
@@ -33,17 +33,18 @@
         placeholder="&searchTextboxPlaceHolder;" accesskey="&searchTextboxPlaceHolder.accesskey;"/>
     </hbox>
     <separator class="thin"/>
 
     <richlistbox id="sitesList" orient="vertical" flex="1">
       <listheader>
         <treecol flex="4" width="50" label="&hostCol.label;" id="hostCol"/>
         <treecol flex="2" width="50" label="&statusCol.label;" id="statusCol"/>
-        <treecol flex="1" width="50" label="&usageCol.label;" id="usageCol"/>
+        <!-- Sorted by usage so the user can quickly see which sites use the most data. -->
+        <treecol flex="1" width="50" label="&usageCol.label;" id="usageCol" data-isCurrentSortCol="true"/>
       </listheader>
     </richlistbox>
   </vbox>
 
   <hbox align="start">
     <button id="removeSelected" label="&removeSelected.label;" accesskey="&removeSelected.accesskey;"/>
     <button id="removeAll"/>
   </hbox>