Bug 1418884 - [Form Autofill] Make getAbbreviatedSubregionName/findOption supports more locales. r=scottwu, lchang draft
authorsteveck-chung <schung@mozilla.com>
Fri, 24 Nov 2017 17:04:00 +0800
changeset 708104 7ff5cceb89beeb7d3d85fef0b3e566891d5e6c4b
parent 706025 a21f4e2ce5186e2dc9ee411b07e9348866b4ef30
child 743097 8749524d50f4633c73f26600b8d4f5777cc5b5c2
push id92286
push userbmo:schung@mozilla.com
push dateWed, 06 Dec 2017 07:15:54 +0000
reviewersscottwu, lchang
bugs1418884
milestone59.0a1
Bug 1418884 - [Form Autofill] Make getAbbreviatedSubregionName/findOption supports more locales. r=scottwu, lchang MozReview-Commit-ID: HD8xFNHJwDR
browser/extensions/formautofill/FormAutofillUtils.jsm
browser/extensions/formautofill/test/mochitest/mochitest.ini
browser/extensions/formautofill/test/mochitest/test_multi_locale_CA_address_form.html
browser/extensions/formautofill/test/unit/test_addressDataLoader.js
--- a/browser/extensions/formautofill/FormAutofillUtils.jsm
+++ b/browser/extensions/formautofill/FormAutofillUtils.jsm
@@ -44,16 +44,17 @@ let AddressDataLoader = {
   // Status of address data loading. We'll load all the countries with basic level 1
   // information while requesting conutry information, and set country to true.
   // Level 1 Set is for recording which country's level 1/level 2 data is loaded,
   // since we only load this when getCountryAddressData called with level 1 parameter.
   _dataLoaded: {
     country: false,
     level1: new Set(),
   },
+
   /**
    * Load address data and extension script into a sandbox from different paths.
    * @param   {string} path
    *          The path for address data and extension script. It could be root of the address
    *          metadata folder(addressmetadata/) or under specific country(addressmetadata/TW/).
    * @returns {object}
    *          A sandbox that contains address data object with properties from extension.
    */
@@ -72,47 +73,97 @@ let AddressDataLoader = {
 
     if (extSandbox.addressDataExt) {
       for (let key in extSandbox.addressDataExt) {
         Object.assign(sandbox.addressData[key], extSandbox.addressDataExt[key]);
       }
     }
     return sandbox;
   },
+
+  /**
+   * Convert certain properties' string value into array. We should make sure
+   * the cached data is parsed.
+   * @param   {object} data Original metadata from addressReferences.
+   * @returns {object} parsed metadata with property value that converts to array.
+   */
+  _parse(data) {
+    if (!data) {
+      return null;
+    }
+
+    const properties = ["languages", "sub_keys", "sub_names", "sub_lnames"];
+    for (let key of properties) {
+      if (!data[key]) {
+        continue;
+      }
+      // No need to normalize data if the value is array already.
+      if (Array.isArray(data[key])) {
+        return data;
+      }
+
+      data[key] = data[key].split("~");
+    }
+    return data;
+  },
+
   /**
    * We'll cache addressData in the loader once the data loaded from scripts.
    * It'll become the example below after loading addressReferences with extension:
    * addressData: {
-                   "data/US": {"lang": "en", ...// Data defined in libaddressinput metadata
+   *               "data/US": {"lang": ["en"], ...// Data defined in libaddressinput metadata
    *                           "alternative_names": ... // Data defined in extension }
    *               "data/CA": {} // Other supported country metadata
    *               "data/TW": {} // Other supported country metadata
    *               "data/TW/台北市": {} // Other supported country level 1 metadata
    *              }
    * @param   {string} country
    * @param   {string} level1
-   * @returns {object}
+   * @returns {object} Default locale metadata
    */
-  getData(country, level1 = null) {
+  _loadData(country, level1 = null) {
     // Load the addressData if needed
     if (!this._dataLoaded.country) {
       this._addressData = this._loadScripts(ADDRESS_METADATA_PATH).addressData;
       this._dataLoaded.country = true;
     }
     if (!level1) {
-      return this._addressData[`data/${country}`];
+      return this._parse(this._addressData[`data/${country}`]);
     }
     // If level1 is set, load addressReferences under country folder with specific
     // country/level 1 for level 2 information.
     if (!this._dataLoaded.level1.has(country)) {
       Object.assign(this._addressData,
                     this._loadScripts(`${ADDRESS_METADATA_PATH}${country}/`).addressData);
       this._dataLoaded.level1.add(country);
     }
-    return this._addressData[`data/${country}/${level1}`];
+    return this._parse(this._addressData[`data/${country}/${level1}`]);
+  },
+
+  /**
+   * Return the region metadata with default locale and other locales (if exists).
+   * @param   {string} country
+   * @param   {string} level1
+   * @returns {object} Return default locale and other locales metadata.
+   */
+  getData(country, level1) {
+    let defaultLocale = this._loadData(country, level1);
+    if (!defaultLocale) {
+      return null;
+    }
+
+    let countryData = this._parse(this._addressData[`data/${country}`]);
+    let locales = [];
+    // TODO: Should be able to support multi-locale level 1/ level 2 metadata query
+    //      in Bug 1421886
+    if (countryData.languages) {
+      let list = countryData.languages.filter(key => key !== countryData.lang);
+      locales = list.map(key => this._parse(this._addressData[`${defaultLocale.id}--${key}`]));
+    }
+    return {defaultLocale, locales};
   },
 };
 
 this.FormAutofillUtils = {
   get AUTOFILL_FIELDS_THRESHOLD() { return 3; },
   get isAutofillEnabled() { return this.isAutofillAddressesEnabled || this.isAutofillCreditCardsEnabled; },
   get isAutofillCreditCardsEnabled() { return this.isAutofillCreditCardsAvailable && this._isAutofillCreditCardsEnabled; },
 
@@ -284,55 +335,84 @@ this.FormAutofillUtils = {
 
   loadDataFromScript(url, sandbox = {}) {
     Services.scriptloader.loadSubScript(url, sandbox, "utf-8");
     return sandbox;
   },
 
   /**
    * Get country address data and fallback to US if not found.
-   * See AddressDataLoader.getData for more details of addressData structure.
+   * See AddressDataLoader._loadData for more details of addressData structure.
    * @param {string} [country=FormAutofillUtils.DEFAULT_REGION]
    *        The country code for requesting specific country's metadata. It'll be
    *        default region if parameter is not set.
    * @param {string} [level1=null]
    *        Retrun address level 1/level 2 metadata if parameter is set.
-   * @returns {object}
-   *          Return the metadata of specific region.
+   * @returns {object|null}
+   *          Return metadata of specific region with default locale and other supported
+   *          locales. We need to return a deafult country metadata for layout format
+   *          and collator, but for sub-region metadata we'll just return null if not found.
    */
-  getCountryAddressData(country = FormAutofillUtils.DEFAULT_REGION, level1 = null) {
+  getCountryAddressRawData(country = FormAutofillUtils.DEFAULT_REGION, level1 = null) {
     let metadata = AddressDataLoader.getData(country, level1);
     if (!metadata) {
       if (level1) {
         return null;
       }
       // Fallback to default region if we couldn't get data from given country.
       if (country != FormAutofillUtils.DEFAULT_REGION) {
         metadata = AddressDataLoader.getData(FormAutofillUtils.DEFAULT_REGION);
       }
     }
 
-    // Fallback to US if we couldn't get data from default region.
-    return metadata || AddressDataLoader.getData("US");
+    // TODO: Now we fallback to US if we couldn't get data from default region,
+    //       but it could be removed in bug 1423464 if it's not necessary.
+    if (!metadata) {
+      metadata = AddressDataLoader.getData("US");
+    }
+    return metadata;
+  },
+
+  /**
+   * Get country address data with default locale.
+   * @param {string} country
+   * @param {string} level1
+   * @returns {object|null} Return metadata of specific region with default locale.
+   */
+  getCountryAddressData(country, level1) {
+    let metadata = this.getCountryAddressRawData(country, level1);
+    return metadata && metadata.defaultLocale;
+  },
+
+  /**
+   * Get country address data with all locales.
+   * @param {string} country
+   * @param {string} level1
+   * @returns {array<object>|null}
+   *          Return metadata of specific region with all the locales.
+   */
+  getCountryAddressDataWithLocales(country, level1) {
+    let metadata = this.getCountryAddressRawData(country, level1);
+    return metadata && [metadata.defaultLocale, ...metadata.locales];
   },
 
   /**
    * Get the collators based on the specified country.
    * @param   {string} country The specified country.
    * @returns {array} An array containing several collator objects.
    */
   getCollators(country) {
     // TODO: Only one language should be used at a time per country. The locale
     //       of the page should be taken into account to do this properly.
     //       We are going to support more countries in bug 1370193 and this
     //       should be addressed when we start to implement that bug.
 
     if (!this._collators[country]) {
       let dataset = this.getCountryAddressData(country);
-      let languages = dataset.languages ? dataset.languages.split("~") : [dataset.lang];
+      let languages = dataset.languages || [dataset.lang];
       this._collators[country] = languages.map(lang => new Intl.Collator(lang, {sensitivity: "base", ignorePunctuation: true}));
     }
     return this._collators[country];
   },
 
   /**
    * Parse a country address format string and outputs an array of fields.
    * Spaces, commas, and other literals are ignored in this implementation.
@@ -433,43 +513,43 @@ this.FormAutofillUtils = {
    * @param   {string[]} subregionValues A list of inferable sub-region values.
    * @param   {string} [country] A country name to be identified.
    * @returns {string} The matching sub-region abbreviation.
    */
   getAbbreviatedSubregionName(subregionValues, country) {
     let values = Array.isArray(subregionValues) ? subregionValues : [subregionValues];
 
     let collators = this.getCollators(country);
-    let {sub_keys: subKeys, sub_names: subNames} = this.getCountryAddressData(country);
+    for (let metadata of this.getCountryAddressDataWithLocales(country)) {
+      let {sub_keys: subKeys, sub_names: subNames, sub_lnames: subLnames} = metadata;
+      // Apply sub_lnames if sub_names does not exist
+      subNames = subNames || subLnames;
 
-    if (!Array.isArray(subKeys)) {
-      subKeys = subKeys.split("~");
-    }
-    if (!Array.isArray(subNames)) {
-      subNames = subNames.split("~");
-    }
+      let speculatedSubIndexes = [];
+      for (const val of values) {
+        let identifiedValue = this.identifyValue(subKeys, subNames, val, collators);
+        if (identifiedValue) {
+          return identifiedValue;
+        }
 
-    let speculatedSubIndexes = [];
-    for (const val of values) {
-      let identifiedValue = this.identifyValue(subKeys, subNames, val, collators);
-      if (identifiedValue) {
-        return identifiedValue;
-      }
+        // Predict the possible state by partial-matching if no exact match.
+        [subKeys, subNames].forEach(sub => {
+          speculatedSubIndexes.push(sub.findIndex(token => {
+            let pattern = new RegExp("\\b" + this.escapeRegExp(token) + "\\b");
 
-      // Predict the possible state by partial-matching if no exact match.
-      [subKeys, subNames].forEach(sub => {
-        speculatedSubIndexes.push(sub.findIndex(token => {
-          let pattern = new RegExp("\\b" + this.escapeRegExp(token) + "\\b");
-
-          return pattern.test(val);
-        }));
-      });
+            return pattern.test(val);
+          }));
+        });
+      }
+      let subKey = subKeys[speculatedSubIndexes.find(i => !!~i)];
+      if (subKey) {
+        return subKey;
+      }
     }
-
-    return subKeys[speculatedSubIndexes.find(i => !!~i)] || null;
+    return null;
   },
 
   /**
    * Find the option element from select element.
    * 1. Try to find the locale using the country from address.
    * 2. First pass try to find exact match.
    * 3. Second pass try to identify values from address value and options,
    *    and look for a match.
@@ -479,51 +559,47 @@ this.FormAutofillUtils = {
    * @returns {DOMElement}
    */
   findAddressSelectOption(selectEl, address, fieldName) {
     let value = address[fieldName];
     if (!value) {
       return null;
     }
 
-    let dataset = this.getCountryAddressData(address.country);
     let collators = this.getCollators(address.country);
 
     for (let option of selectEl.options) {
       if (this.strCompare(value, option.value, collators) ||
           this.strCompare(value, option.text, collators)) {
         return option;
       }
     }
 
     switch (fieldName) {
       case "address-level1": {
-        if (!Array.isArray(dataset.sub_keys)) {
-          dataset.sub_keys = dataset.sub_keys.split("~");
-        }
-        if (!Array.isArray(dataset.sub_names)) {
-          dataset.sub_names = dataset.sub_names.split("~");
-        }
-        let keys = dataset.sub_keys;
-        let names = dataset.sub_names;
-        let identifiedValue = this.identifyValue(keys, names, value, collators);
-
-        // No point going any further if we cannot identify value from address
+        let {country} = address;
+        let identifiedValue = this.getAbbreviatedSubregionName([value], country);
+        // No point going any further if we cannot identify value from address level 1
         if (!identifiedValue) {
           return null;
         }
+        for (let dataset of this.getCountryAddressDataWithLocales(country)) {
+          let keys = dataset.sub_keys;
+          // Apply sub_lnames if sub_names does not exist
+          let names = dataset.sub_names || dataset.sub_lnames;
 
-        // Go through options one by one to find a match.
-        // Also check if any option contain the address-level1 key.
-        let pattern = new RegExp("\\b" + this.escapeRegExp(identifiedValue) + "\\b", "i");
-        for (let option of selectEl.options) {
-          let optionValue = this.identifyValue(keys, names, option.value, collators);
-          let optionText = this.identifyValue(keys, names, option.text, collators);
-          if (identifiedValue === optionValue || identifiedValue === optionText || pattern.test(option.value)) {
-            return option;
+          // Go through options one by one to find a match.
+          // Also check if any option contain the address-level1 key.
+          let pattern = new RegExp("\\b" + this.escapeRegExp(identifiedValue) + "\\b", "i");
+          for (let option of selectEl.options) {
+            let optionValue = this.identifyValue(keys, names, option.value, collators);
+            let optionText = this.identifyValue(keys, names, option.text, collators);
+            if (identifiedValue === optionValue || identifiedValue === optionText || pattern.test(option.value)) {
+              return option;
+            }
           }
         }
         break;
       }
       case "country": {
         if (this.getCountryAddressData(value).alternative_names) {
           for (let option of selectEl.options) {
             if (this.identifyCountryCode(option.text, value) || this.identifyCountryCode(option.value, value)) {
--- a/browser/extensions/formautofill/test/mochitest/mochitest.ini
+++ b/browser/extensions/formautofill/test/mochitest/mochitest.ini
@@ -11,10 +11,11 @@ support-files =
 [test_basic_autocomplete_form.html]
 [test_basic_creditcard_autocomplete_form.html]
 scheme=https
 [test_clear_form.html]
 [test_creditcard_autocomplete_off.html]
 scheme=https
 [test_form_changes.html]
 [test_formautofill_preview_highlight.html]
+[test_multi_locale_CA_address_form.html]
 [test_multiple_forms.html]
 [test_on_address_submission.html]
new file mode 100644
--- /dev/null
+++ b/browser/extensions/formautofill/test/mochitest/test_multi_locale_CA_address_form.html
@@ -0,0 +1,201 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>Test basic autofill</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <script type="text/javascript" src="/tests/SimpleTest/SpawnTask.js"></script>
+  <script type="text/javascript" src="formautofill_common.js"></script>
+  <script type="text/javascript" src="satchel_common.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+Form autofill test: simple form address autofill
+
+<script>
+/* import-globals-from ../../../../../testing/mochitest/tests/SimpleTest/SpawnTask.js */
+/* import-globals-from ../../../../../toolkit/components/satchel/test/satchel_common.js */
+/* import-globals-from formautofill_common.js */
+
+"use strict";
+
+let MOCK_STORAGE = [{
+  organization: "Mozilla Vancouver",
+  "street-address": "163 W Hastings St.\n#209\n3-line",
+  tel: "+17787851540",
+  country: "CA",
+  "address-level1": "BC",
+}, {
+  organization: "Mozilla Toronto",
+  "street-address": "366 Adelaide St.\nW Suite 500\n3-line",
+  tel: "+14168483114",
+  country: "CA",
+  "address-level1": "ON",
+}, {
+  organization: "Prince of Wales Northern Heritage",
+  "street-address": "4750 48 St.\nYellowknife\n3-line",
+  tel: "+18677679347",
+  country: "CA",
+  "address-level1": "Northwest Territories",
+}, {
+  organization: "ExpoCité",
+  "street-address": "250 Boulevard Wilfrid-Hamel\nVille de Québec\n3-line",
+  tel: "+14186917110",
+  country: "CA",
+  "address-level1": "Québec",
+}];
+
+function checkElementFilled(element, expectedvalue) {
+  return [
+    new Promise(resolve => {
+      element.addEventListener("input", function onInput() {
+        ok(true, "Checking " + element.name + " field fires input event");
+        resolve();
+      }, {once: true});
+    }),
+    new Promise(resolve => {
+      element.addEventListener("change", function onChange() {
+        ok(true, "Checking " + element.name + " field fires change event");
+        is(element.value, expectedvalue, "Checking " + element.name + " field");
+        resolve();
+      }, {once: true});
+    }),
+  ];
+}
+
+function checkAutoCompleteInputFilled(element, expectedvalue) {
+  return new Promise(resolve => {
+    element.addEventListener("DOMAutoComplete", function onChange() {
+      is(element.value, expectedvalue, "Checking " + element.name + " field");
+      resolve();
+    }, {once: true});
+  });
+}
+
+function checkFormFilled(selector, address) {
+  info("expecting form filled");
+  let promises = [];
+  let form = document.querySelector(selector);
+  for (let prop in address) {
+    let element = form.querySelector(`[name=${prop}]`);
+    if (document.activeElement == element) {
+      promises.push(checkAutoCompleteInputFilled(element, address[prop]));
+    } else {
+      let converted = address[prop];
+      if (prop == "street-address") {
+        converted = FormAutofillUtils.toOneLineAddress(converted);
+      }
+      promises.push(...checkElementFilled(element, converted));
+    }
+  }
+  doKey("return");
+  return Promise.all(promises);
+}
+
+async function setupAddressStorage() {
+  for (let address of MOCK_STORAGE) {
+    await addAddress(address);
+  }
+}
+
+initPopupListener();
+
+// Autofill the address with address level 1 code.
+add_task(async function autofill_with_level1_code() {
+  await setupAddressStorage();
+
+  await setInput("#organization-en", "Mozilla Toronto");
+  doKey("down");
+  await expectPopup();
+
+  doKey("down");
+  // Replace address level 1 code with full name in English for test result
+  let result = Object.assign({}, MOCK_STORAGE[1], {"address-level1": "Ontario"});
+  await checkFormFilled("#form-en", result);
+
+  await setInput("#organization-fr", "Mozilla Vancouver");
+  doKey("down");
+  await expectPopup();
+
+  doKey("down");
+  // Replace address level 1 code with full name in French for test result
+  result = Object.assign({}, MOCK_STORAGE[0], {"address-level1": "Colombie-Britannique"});
+  await checkFormFilled("#form-fr", result);
+  document.querySelector("#form-en").reset();
+  document.querySelector("#form-fr").reset();
+});
+
+// Autofill the address with address level 1 full name.
+add_task(async function autofill_with_level1_full_name() {
+  await setInput("#organization-en", "ExpoCité");
+  doKey("down");
+  await expectPopup();
+
+  doKey("down");
+  // Replace address level 1 code with full name in French for test result
+  let result = Object.assign({}, MOCK_STORAGE[3], {"address-level1": "Quebec"});
+  await checkFormFilled("#form-en", result);
+
+  await setInput("#organization-fr", "Prince of Wales Northern Heritage");
+  doKey("down");
+  await expectPopup();
+
+  doKey("down");
+  // Replace address level 1 code with full name in English for test result
+  result = Object.assign({}, MOCK_STORAGE[2], {"address-level1": "Territoires du Nord-Ouest"});
+  await checkFormFilled("#form-fr", result);
+});
+
+</script>
+
+<p id="display"></p>
+
+<div id="content">
+
+  <form id="form-en">
+    <p>This is a basic CA form with en address level 1 select.</p>
+    <p><label>organization: <input id="organization-en" name="organization" autocomplete="organization" type="text"></label></p>
+    <p><label>streetAddress: <input id="street-address-en" name="street-address" autocomplete="street-address" type="text"></label></p>
+    <p><label>address-line1: <input id="address-line1-en" name="address-line1" autocomplete="address-line1" type="text"></label></p>
+    <p><label>tel: <input id="tel-en" name="tel" autocomplete="tel" type="text"></label></p>
+    <p><label>email: <input id="email-en" name="email" autocomplete="email" type="text"></label></p>
+    <p><label>country: <select id="country-en" name="country" autocomplete="country">
+      <option/>
+      <option value="US">United States</option>
+      <option value="CA">Canada</option>
+    </select></label></p>
+    <p><label>states: <select id="address-level1-en" name="address-level1" autocomplete="address-level1">
+      <option/>
+      <option value="British Columbia">British Columbia</option>
+      <option value="Ontario">Ontario</option>
+      <option value="Northwest Territories">Northwest Territories</option>
+      <option value="Quebec">Quebec</option>
+    </select></label></p>
+  </form>
+
+  <form id="form-fr">
+    <p>This is a basic CA form with fr address level 1 select.</p>
+    <p><label>organization: <input id="organization-fr" name="organization" autocomplete="organization" type="text"></label></p>
+    <p><label>streetAddress: <input id="street-address-fr" name="street-address" autocomplete="street-address" type="text"></label></p>
+    <p><label>address-line1: <input id="address-line1-fr" name="address-line1" autocomplete="address-line1" type="text"></label></p>
+    <p><label>tel: <input id="tel-fr" name="tel" autocomplete="tel" type="text"></label></p>
+    <p><label>email: <input id="email-fr" name="email" autocomplete="email" type="text"></label></p>
+    <p><label>country: <select id="country-fr" name="country" autocomplete="country">
+      <option/>
+      <option value="US">United States</option>
+      <option value="CA">Canada</option>
+    </select></label></p>
+    <p><label>states: <select id="address-level1-fr" name="address-level1" autocomplete="address-level1">
+      <option/>
+      <option value="Colombie-Britannique">Colombie-Britannique</option>
+      <option value="Ontario">Ontario</option>
+      <option value="Territoires du Nord-Ouest">Territoires du Nord-Ouest</option>
+      <option value="Québec">Québec</option>
+    </select></label></p>
+  </form>
+
+</div>
+
+<pre id="test"></pre>
+</body>
+</html>
--- a/browser/extensions/formautofill/test/unit/test_addressDataLoader.js
+++ b/browser/extensions/formautofill/test/unit/test_addressDataLoader.js
@@ -64,10 +64,18 @@ add_task(async function test_loadDataSta
 });
 
 SUPPORT_COUNTRIES_TESTCASES.forEach(testcase => {
   add_task(async function test_support_country() {
     do_print("Starting testcase: Check " + testcase.country + " metadata");
     let metadata = FormAutofillUtils.getCountryAddressData(testcase.country);
     Assert.ok(testcase.properties.every(key => metadata[key]),
               "These properties should exist: " + testcase.properties);
+    // Verify the multi-locale country
+    if (metadata.languages && metadata.languages.length > 1) {
+      let locales = FormAutofillUtils.getCountryAddressDataWithLocales(testcase.country);
+      Assert.equal(metadata.languages.length, locales.length, "Total supported locales should be matched");
+      metadata.languages.forEach((lang, index) => {
+        Assert.equal(lang, locales[index].lang, `Should support ${lang}`);
+      });
+    }
   });
 });