Bug 1389099 - Properly encode multi-byte translations in CSS draft
authorRob Wu <rob@robwu.nl>
Tue, 08 May 2018 22:35:22 +0200
changeset 796409 8274c60e07d10f5ea290238c57ff58809d8249e3
parent 796393 24bae072acb09114c367e6b9ffde9261b2ad8a58
push id110245
push userbmo:rob@robwu.nl
push dateThu, 17 May 2018 17:03:01 +0000
bugs1389099
milestone62.0a1
Bug 1389099 - Properly encode multi-byte translations in CSS The localization filter was not unicode-aware because convertToStream assigns the output to a nsIStringInputStream (which takes 8-bit chars). The input was read as a 8-bit string, but after localization it can contain wide strings if a translation has a multi-byte character. To fix this, the input stream is now first read as a UTF-8 string, then localized, and finally exported via a nsIArrayBufferInputStream.. MozReview-Commit-ID: LjCxczIFKCR
toolkit/components/extensions/test/xpcshell/test_ext_i18n_css.js
toolkit/components/utils/simpleServices.js
--- a/toolkit/components/extensions/test/xpcshell/test_ext_i18n_css.js
+++ b/toolkit/components/extensions/test/xpcshell/test_ext_i18n_css.js
@@ -15,16 +15,20 @@ const {
   promiseShutdownManager,
   promiseStartupManager,
 } = AddonTestUtils;
 
 AddonTestUtils.init(this);
 
 createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "42");
 
+// Some multibyte characters. This sample was taken from the encoding/api-basics.html web platform test.
+const MULTIBYTE_STRING = "z\xA2\u6C34\uD834\uDD1E\uF8FF\uDBFF\uDFFD\uFFFE";
+let getCSS = (a, b) => `a { content: '${a}'; } b { content: '${b}'; }`;
+
 let extensionData = {
   background: function() {
     function backgroundFetch(url) {
       return new Promise((resolve, reject) => {
         let xhr = new XMLHttpRequest();
         xhr.overrideMimeType("text/plain");
         xhr.open("GET", url);
         xhr.onload = () => { resolve(xhr.responseText); };
@@ -37,27 +41,27 @@ let extensionData = {
       browser.test.assertEq("body { max-width: 42px; }", results[0], "CSS file localized");
       browser.test.assertEq("body { max-width: 42px; }", results[1], "CSS file localized");
 
       browser.test.assertEq("body { __MSG_foo__; }", results[2], "Text file not localized");
 
       browser.test.notifyPass("i18n-css");
     });
 
-    browser.test.sendMessage("ready", browser.runtime.getURL("foo.css"));
+    browser.test.sendMessage("ready", browser.runtime.getURL("/"));
   },
 
   manifest: {
     "applications": {
       "gecko": {
         "id": "i18n_css@mochi.test",
       },
     },
 
-    "web_accessible_resources": ["foo.css", "foo.txt", "locale.css"],
+    "web_accessible_resources": ["foo.css", "foo.txt", "locale.css", "multibyte.css"],
 
     "content_scripts": [
       {
         "matches": ["http://*/*/file_sample.html"],
         "css": ["foo.css"],
         "run_at": "document_start",
       },
       {
@@ -70,66 +74,71 @@ let extensionData = {
   },
 
   files: {
     "_locales/en/messages.json": JSON.stringify({
       "foo": {
         "message": "max-width: 42px",
         "description": "foo",
       },
+      "multibyteKey": {
+        "message": MULTIBYTE_STRING,
+      },
     }),
 
     "content.js": function() {
       let style = getComputedStyle(document.body);
       browser.test.sendMessage("content-maxWidth", style.maxWidth);
     },
 
     "foo.css": "body { __MSG_foo__; }",
     "bar.CsS": "body { __MSG_foo__; }",
     "foo.txt": "body { __MSG_foo__; }",
     "locale.css": '* { content: "__MSG_@@ui_locale__ __MSG_@@bidi_dir__ __MSG_@@bidi_reversed_dir__ __MSG_@@bidi_start_edge__ __MSG_@@bidi_end_edge__" }',
+    "multibyte.css": getCSS("__MSG_multibyteKey__", MULTIBYTE_STRING),
   },
 };
 
 async function test_i18n_css(options = {}) {
   extensionData.useAddonManager = options.useAddonManager;
   let extension = ExtensionTestUtils.loadExtension(extensionData);
 
   await extension.startup();
-  let cssURL = await extension.awaitMessage("ready");
+  let baseURL = await extension.awaitMessage("ready");
 
   let contentPage = await ExtensionTestUtils.loadContentPage(`${BASE_URL}/file_sample.html`);
 
-  let css = await contentPage.fetch(cssURL);
+  let css = await contentPage.fetch(baseURL + "foo.css");
 
   equal(css, "body { max-width: 42px; }", "CSS file localized in mochitest scope");
 
   let maxWidth = await extension.awaitMessage("content-maxWidth");
 
   equal(maxWidth, "42px", "stylesheet correctly applied");
 
-  cssURL = cssURL.replace(/foo.css$/, "locale.css");
+  css = await contentPage.fetch(baseURL + "locale.css");
+  equal(css, '* { content: "en-US ltr rtl left right" }', "CSS file localized in mochitest scope");
 
-  css = await contentPage.fetch(cssURL);
-  equal(css, '* { content: "en-US ltr rtl left right" }', "CSS file localized in mochitest scope");
+  css = await contentPage.fetch(baseURL + "multibyte.css");
+  equal(css, getCSS(MULTIBYTE_STRING, MULTIBYTE_STRING), "CSS file contains multibyte string");
 
   await contentPage.close();
 
   // We don't currently have a good way to mock this.
   if (false) {
     const DIR = "intl.uidirection";
 
     // We don't wind up actually switching the chrome registry locale, since we
     // don't have a chrome package for Hebrew. So just override it, and force
     // RTL directionality.
     const origReqLocales = Services.locale.getRequestedLocales();
     Services.locale.setRequestedLocales(["he"]);
     Preferences.set(DIR, 1);
 
-    css = await fetch(cssURL);
+    css = await fetch(baseURL + "locale.css");
     equal(css, '* { content: "he rtl ltr right left" }', "CSS file localized in mochitest scope");
 
     Services.locale.setRequestedLocales(origReqLocales);
     Preferences.reset(DIR);
   }
 
   await extension.awaitFinish("i18n-css");
   await extension.unload();
--- a/toolkit/components/utils/simpleServices.js
+++ b/toolkit/components/utils/simpleServices.js
@@ -20,16 +20,19 @@ ChromeUtils.defineModuleGetter(this, "Ne
                                "resource://gre/modules/NetUtil.jsm");
 ChromeUtils.defineModuleGetter(this, "Services",
                                "resource://gre/modules/Services.jsm");
 
 XPCOMUtils.defineLazyServiceGetter(this, "catMan", "@mozilla.org/categorymanager;1",
                                    "nsICategoryManager");
 XPCOMUtils.defineLazyServiceGetter(this, "streamConv", "@mozilla.org/streamConverters;1",
                                    "nsIStreamConverterService");
+const ArrayBufferInputStream = Components.Constructor(
+  "@mozilla.org/io/arraybuffer-input-stream;1",
+  "nsIArrayBufferInputStream", "setData");
 
 /*
  * This class provides a stream filter for locale messages in CSS files served
  * by the moz-extension: protocol handler.
  *
  * See SubstituteChannel in netwerk/protocol/res/ExtensionProtocolHandler.cpp
  * for usage.
  */
@@ -63,56 +66,56 @@ AddonLocalizationConverter.prototype = {
     let addon = WebExtensionPolicy.getByURI(uri);
     if (!addon) {
       throw new Components.Exception("Invalid context", Cr.NS_ERROR_INVALID_ARG);
     }
     return addon;
   },
 
   convertToStream(aAddon, aString) {
-    let stream = Cc["@mozilla.org/io/string-input-stream;1"]
-      .createInstance(Ci.nsIStringInputStream);
-
-    stream.data = aAddon.localize(aString);
-    return stream;
+    aString = aAddon.localize(aString);
+    let bytes = new TextEncoder().encode(aString).buffer;
+    return new ArrayBufferInputStream(bytes, 0, bytes.byteLength);
   },
 
   convert(aStream, aFromType, aToType, aContext) {
     this.checkTypes(aFromType, aToType);
     let addon = this.getAddon(aContext);
 
-    let string = (
-      aStream.available() ?
-      NetUtil.readInputStreamToString(aStream, aStream.available()) : ""
-    );
+    let count = aStream.available();
+    let string = count ?
+      new TextDecoder().decode(NetUtil.readInputStream(aStream, count)) : "";
     return this.convertToStream(addon, string);
   },
 
   asyncConvertData(aFromType, aToType, aListener, aContext) {
     this.checkTypes(aFromType, aToType);
     this.addon = this.getAddon(aContext);
     this.listener = aListener;
   },
 
   onStartRequest(aRequest, aContext) {
     this.parts = [];
+    this.decoder = new TextDecoder();
   },
 
   onDataAvailable(aRequest, aContext, aInputStream, aOffset, aCount) {
-    this.parts.push(NetUtil.readInputStreamToString(aInputStream, aCount));
+    let bytes = NetUtil.readInputStream(aInputStream, aCount);
+    this.parts.push(this.decoder.decode(bytes, {stream: true}));
   },
 
   onStopRequest(aRequest, aContext, aStatusCode) {
     try {
       this.listener.onStartRequest(aRequest, null);
       if (Components.isSuccessCode(aStatusCode)) {
+        this.parts.push(this.decoder.decode());
         let string = this.parts.join("");
         let stream = this.convertToStream(this.addon, string);
 
-        this.listener.onDataAvailable(aRequest, null, stream, 0, stream.data.length);
+        this.listener.onDataAvailable(aRequest, null, stream, 0, stream.available());
       }
     } catch (e) {
       aStatusCode = e.result || Cr.NS_ERROR_FAILURE;
     }
     this.listener.onStopRequest(aRequest, null, aStatusCode);
   },
 };