Bug 1389099 - Properly encode multi-byte translations in CSS
The localization filter was not unicode-aware because convertToStream
assigns the output to a nsIStringInputStream (which takes 8-bit chars).
The input was read as a 8-bit string, but after localization it can
contain wide strings if a translation has a multi-byte character.
To fix this, the input stream is now first read as a UTF-8 string, then
localized, and finally exported via a nsIArrayBufferInputStream..
MozReview-Commit-ID: LjCxczIFKCR
--- a/toolkit/components/extensions/test/xpcshell/test_ext_i18n_css.js
+++ b/toolkit/components/extensions/test/xpcshell/test_ext_i18n_css.js
@@ -15,16 +15,20 @@ const {
promiseShutdownManager,
promiseStartupManager,
} = AddonTestUtils;
AddonTestUtils.init(this);
createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "42");
+// Some multibyte characters. This sample was taken from the encoding/api-basics.html web platform test.
+const MULTIBYTE_STRING = "z\xA2\u6C34\uD834\uDD1E\uF8FF\uDBFF\uDFFD\uFFFE";
+let getCSS = (a, b) => `a { content: '${a}'; } b { content: '${b}'; }`;
+
let extensionData = {
background: function() {
function backgroundFetch(url) {
return new Promise((resolve, reject) => {
let xhr = new XMLHttpRequest();
xhr.overrideMimeType("text/plain");
xhr.open("GET", url);
xhr.onload = () => { resolve(xhr.responseText); };
@@ -37,27 +41,27 @@ let extensionData = {
browser.test.assertEq("body { max-width: 42px; }", results[0], "CSS file localized");
browser.test.assertEq("body { max-width: 42px; }", results[1], "CSS file localized");
browser.test.assertEq("body { __MSG_foo__; }", results[2], "Text file not localized");
browser.test.notifyPass("i18n-css");
});
- browser.test.sendMessage("ready", browser.runtime.getURL("foo.css"));
+ browser.test.sendMessage("ready", browser.runtime.getURL("/"));
},
manifest: {
"applications": {
"gecko": {
"id": "i18n_css@mochi.test",
},
},
- "web_accessible_resources": ["foo.css", "foo.txt", "locale.css"],
+ "web_accessible_resources": ["foo.css", "foo.txt", "locale.css", "multibyte.css"],
"content_scripts": [
{
"matches": ["http://*/*/file_sample.html"],
"css": ["foo.css"],
"run_at": "document_start",
},
{
@@ -70,66 +74,71 @@ let extensionData = {
},
files: {
"_locales/en/messages.json": JSON.stringify({
"foo": {
"message": "max-width: 42px",
"description": "foo",
},
+ "multibyteKey": {
+ "message": MULTIBYTE_STRING,
+ },
}),
"content.js": function() {
let style = getComputedStyle(document.body);
browser.test.sendMessage("content-maxWidth", style.maxWidth);
},
"foo.css": "body { __MSG_foo__; }",
"bar.CsS": "body { __MSG_foo__; }",
"foo.txt": "body { __MSG_foo__; }",
"locale.css": '* { content: "__MSG_@@ui_locale__ __MSG_@@bidi_dir__ __MSG_@@bidi_reversed_dir__ __MSG_@@bidi_start_edge__ __MSG_@@bidi_end_edge__" }',
+ "multibyte.css": getCSS("__MSG_multibyteKey__", MULTIBYTE_STRING),
},
};
async function test_i18n_css(options = {}) {
extensionData.useAddonManager = options.useAddonManager;
let extension = ExtensionTestUtils.loadExtension(extensionData);
await extension.startup();
- let cssURL = await extension.awaitMessage("ready");
+ let baseURL = await extension.awaitMessage("ready");
let contentPage = await ExtensionTestUtils.loadContentPage(`${BASE_URL}/file_sample.html`);
- let css = await contentPage.fetch(cssURL);
+ let css = await contentPage.fetch(baseURL + "foo.css");
equal(css, "body { max-width: 42px; }", "CSS file localized in mochitest scope");
let maxWidth = await extension.awaitMessage("content-maxWidth");
equal(maxWidth, "42px", "stylesheet correctly applied");
- cssURL = cssURL.replace(/foo.css$/, "locale.css");
+ css = await contentPage.fetch(baseURL + "locale.css");
+ equal(css, '* { content: "en-US ltr rtl left right" }', "CSS file localized in mochitest scope");
- css = await contentPage.fetch(cssURL);
- equal(css, '* { content: "en-US ltr rtl left right" }', "CSS file localized in mochitest scope");
+ css = await contentPage.fetch(baseURL + "multibyte.css");
+ equal(css, getCSS(MULTIBYTE_STRING, MULTIBYTE_STRING), "CSS file contains multibyte string");
await contentPage.close();
// We don't currently have a good way to mock this.
if (false) {
const DIR = "intl.uidirection";
// We don't wind up actually switching the chrome registry locale, since we
// don't have a chrome package for Hebrew. So just override it, and force
// RTL directionality.
const origReqLocales = Services.locale.getRequestedLocales();
Services.locale.setRequestedLocales(["he"]);
Preferences.set(DIR, 1);
- css = await fetch(cssURL);
+ css = await fetch(baseURL + "locale.css");
equal(css, '* { content: "he rtl ltr right left" }', "CSS file localized in mochitest scope");
Services.locale.setRequestedLocales(origReqLocales);
Preferences.reset(DIR);
}
await extension.awaitFinish("i18n-css");
await extension.unload();
--- a/toolkit/components/utils/simpleServices.js
+++ b/toolkit/components/utils/simpleServices.js
@@ -20,16 +20,19 @@ ChromeUtils.defineModuleGetter(this, "Ne
"resource://gre/modules/NetUtil.jsm");
ChromeUtils.defineModuleGetter(this, "Services",
"resource://gre/modules/Services.jsm");
XPCOMUtils.defineLazyServiceGetter(this, "catMan", "@mozilla.org/categorymanager;1",
"nsICategoryManager");
XPCOMUtils.defineLazyServiceGetter(this, "streamConv", "@mozilla.org/streamConverters;1",
"nsIStreamConverterService");
+const ArrayBufferInputStream = Components.Constructor(
+ "@mozilla.org/io/arraybuffer-input-stream;1",
+ "nsIArrayBufferInputStream", "setData");
/*
* This class provides a stream filter for locale messages in CSS files served
* by the moz-extension: protocol handler.
*
* See SubstituteChannel in netwerk/protocol/res/ExtensionProtocolHandler.cpp
* for usage.
*/
@@ -63,56 +66,56 @@ AddonLocalizationConverter.prototype = {
let addon = WebExtensionPolicy.getByURI(uri);
if (!addon) {
throw new Components.Exception("Invalid context", Cr.NS_ERROR_INVALID_ARG);
}
return addon;
},
convertToStream(aAddon, aString) {
- let stream = Cc["@mozilla.org/io/string-input-stream;1"]
- .createInstance(Ci.nsIStringInputStream);
-
- stream.data = aAddon.localize(aString);
- return stream;
+ aString = aAddon.localize(aString);
+ let bytes = new TextEncoder().encode(aString).buffer;
+ return new ArrayBufferInputStream(bytes, 0, bytes.byteLength);
},
convert(aStream, aFromType, aToType, aContext) {
this.checkTypes(aFromType, aToType);
let addon = this.getAddon(aContext);
- let string = (
- aStream.available() ?
- NetUtil.readInputStreamToString(aStream, aStream.available()) : ""
- );
+ let count = aStream.available();
+ let string = count ?
+ new TextDecoder().decode(NetUtil.readInputStream(aStream, count)) : "";
return this.convertToStream(addon, string);
},
asyncConvertData(aFromType, aToType, aListener, aContext) {
this.checkTypes(aFromType, aToType);
this.addon = this.getAddon(aContext);
this.listener = aListener;
},
onStartRequest(aRequest, aContext) {
this.parts = [];
+ this.decoder = new TextDecoder();
},
onDataAvailable(aRequest, aContext, aInputStream, aOffset, aCount) {
- this.parts.push(NetUtil.readInputStreamToString(aInputStream, aCount));
+ let bytes = NetUtil.readInputStream(aInputStream, aCount);
+ this.parts.push(this.decoder.decode(bytes, {stream: true}));
},
onStopRequest(aRequest, aContext, aStatusCode) {
try {
this.listener.onStartRequest(aRequest, null);
if (Components.isSuccessCode(aStatusCode)) {
+ this.parts.push(this.decoder.decode());
let string = this.parts.join("");
let stream = this.convertToStream(this.addon, string);
- this.listener.onDataAvailable(aRequest, null, stream, 0, stream.data.length);
+ this.listener.onDataAvailable(aRequest, null, stream, 0, stream.available());
}
} catch (e) {
aStatusCode = e.result || Cr.NS_ERROR_FAILURE;
}
this.listener.onStopRequest(aRequest, null, aStatusCode);
},
};