--- a/intl/encoding_glue/Cargo.toml
+++ b/intl/encoding_glue/Cargo.toml
@@ -6,11 +6,11 @@ authors = ["Henri Sivonen <hsivonen@hsiv
license = "MIT/Apache-2.0"
[features]
simd-accel = ["encoding_rs/simd-accel"]
no-static-ideograph-encoder-tables = ["encoding_rs/no-static-ideograph-encoder-tables"]
parallel-utf8 = ["encoding_rs/parallel-utf8"]
[dependencies]
-encoding_rs = "0.6.5"
+encoding_rs = "0.7.0"
nsstring = { path = "../../xpcom/rust/nsstring" }
nserror = { path = "../../xpcom/rust/nserror" }
deleted file mode 100644
--- a/testing/web-platform/meta/encoding/replacement-encodings.html.ini
+++ /dev/null
@@ -1,5 +0,0 @@
-[replacement-encodings.html]
- type: testharness
- [replacement - non-empty input decodes to one replacement character.]
- expected: FAIL
-
--- a/third_party/rust/encoding_c/.cargo-checksum.json
+++ b/third_party/rust/encoding_c/.cargo-checksum.json
@@ -1,1 +1,1 @@
-{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","CONTRIBUTING.md":"8cd9262df951c4b42078aa55064ca3b8ef2676c06b8fc7c281c02ee3f1ae04a8","COPYRIGHT":"65fb11bb8d2aac1ea00620273e0595ff71f4a335d25b67acbccbaa1b9ad5a409","Cargo.toml":"84c1f6d8f47292881a623857617a3f293a0e0e48d5095a992fbf1ffa18defc72","LICENSE-APACHE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","LICENSE-MIT":"90df74ddb43e7f5aa5068890eacc151ecca7c997c9515cf17aea30b8734075bf","README.md":"1ee8a369dd5f599399fa52129c1c41dccda9a53b1152c0e08362f63a2f7d978e","build-disabled.rs":"d65ed45d33ce834ab9f9f7c5f308e0a72605aa34ede6dca45a2077a2deee5cfa","include/encoding_rs.h":"dc015596eb8b4b0b2e79569a519e81c14301db8f5b96b4013989645a67a73422","include/encoding_rs_cpp.h":"f93c0e2b3e1ec4f1efb1fcee1f43e8d1424faf3e26d7084404c5ba5f2f6a2c4d","include/encoding_rs_statics.h":"05561c00809483282daab356113c67c31ff5df5bbe4dbfd79a7fa56cf36bbea7","src/lib.rs":"ad3546071aa9ee7cf2c6735ddbfcb4e77e1e9aa5ddce283bc427d73c3e931655"},"package":"1e563195f16d7439103c49281b8b00bd0f223c16e19f0d500448239a27712570"}
\ No newline at end of file
+{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","CONTRIBUTING.md":"8cd9262df951c4b42078aa55064ca3b8ef2676c06b8fc7c281c02ee3f1ae04a8","COPYRIGHT":"65fb11bb8d2aac1ea00620273e0595ff71f4a335d25b67acbccbaa1b9ad5a409","Cargo.toml":"ff4fa375e664d63f9949265be588ebb23e09cdda7632ac3375d60af8fa8cb185","LICENSE-APACHE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","LICENSE-MIT":"90df74ddb43e7f5aa5068890eacc151ecca7c997c9515cf17aea30b8734075bf","README.md":"089e6895e665a791c081fc7d7d9e953e2d4bb46c3774a050e9c6b0b574beabac","build-disabled.rs":"d65ed45d33ce834ab9f9f7c5f308e0a72605aa34ede6dca45a2077a2deee5cfa","include/encoding_rs.h":"b37624428fa0cdc61d0e93b8b6f1bcf58768f0a8eabd0ae2b606d01627ac9bb3","include/encoding_rs_cpp.h":"8bd8c41b6a919f939f57fd7c733129f3bd4297cc38a6a488899dcac142f7ad83","include/encoding_rs_statics.h":"05561c00809483282daab356113c67c31ff5df5bbe4dbfd79a7fa56cf36bbea7","src/lib.rs":"9299d8052a6af4932290f34d2a7abf5ba4e497bf56941e62cb802c5c058f96bc"},"package":"93ec52324ca72f423237a413ca0e1c60654c8b3d0934fcd5fd888508dfcc4ba7"}
\ No newline at end of file
--- a/third_party/rust/encoding_c/Cargo.toml
+++ b/third_party/rust/encoding_c/Cargo.toml
@@ -1,25 +1,29 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+#
+# When uploading crates to the registry Cargo will automatically
+# "normalize" Cargo.toml files for maximal compatibility
+# with all versions of Cargo and also rewrite `path` dependencies
+# to registry (e.g. crates.io) dependencies
+#
+# If you believe there's an error in this file please file an
+# issue against the rust-lang/cargo repository. If you're
+# editing this file be aware that the upstream Cargo.toml
+# will likely look very different (and much more reasonable)
+
[package]
name = "encoding_c"
-description = "C API for encoding_rs"
-version = "0.7.6"
+version = "0.8.0"
authors = ["Henri Sivonen <hsivonen@hsivonen.fi>"]
-license = "MIT/Apache-2.0"
-readme = "README.md"
+description = "C API for encoding_rs"
+homepage = "https://docs.rs/encoding_c/"
documentation = "https://docs.rs/encoding_c/"
-homepage = "https://docs.rs/encoding_c/"
-repository = "https://github.com/hsivonen/encoding_c"
+readme = "README.md"
keywords = ["ffi", "capi", "encoding", "unicode", "charset"]
-# Uncomment the line below and rename build-disabled.rs to build.rs to re-run cheddar.
-# build = "build.rs"
+license = "MIT/Apache-2.0"
+repository = "https://github.com/hsivonen/encoding_c"
+[dependencies.encoding_rs]
+version = "0.7.0"
[features]
simd-accel = ["encoding_rs/simd-accel"]
no-static-ideograph-encoder-tables = ["encoding_rs/no-static-ideograph-encoder-tables"]
-parallel-utf8 = ["encoding_rs/parallel-utf8"]
-
-[dependencies]
-encoding_rs = "0.6.11"
-
-# Uncomment the lines below to re-run cheddar.
-# [build-dependencies]
-# rusty-cheddar = "0.3.3"
--- a/third_party/rust/encoding_c/README.md
+++ b/third_party/rust/encoding_c/README.md
@@ -6,28 +6,41 @@
encoding_c is an FFI wrapper for [encoding_rs](https://github.com/hsivonen/encoding_rs).
## Licensing
Please see the file named
[COPYRIGHT](https://github.com/hsivonen/encoding_c/blob/master/COPYRIGHT).
+## No Unwinding Support!
+
+This crate is meant for use in binaries compiled with `panic = 'abort'`, which
+is _required_ for correctness! Unwinding across FFI is Undefined Behavior, and
+this crate does nothing to try to prevent unwinding across the FFI if
+compliled with unwinding enabled.
+
## C/C++ Headers
`include/encoding_rs.h` and `include/encoding_rs_statics.h` are needed for C
usage.
`include/encoding_rs_cpp.h` is a sample C++ API built on top of the C API using
GSL and the C++ standard library. Since C++ project typically roll their own
string classes, etc., it's probably necessary for C++ projects to manually
adapt the header to their replacements of standard-library types.
## Release Notes
+### 0.8.0
+
+* Update to encoding_rs 0.7.0.
+* Drop `encoding_for_name()`.
+* Deal correctly with the `data()` method of `gsl::span` returning `nullptr`.
+
### 0.7.6
* Rename `ENCODING_RS_NON_NULL_CONST_ENCODING_PTR` to
`ENCODING_RS_NOT_NULL_CONST_ENCODING_PTR`. (Not a breaking change,
because defining that macro broke the build previously, so the
macro couldn't have been used.)
* Use the macro only for statics and not for return values.
--- a/third_party/rust/encoding_c/include/encoding_rs.h
+++ b/third_party/rust/encoding_c/include/encoding_rs.h
@@ -100,40 +100,16 @@ ENCODING_RS_ENCODING const* encoding_for
/// optimization for slices within `Option`.
///
/// # Undefined behavior
///
/// UB ensues if `buffer` and `*buffer_len` don't designate a valid memory
/// block of if `buffer` is `NULL`.
ENCODING_RS_ENCODING const* encoding_for_bom(uint8_t const* buffer, size_t* buffer_len);
-/// If the argument matches exactly (case-sensitively; no whitespace
-/// removal performed) the name of an encoding, returns
-/// `const ENCODING_RS_ENCODING*` representing that encoding. Otherwise panics.
-///
-/// The motivating use case for this function is interoperability with
-/// legacy Gecko code that represents encodings as name string instead of
-/// type-safe `ENCODING_RS_ENCODING` objects. Using this function for other purposes is
-/// most likely the wrong thing to do.
-///
-/// `name` must be non-`NULL` even if `name_len` is zero. When `name_len`
-/// is zero, it is OK for `name` to be something non-dereferencable,
-/// such as `0x1`. This is required due to Rust's optimization for slices
-/// within `Option`.
-///
-/// # Panics
-///
-/// Panics if the argument is not the name of an encoding.
-///
-/// # Undefined behavior
-///
-/// UB ensues if `name` and `name_len` don't designate a valid memory block
-/// of if `name` is `NULL`.
-ENCODING_RS_ENCODING const* encoding_for_name(uint8_t const* name, size_t name_len);
-
/// Writes the name of the given `ENCODING_RS_ENCODING` to a caller-supplied buffer as
/// ASCII and returns the number of bytes / ASCII characters written.
///
/// The output is not null-terminated.
///
/// The caller _MUST_ ensure that `name_out` points to a buffer whose length
/// is at least `ENCODING_NAME_MAX_LENGTH` bytes.
///
--- a/third_party/rust/encoding_c/include/encoding_rs_cpp.h
+++ b/third_party/rust/encoding_c/include/encoding_rs_cpp.h
@@ -13,19 +13,21 @@
#define encoding_rs_cpp_h_
#include "gsl/gsl"
#include <experimental/optional>
#include <memory>
#include <string>
#include <tuple>
-#include "encoding_rs.h"
+class Encoding;
+class Decoder;
+class Encoder;
-class Encoding;
+#include "encoding_rs.h"
/**
* A converter that decodes a byte stream into Unicode according to a
* character encoding in a streaming (incremental) manner.
*
* The various `decode_*` methods take an input buffer (`src`) and an output
* buffer `dst` both of which are caller-allocated. There are variants for
* both UTF-8 and UTF-16 output buffers.
@@ -178,17 +180,18 @@ public:
*/
inline std::tuple<uint32_t, size_t, size_t, bool> decode_to_utf8(
gsl::span<const uint8_t> src, gsl::span<uint8_t> dst, bool last)
{
size_t src_read = src.size();
size_t dst_written = dst.size();
bool had_replacements;
uint32_t result =
- decoder_decode_to_utf8(this, src.data(), &src_read, dst.data(),
+ decoder_decode_to_utf8(this, null_to_bogus<const uint8_t>(src.data()),
+ &src_read, null_to_bogus<uint8_t>(dst.data()),
&dst_written, last, &had_replacements);
return std::make_tuple(result, src_read, dst_written, had_replacements);
}
/**
* Incrementally decode a byte stream into UTF-8 _without replacement_.
*
* See the documentation of the class for documentation for `decode_*`
@@ -196,17 +199,18 @@ public:
*/
inline std::tuple<uint32_t, size_t, size_t>
decode_to_utf8_without_replacement(gsl::span<const uint8_t> src,
gsl::span<uint8_t> dst, bool last)
{
size_t src_read = src.size();
size_t dst_written = dst.size();
uint32_t result = decoder_decode_to_utf8_without_replacement(
- this, src.data(), &src_read, dst.data(), &dst_written, last);
+ this, null_to_bogus<const uint8_t>(src.data()), &src_read,
+ null_to_bogus<uint8_t>(dst.data()), &dst_written, last);
return std::make_tuple(result, src_read, dst_written);
}
/**
* Query the worst-case UTF-16 output size (with or without replacement).
*
* Returns the size of the output buffer in UTF-16 code units (`char16_t`)
* that will not overflow given the current state of the decoder and
@@ -231,17 +235,18 @@ public:
*/
inline std::tuple<uint32_t, size_t, size_t, bool> decode_to_utf16(
gsl::span<const uint8_t> src, gsl::span<char16_t> dst, bool last)
{
size_t src_read = src.size();
size_t dst_written = dst.size();
bool had_replacements;
uint32_t result =
- decoder_decode_to_utf16(this, src.data(), &src_read, dst.data(),
+ decoder_decode_to_utf16(this, null_to_bogus<const uint8_t>(src.data()),
+ &src_read, null_to_bogus<char16_t>(dst.data()),
&dst_written, last, &had_replacements);
return std::make_tuple(result, src_read, dst_written, had_replacements);
}
/**
* Incrementally decode a byte stream into UTF-16 _without replacement_.
*
* See the documentation of the class for documentation for `decode_*`
@@ -249,22 +254,35 @@ public:
*/
inline std::tuple<uint32_t, size_t, size_t>
decode_to_utf16_without_replacement(gsl::span<const uint8_t> src,
gsl::span<char16_t> dst, bool last)
{
size_t src_read = src.size();
size_t dst_written = dst.size();
uint32_t result = decoder_decode_to_utf16_without_replacement(
- this, src.data(), &src_read, dst.data(), &dst_written, last);
+ this, null_to_bogus<const uint8_t>(src.data()), &src_read,
+ null_to_bogus<char16_t>(dst.data()), &dst_written, last);
return std::make_tuple(result, src_read, dst_written);
}
private:
+ /**
+ * Replaces `nullptr` with a bogus pointer suitable for use as part of a
+ * zero-length Rust slice.
+ */
+ template <class T>
+ static inline T* null_to_bogus(T* ptr)
+ {
+ return ptr ? ptr : reinterpret_cast<T*>(0x1);
+ }
+
Decoder() = delete;
+ Decoder(const Decoder&) = delete;
+ Decoder& operator=(const Decoder&) = delete;
};
/**
* A converter that encodes a Unicode stream into bytes according to a
* character encoding in a streaming (incremental) manner.
*
* The various `encode_*` methods take an input buffer (`src`) and an output
* buffer `dst` both of which are caller-allocated. There are variants for
@@ -433,17 +451,18 @@ public:
*/
inline std::tuple<uint32_t, size_t, size_t, bool> encode_from_utf8(
gsl::span<const uint8_t> src, gsl::span<uint8_t> dst, bool last)
{
size_t src_read = src.size();
size_t dst_written = dst.size();
bool had_replacements;
uint32_t result =
- encoder_encode_from_utf8(this, src.data(), &src_read, dst.data(),
+ encoder_encode_from_utf8(this, null_to_bogus<const uint8_t>(src.data()),
+ &src_read, null_to_bogus<uint8_t>(dst.data()),
&dst_written, last, &had_replacements);
return std::make_tuple(result, src_read, dst_written, had_replacements);
}
/**
* Incrementally encode into byte stream from UTF-8 _without replacement_.
*
* See the documentation of the class for documentation for `encode_*`
@@ -451,17 +470,18 @@ public:
*/
inline std::tuple<uint32_t, size_t, size_t>
encode_from_utf8_without_replacement(gsl::span<const uint8_t> src,
gsl::span<uint8_t> dst, bool last)
{
size_t src_read = src.size();
size_t dst_written = dst.size();
uint32_t result = encoder_encode_from_utf8_without_replacement(
- this, src.data(), &src_read, dst.data(), &dst_written, last);
+ this, null_to_bogus<const uint8_t>(src.data()), &src_read,
+ null_to_bogus<uint8_t>(dst.data()), &dst_written, last);
return std::make_tuple(result, src_read, dst_written);
}
/**
* Query the worst-case output size when encoding from UTF-16 with
* replacement.
*
* Returns the size of the output buffer in bytes that will not overflow
@@ -500,17 +520,18 @@ public:
*/
inline std::tuple<uint32_t, size_t, size_t, bool> encode_from_utf16(
gsl::span<const char16_t> src, gsl::span<uint8_t> dst, bool last)
{
size_t src_read = src.size();
size_t dst_written = dst.size();
bool had_replacements;
uint32_t result =
- encoder_encode_from_utf16(this, src.data(), &src_read, dst.data(),
+ encoder_encode_from_utf16(this, null_to_bogus<const char16_t>(src.data()),
+ &src_read, null_to_bogus<uint8_t>(dst.data()),
&dst_written, last, &had_replacements);
return std::make_tuple(result, src_read, dst_written, had_replacements);
}
/**
* Incrementally encode into byte stream from UTF-16 _without replacement_.
*
* See the documentation of the class for documentation for `encode_*`
@@ -518,22 +539,35 @@ public:
*/
inline std::tuple<uint32_t, size_t, size_t>
encode_from_utf16_without_replacement(gsl::span<const char16_t> src,
gsl::span<uint8_t> dst, bool last)
{
size_t src_read = src.size();
size_t dst_written = dst.size();
uint32_t result = encoder_encode_from_utf16_without_replacement(
- this, src.data(), &src_read, dst.data(), &dst_written, last);
+ this, null_to_bogus<const char16_t>(src.data()), &src_read,
+ null_to_bogus<uint8_t>(dst.data()), &dst_written, last);
return std::make_tuple(result, src_read, dst_written);
}
private:
+ /**
+ * Replaces `nullptr` with a bogus pointer suitable for use as part of a
+ * zero-length Rust slice.
+ */
+ template <class T>
+ static inline T* null_to_bogus(T* ptr)
+ {
+ return ptr ? ptr : reinterpret_cast<T*>(0x1);
+ }
+
Encoder() = delete;
+ Encoder(const Encoder&) = delete;
+ Encoder& operator=(const Encoder&) = delete;
};
/**
* An encoding as defined in the Encoding Standard
* (https://encoding.spec.whatwg.org/).
*
* An _encoding_ defines a mapping from a byte sequence to a Unicode code point
* sequence and, in most cases, vice versa. Each encoding has a name, an output
@@ -594,39 +628,44 @@ public:
* This is the right method to use if the action upon the method returning
* `nullptr` is to use a fallback encoding (e.g. `WINDOWS_1252_ENCODING`)
* instead. When the action upon the method returning `nullptr` is not to
* proceed with a fallback but to refuse processing,
* `for_label_no_replacement()` is more appropriate.
*/
static inline const Encoding* for_label(gsl::cstring_span<> label)
{
- return encoding_for_label(reinterpret_cast<const uint8_t*>(label.data()),
+ return encoding_for_label(null_to_bogus<const uint8_t>(
+ reinterpret_cast<const uint8_t*>(label.data())),
label.length());
}
/**
* This method behaves the same as `for_label()`, except when `for_label()`
* would return `REPLACEMENT_ENCODING`, this method returns `nullptr` instead.
*
* This method is useful in scenarios where a fatal error is required
* upon invalid label, because in those cases the caller typically wishes
* to treat the labels that map to the replacement encoding as fatal
* errors, too.
*
* It is not OK to use this method when the action upon the method returning
* `nullptr` is to use a fallback encoding (e.g. `WINDOWS_1252_ENCODING`). In
- * such a case, the `for_label()` method should be used instead in order to avoid
- * unsafe fallback for labels that `for_label()` maps to `REPLACEMENT_ENCODING`.
+ * such a case, the `for_label()` method should be used instead in order to
+ * avoid
+ * unsafe fallback for labels that `for_label()` maps to
+ * `REPLACEMENT_ENCODING`.
*/
static inline const Encoding* for_label_no_replacement(
gsl::cstring_span<> label)
{
return encoding_for_label_no_replacement(
- reinterpret_cast<const uint8_t*>(label.data()), label.length());
+ null_to_bogus<const uint8_t>(
+ reinterpret_cast<const uint8_t*>(label.data())),
+ label.length());
}
/**
* Performs non-incremental BOM sniffing.
*
* The argument must either be a buffer representing the entire input
* stream (non-streaming case) or a buffer representing at least the first
* three bytes of the input stream (streaming case).
@@ -634,42 +673,22 @@ public:
* Returns `make_tuple(UTF_8_ENCODING, 3)`, `make_tuple(UTF_16LE_ENCODING, 2)`
* or `make_tuple(UTF_16BE_ENCODING, 3)` if the argument starts with the
* UTF-8, UTF-16LE or UTF-16BE BOM or `make_tuple(nullptr, 0)` otherwise.
*/
static inline std::tuple<const Encoding*, size_t> for_bom(
gsl::span<const uint8_t> buffer)
{
size_t len = buffer.size();
- const Encoding* encoding = encoding_for_bom(buffer.data(), &len);
+ const Encoding* encoding =
+ encoding_for_bom(null_to_bogus<const uint8_t>(buffer.data()), &len);
return std::make_tuple(encoding, len);
}
/**
- * If the argument matches exactly (case-sensitively; no whitespace
- * removal performed) the name of an encoding, returns
- * `const Encoding*` representing that encoding. Otherwise panics.
- *
- * The motivating use case for this method is interoperability with
- * legacy Gecko code that represents encodings as name string instead of
- * type-safe `Encoding` objects. Using this method for other purposes is
- * most likely the wrong thing to do.
- *
- * # Panics
- *
- * Panics if the argument is not the name of an encoding.
- */
- static inline gsl::not_null<const Encoding*> for_name(
- gsl::cstring_span<> name)
- {
- return encoding_for_name(reinterpret_cast<const uint8_t*>(name.data()),
- name.length());
- }
-
- /**
* Returns the name of this encoding.
*
* This name is appropriate to return as-is from the DOM
* `document.characterSet` property.
*/
inline std::string name() const
{
std::string name(ENCODING_NAME_MAX_LENGTH, '\0');
@@ -837,17 +856,18 @@ public:
* a segment of the input instead of the whole input. Use
* `new_decoder_without_bom_handling()` when decoding segmented input.
*/
inline std::experimental::optional<std::string>
decode_without_bom_handling_and_without_replacement(
gsl::span<const uint8_t> bytes) const
{
auto decoder = new_decoder_without_bom_handling();
- size_t needed = decoder->max_utf8_buffer_length_without_replacement(bytes.size());
+ size_t needed =
+ decoder->max_utf8_buffer_length_without_replacement(bytes.size());
if (needed == SIZE_MAX) {
throw std::overflow_error("Overflow in buffer size computation.");
}
std::string string(needed, '\0');
uint32_t result;
size_t read;
size_t written;
std::tie(result, read, written) =
@@ -962,19 +982,17 @@ public:
throw std::overflow_error("Overflow in buffer size computation.");
}
std::u16string string(needed, '\0');
uint32_t result;
size_t read;
size_t written;
bool had_errors;
std::tie(result, read, written, had_errors) = decoder->decode_to_utf16(
- bytes,
- gsl::make_span(&string[0], string.size()),
- true);
+ bytes, gsl::make_span(&string[0], string.size()), true);
assert(read == static_cast<size_t>(bytes.size()));
assert(written <= static_cast<size_t>(string.size()));
assert(result == INPUT_EMPTY);
string.resize(written);
return std::make_tuple(string, had_errors);
}
/**
@@ -1005,19 +1023,17 @@ public:
throw std::overflow_error("Overflow in buffer size computation.");
}
std::u16string string(needed, '\0');
uint32_t result;
size_t read;
size_t written;
std::tie(result, read, written) =
decoder->decode_to_utf16_without_replacement(
- bytes,
- gsl::make_span(&string[0], string.size()),
- true);
+ bytes, gsl::make_span(&string[0], string.size()), true);
assert(result != OUTPUT_FULL);
if (result == INPUT_EMPTY) {
assert(read == static_cast<size_t>(bytes.size()));
assert(written <= static_cast<size_t>(string.size()));
string.resize(written);
return string;
}
}
@@ -1047,31 +1063,33 @@ public:
gsl::span<const uint8_t> string) const
{
auto output_enc = output_encoding();
if (output_enc == UTF_8_ENCODING) {
std::vector<uint8_t> vec(string.size());
std::memcpy(&vec[0], string.data(), string.size());
}
auto encoder = output_enc->new_encoder();
- size_t needed = encoder->max_buffer_length_from_utf8_if_no_unmappables(string.size());
+ size_t needed =
+ encoder->max_buffer_length_from_utf8_if_no_unmappables(string.size());
if (needed == SIZE_MAX) {
throw std::overflow_error("Overflow in buffer size computation.");
}
std::vector<uint8_t> vec(needed);
bool total_had_errors = false;
size_t total_read = 0;
size_t total_written = 0;
uint32_t result;
size_t read;
size_t written;
bool had_errors;
for (;;) {
std::tie(result, read, written, had_errors) = encoder->encode_from_utf8(
- gsl::make_span(string).subspan(total_read), gsl::make_span(vec).subspan(total_written), true);
+ gsl::make_span(string).subspan(total_read),
+ gsl::make_span(vec).subspan(total_written), true);
total_read += read;
total_written += written;
total_had_errors |= had_errors;
if (result == INPUT_EMPTY) {
assert(total_read == static_cast<size_t>(string.size()));
assert(total_written <= static_cast<size_t>(vec.size()));
vec.resize(total_written);
return std::make_tuple(vec, output_enc, total_had_errors);
@@ -1106,31 +1124,33 @@ public:
* a segment of the input instead of the whole input. Use `new_encoder()`
* when encoding segmented output.
*/
inline std::tuple<std::vector<uint8_t>, const Encoding*, bool> encode(
gsl::span<const char16_t> string) const
{
auto output_enc = output_encoding();
auto encoder = output_enc->new_encoder();
- size_t needed = encoder->max_buffer_length_from_utf16_if_no_unmappables(string.size());
+ size_t needed =
+ encoder->max_buffer_length_from_utf16_if_no_unmappables(string.size());
if (needed == SIZE_MAX) {
throw std::overflow_error("Overflow in buffer size computation.");
}
std::vector<uint8_t> vec(needed);
bool total_had_errors = false;
size_t total_read = 0;
size_t total_written = 0;
uint32_t result;
size_t read;
size_t written;
bool had_errors;
for (;;) {
std::tie(result, read, written, had_errors) = encoder->encode_from_utf16(
- gsl::make_span(string).subspan(total_read), gsl::make_span(vec).subspan(total_written), true);
+ gsl::make_span(string).subspan(total_read),
+ gsl::make_span(vec).subspan(total_written), true);
total_read += read;
total_written += written;
total_had_errors |= had_errors;
if (result == INPUT_EMPTY) {
assert(total_read == static_cast<size_t>(string.size()));
assert(total_written <= static_cast<size_t>(vec.size()));
vec.resize(total_written);
return std::make_tuple(vec, output_enc, total_had_errors);
@@ -1221,17 +1241,18 @@ public:
* Instantiates a new decoder for this encoding with BOM handling disabled
* into memory occupied by a previously-instantiated decoder.
*
* If the input starts with bytes that look like a BOM, those bytes are
* not treated as a BOM. (Hence, the decoder never morphs into a decoder
* for another encoding.)
*
* _Note:_ If the caller has performed BOM sniffing on its own but has not
- * removed the BOM, the caller should use `new_decoder_with_bom_removal_into()`
+ * removed the BOM, the caller should use
+ * `new_decoder_with_bom_removal_into()`
* instead of this method to cause the BOM to be removed.
*/
inline void new_decoder_without_bom_handling_into(Decoder& decoder) const
{
encoding_new_decoder_without_bom_handling_into(this, &decoder);
}
/**
@@ -1255,42 +1276,57 @@ public:
/**
* Validates UTF-8.
*
* Returns the index of the first byte that makes the input malformed as
* UTF-8 or the length of the input if the input is entirely valid.
*/
static inline size_t utf8_valid_up_to(gsl::span<const uint8_t> buffer)
{
- return encoding_utf8_valid_up_to(buffer.data(), buffer.size());
+ return encoding_utf8_valid_up_to(
+ null_to_bogus<const uint8_t>(buffer.data()), buffer.size());
}
/**
* Validates ASCII.
*
* Returns the index of the first byte that makes the input malformed as
* ASCII or the length of the input if the input is entirely valid.
*/
static inline size_t ascii_valid_up_to(gsl::span<const uint8_t> buffer)
{
- return encoding_ascii_valid_up_to(buffer.data(), buffer.size());
+ return encoding_ascii_valid_up_to(
+ null_to_bogus<const uint8_t>(buffer.data()), buffer.size());
}
/**
* Validates ISO-2022-JP ASCII-state data.
*
* Returns the index of the first byte that makes the input not
* representable in the ASCII state of ISO-2022-JP or the length of the
* input if the input is entirely representable in the ASCII state of
* ISO-2022-JP.
*/
static inline size_t iso_2022_jp_ascii_valid_up_to(
gsl::span<const uint8_t> buffer)
{
- return encoding_iso_2022_jp_ascii_valid_up_to(buffer.data(), buffer.size());
+ return encoding_iso_2022_jp_ascii_valid_up_to(
+ null_to_bogus<const uint8_t>(buffer.data()), buffer.size());
}
private:
+ /**
+ * Replaces `nullptr` with a bogus pointer suitable for use as part of a
+ * zero-length Rust slice.
+ */
+ template <class T>
+ static inline T* null_to_bogus(T* ptr)
+ {
+ return ptr ? ptr : reinterpret_cast<T*>(0x1);
+ }
+
Encoding() = delete;
+ Encoding(const Encoding&) = delete;
+ Encoding& operator=(const Encoding&) = delete;
~Encoding() = delete;
};
#endif // encoding_rs_cpp_h_
--- a/third_party/rust/encoding_c/src/lib.rs
+++ b/third_party/rust/encoding_c/src/lib.rs
@@ -2,29 +2,29 @@
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
-#![doc(html_root_url = "https://docs.rs/encoding_c/0.7.6")]
+#![doc(html_root_url = "https://docs.rs/encoding_c/0.8.0")]
//! The C API for encoding_rs.
//!
//! # Mapping from Rust
//!
//! ## Naming convention
//!
//! The wrapper function for each method has a name that starts with the name
//! of the struct lower-cased, followed by an underscore and ends with the
//! name of the method.
//!
-//! For example, `Encoding::for_name()` is wrapped as `encoding_for_name()`.
+//! For example, `Encoding::for_label()` is wrapped as `encoding_for_label()`.
//!
//! ## Arguments
//!
//! Functions that wrap non-static methods take the `self` object as their
//! first argument.
//!
//! Slice argument `foo` is decomposed into a pointer `foo` and a length
//! `foo_len`.
@@ -364,44 +364,16 @@ pub unsafe extern "C" fn encoding_for_bo
let (encoding, bom_length) = match Encoding::for_bom(buffer_slice) {
Some((encoding, bom_length)) => (encoding as *const Encoding, bom_length),
None => (::std::ptr::null(), 0),
};
*buffer_len = bom_length;
encoding
}
-/// If the argument matches exactly (case-sensitively; no whitespace
-/// removal performed) the name of an encoding, returns
-/// `const Encoding*` representing that encoding. Otherwise panics.
-///
-/// The motivating use case for this function is interoperability with
-/// legacy Gecko code that represents encodings as name string instead of
-/// type-safe `Encoding` objects. Using this function for other purposes is
-/// most likely the wrong thing to do.
-///
-/// `name` must be non-`NULL` even if `name_len` is zero. When `name_len`
-/// is zero, it is OK for `name` to be something non-dereferencable,
-/// such as `0x1`. This is required due to Rust's optimization for slices
-/// within `Option`.
-///
-/// # Panics
-///
-/// Panics if the argument is not the name of an encoding.
-///
-/// # Undefined behavior
-///
-/// UB ensues if `name` and `name_len` don't designate a valid memory block
-/// of if `name` is `NULL`.
-#[no_mangle]
-pub unsafe extern "C" fn encoding_for_name(name: *const u8, name_len: usize) -> *const Encoding {
- let name_slice = ::std::slice::from_raw_parts(name, name_len);
- Encoding::for_name(name_slice)
-}
-
/// Writes the name of the given `Encoding` to a caller-supplied buffer as
/// ASCII and returns the number of bytes / ASCII characters written.
///
/// The output is not null-terminated.
///
/// The caller _MUST_ ensure that `name_out` points to a buffer whose length
/// is at least `ENCODING_NAME_MAX_LENGTH` bytes.
///
--- a/third_party/rust/encoding_rs/.cargo-checksum.json
+++ b/third_party/rust/encoding_rs/.cargo-checksum.json
@@ -1,1 +1,1 @@
-{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",".travis.yml":"4d1af7257c9619f7ae66fc271ba2c1be5f063640ae8ceaa235c8c8aaf32f44ea","CONTRIBUTING.md":"e4ffa92c979c7e6ca7b676842a708ea05b84181327fcde43dfcd8038b678a057","COPYRIGHT":"20d4fff11cca11529df3f02096fbe8ffe350219cdb07cdedea34e6a762866da5","Cargo.toml":"b74676e1affb0a2b528507be488bd9588db646b3b05807dada63cbe7b0747fc6","Ideas.md":"c1be4cc91621f52f38ea7febda7a4bb68086189cacc834c7edac4ba1a9da02fe","LICENSE-APACHE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","LICENSE-MIT":"74aa8b6d04c36bb640ee81187a3f24a2fa94e36d4c1d4f2ca164c3784ae87a83","README.md":"276d801faf4d6347c3ea32ae252cab95df653c846beaac535c5d70cf32094f5e","generate-encoding-data.py":"0b62de6d3b6368166b78a9259f06dc8d0f558504a0ed866dbe75dc2efb4bf464","rustfmt.toml":"c01c06dfbdfcf30730535aab911d69068febb921e2faef9571ceeb6a5c2a3eab","src/ascii.rs":"240c607d3bad850c57d1e96871d5c0371278ed3923c38354bbb4c8a876c9a515","src/big5.rs":"614d479aabc63007f778d1f776a37b885e13d20b7c6c7a2818a729bde342f8a6","src/data.rs":"412c842c698c3ce1cec4a27ab19ca275372ac28940ac49cdf3e0dad71a2c2812","src/euc_jp.rs":"feda0ade5e1c3e4abd7637c59373b977662007990fd164ea7db1acc502ba3534","src/euc_kr.rs":"23e08359ccbe7602f3a90fce78dc76fd4065c236820ac0d11c9d9325045da0e6","src/gb18030.rs":"aa9de27a41715dfb02a3b9161d86e3775f635f625f70d3abaadcd583ee7022c0","src/handles.rs":"8b0691ab21d638bd20078e33247f13afbc8012ff4b843a2fd03e3314353e8520","src/iso_2022_jp.rs":"285e7cea6df41d182a345a0f394a2348b1c313f0d55ed48c349824f2a6aff526","src/lib.rs":"dad6465f541ccdb171312879999d842dcbf11bc09119d81963df3a20f7d4e474","src/macros.rs":"9ab30e7194f61f268cd7d899cabb06ff9ca7717663926fd583b20334f49ac8d3","src/replacement.rs":"782f03f04d110e9a0656262bf4296aa0ab8199e196cb63239c30d9649996caa4","src/shift_jis.rs":"84df4ff58b60e0827d6c0c7049f2cf19033f2b9e25a9186bcfb0bbb05e87b380","src/simd_funcs.rs":"ff30e10bfb58fb8f56f0cc0b4dbcc4af6b343487562ee279ace8b31afd7bcccc","src/single_byte.rs":"0342a921427ed160f5cbe4532490aff5db00886a36b70273f54d8f6a9dcf6974","src/test_data/big5_in.txt":"4c5a8691f8dc717311889c63894026d2fb62725a86c4208ca274a9cc8d42a503","src/test_data/big5_in_ref.txt":"99d399e17750cf9c7cf30bb253dbfe35b81c4fcbdead93cfa48b1429213473c7","src/test_data/big5_out.txt":"6193ca97c297aa20e09396038d18e938bb7ea331c26f0f2454097296723a0b13","src/test_data/big5_out_ref.txt":"36567691f557df144f6cc520015a87038dfa156f296fcf103b56ae9a718be1fc","src/test_data/euc_kr_in.txt":"c86a7224f3215fa0d04e685622a752fdc72763e8ae076230c7fd62de57ec4074","src/test_data/euc_kr_in_ref.txt":"1f419f4ca47d708b54c73c461545a022ae2e20498fdbf8005a483d752a204883","src/test_data/euc_kr_out.txt":"e7f32e026f70be1e1b58e0047baf7d3d2c520269c4f9b9992e158b4decb0a1a3","src/test_data/euc_kr_out_ref.txt":"c9907857980b20b8e9e3b584482ed6567a2be6185d72237b6322f0404944924e","src/test_data/gb18030_in.txt":"ab7231b2d3e9afacdbd7d7f3b9e5361a7ff9f7e1cfdb4f3bd905b9362b309e53","src/test_data/gb18030_in_ref.txt":"dc5069421adca2043c55f5012b55a76fdff651d22e6e699fd0978f8d5706815c","src/test_data/gb18030_out.txt":"f0208d527f5ca63de7d9a0323be8d5cf12d8a104b2943d92c2701f0c3364dac1","src/test_data/gb18030_out_ref.txt":"6819fe47627e4ea01027003fc514b9f21a1322e732d7f1fb92cc6c5455bc6c07","src/test_data/iso_2022_jp_in.txt":"cd24bbdcb1834e25db54646fbf4c41560a13dc7540f6be3dba4f5d97d44513af","src/test_data/iso_2022_jp_in_ref.txt":"3dc4e6a5e06471942d086b16c9440945e78415f6f3f47e43717e4bc2eac2cdf5","src/test_data/iso_2022_jp_out.txt":"9b6f015329dda6c3f9ee5ce6dbd6fa9c89acc21283e886836c78b8d833480c21","src/test_data/iso_2022_jp_out_ref.txt":"78cb260093a20116ad9a42f43b05d1848c5ab100b6b9a850749809e943884b35","src/test_data/jis0208_in.txt":"6df3030553ffb0a6615bb33dc8ea9dca6d9623a9028e2ffec754ce3c3da824cc","src/test_data/jis0208_in_ref.txt":"3dc4e6a5e06471942d086b16c9440945e78415f6f3f47e43717e4bc2eac2cdf5","src/test_data/jis0208_out.txt":"4ec24477e1675ce750733bdc3c5add1cd27b6bd4ce1f09289564646e9654e857","src/test_data/jis0208_out_ref.txt":"c3e1cef5032b2b1d93a406f31ff940c4e2dfe8859b8b17ca2761fee7a75a0e48","src/test_data/jis0212_in.txt":"c011f0dd72bd7c8cd922df9374ef8d2769a77190514c77f6c62b415852eeb9fe","src/test_data/jis0212_in_ref.txt":"7d9458b3d2f73e7092a7f505c08ce1d233dde18aa679fbcf9889256239cc9e06","src/test_data/shift_jis_in.txt":"02e389ccef0dd2122e63f503899402cb7f797912c2444cc80ab93131116c5524","src/test_data/shift_jis_in_ref.txt":"512f985950ca902e643c88682dba9708b7c38d3c5ec2925168ab00ac94ab19f9","src/test_data/shift_jis_out.txt":"5fbc44da7bf639bf6cfe0fa1fd3eba7102b88f81919c9ea991302712f69426fb","src/test_data/shift_jis_out_ref.txt":"466322c6fed8286c64582731755290c2296508efdd258826e6279686649b481f","src/test_labels_names.rs":"0bcf7eeb8bb33cbc88bd3bd8462437501a43055db02c40a12a15fae8e68dd1cb","src/testing.rs":"60f85c6fb63fd4ab62e90dfa005920e79e0e1885795dc13a7a3c1980507925b1","src/utf_16.rs":"8155c1c0acaab2826ee9f99ba911fbd3125707d797327e630977bc2f3f9b1064","src/utf_8.rs":"14cd64de87d8fc5f814f52f76390bda0b4c705da98e73e376fb424ca02119ba5","src/utf_8_core.rs":"0229de223eef17ad16751a646bcd3839c24f24069d660a4dc61b8a5fad19d16f","src/variant.rs":"93dfec2dcfc9fd9711bb55d48177f4a0e9479c7fbd055f08db3853338569da83","src/x_user_defined.rs":"420fae797ea94e7a51eb005b97621ab32d68a8532c565afc60ecce6bdd84b6bd"},"package":"e00a1b1e95eb46988805ceee6f34cd95c46a6753e290cb3ff0486931989d4a4c"}
\ No newline at end of file
+{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",".travis.yml":"dc509cc3b8f44fbdf1d806f533c3f005afaf0fd77cd266b38cb69bab3e4ea136","CONTRIBUTING.md":"e4ffa92c979c7e6ca7b676842a708ea05b84181327fcde43dfcd8038b678a057","COPYRIGHT":"20d4fff11cca11529df3f02096fbe8ffe350219cdb07cdedea34e6a762866da5","Cargo.toml":"3865d683e7d7fc4a0e05f2eaf9f808ea6b550eb24fe4f99377749bc1185195bb","Ideas.md":"c1be4cc91621f52f38ea7febda7a4bb68086189cacc834c7edac4ba1a9da02fe","LICENSE-APACHE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","LICENSE-MIT":"74aa8b6d04c36bb640ee81187a3f24a2fa94e36d4c1d4f2ca164c3784ae87a83","README.md":"7d93a6ff036022bd986fc608ea5cda33cd3e865cd85c7ad55494336304d9163e","generate-encoding-data.py":"8a0a5162098d355e4df63532819769fd6626a66a0aa93f2762e315d6147aa0a5","rustfmt.toml":"c01c06dfbdfcf30730535aab911d69068febb921e2faef9571ceeb6a5c2a3eab","src/ascii.rs":"a74affce835efdf930256ee7153fc29e08794df369b895df262b1920bff29b5a","src/big5.rs":"614d479aabc63007f778d1f776a37b885e13d20b7c6c7a2818a729bde342f8a6","src/data.rs":"412c842c698c3ce1cec4a27ab19ca275372ac28940ac49cdf3e0dad71a2c2812","src/euc_jp.rs":"feda0ade5e1c3e4abd7637c59373b977662007990fd164ea7db1acc502ba3534","src/euc_kr.rs":"23e08359ccbe7602f3a90fce78dc76fd4065c236820ac0d11c9d9325045da0e6","src/gb18030.rs":"aa9de27a41715dfb02a3b9161d86e3775f635f625f70d3abaadcd583ee7022c0","src/handles.rs":"8b0691ab21d638bd20078e33247f13afbc8012ff4b843a2fd03e3314353e8520","src/iso_2022_jp.rs":"285e7cea6df41d182a345a0f394a2348b1c313f0d55ed48c349824f2a6aff526","src/lib.rs":"b2898ab31ccc5626474bdcb18e9447bc53ba1c225d8fb4deecadf59e0d6d09d8","src/macros.rs":"9ab30e7194f61f268cd7d899cabb06ff9ca7717663926fd583b20334f49ac8d3","src/replacement.rs":"782f03f04d110e9a0656262bf4296aa0ab8199e196cb63239c30d9649996caa4","src/shift_jis.rs":"84df4ff58b60e0827d6c0c7049f2cf19033f2b9e25a9186bcfb0bbb05e87b380","src/simd_funcs.rs":"a2def7fc35fa8d3eeac1d7fbcd005d50b018a336a2fa7cfd42bfdcf04e564008","src/single_byte.rs":"0342a921427ed160f5cbe4532490aff5db00886a36b70273f54d8f6a9dcf6974","src/test_data/big5_in.txt":"4c5a8691f8dc717311889c63894026d2fb62725a86c4208ca274a9cc8d42a503","src/test_data/big5_in_ref.txt":"99d399e17750cf9c7cf30bb253dbfe35b81c4fcbdead93cfa48b1429213473c7","src/test_data/big5_out.txt":"6193ca97c297aa20e09396038d18e938bb7ea331c26f0f2454097296723a0b13","src/test_data/big5_out_ref.txt":"36567691f557df144f6cc520015a87038dfa156f296fcf103b56ae9a718be1fc","src/test_data/euc_kr_in.txt":"c86a7224f3215fa0d04e685622a752fdc72763e8ae076230c7fd62de57ec4074","src/test_data/euc_kr_in_ref.txt":"1f419f4ca47d708b54c73c461545a022ae2e20498fdbf8005a483d752a204883","src/test_data/euc_kr_out.txt":"e7f32e026f70be1e1b58e0047baf7d3d2c520269c4f9b9992e158b4decb0a1a3","src/test_data/euc_kr_out_ref.txt":"c9907857980b20b8e9e3b584482ed6567a2be6185d72237b6322f0404944924e","src/test_data/gb18030_in.txt":"ab7231b2d3e9afacdbd7d7f3b9e5361a7ff9f7e1cfdb4f3bd905b9362b309e53","src/test_data/gb18030_in_ref.txt":"dc5069421adca2043c55f5012b55a76fdff651d22e6e699fd0978f8d5706815c","src/test_data/gb18030_out.txt":"f0208d527f5ca63de7d9a0323be8d5cf12d8a104b2943d92c2701f0c3364dac1","src/test_data/gb18030_out_ref.txt":"6819fe47627e4ea01027003fc514b9f21a1322e732d7f1fb92cc6c5455bc6c07","src/test_data/iso_2022_jp_in.txt":"cd24bbdcb1834e25db54646fbf4c41560a13dc7540f6be3dba4f5d97d44513af","src/test_data/iso_2022_jp_in_ref.txt":"3dc4e6a5e06471942d086b16c9440945e78415f6f3f47e43717e4bc2eac2cdf5","src/test_data/iso_2022_jp_out.txt":"9b6f015329dda6c3f9ee5ce6dbd6fa9c89acc21283e886836c78b8d833480c21","src/test_data/iso_2022_jp_out_ref.txt":"78cb260093a20116ad9a42f43b05d1848c5ab100b6b9a850749809e943884b35","src/test_data/jis0208_in.txt":"6df3030553ffb0a6615bb33dc8ea9dca6d9623a9028e2ffec754ce3c3da824cc","src/test_data/jis0208_in_ref.txt":"3dc4e6a5e06471942d086b16c9440945e78415f6f3f47e43717e4bc2eac2cdf5","src/test_data/jis0208_out.txt":"4ec24477e1675ce750733bdc3c5add1cd27b6bd4ce1f09289564646e9654e857","src/test_data/jis0208_out_ref.txt":"c3e1cef5032b2b1d93a406f31ff940c4e2dfe8859b8b17ca2761fee7a75a0e48","src/test_data/jis0212_in.txt":"c011f0dd72bd7c8cd922df9374ef8d2769a77190514c77f6c62b415852eeb9fe","src/test_data/jis0212_in_ref.txt":"7d9458b3d2f73e7092a7f505c08ce1d233dde18aa679fbcf9889256239cc9e06","src/test_data/shift_jis_in.txt":"02e389ccef0dd2122e63f503899402cb7f797912c2444cc80ab93131116c5524","src/test_data/shift_jis_in_ref.txt":"512f985950ca902e643c88682dba9708b7c38d3c5ec2925168ab00ac94ab19f9","src/test_data/shift_jis_out.txt":"5fbc44da7bf639bf6cfe0fa1fd3eba7102b88f81919c9ea991302712f69426fb","src/test_data/shift_jis_out_ref.txt":"466322c6fed8286c64582731755290c2296508efdd258826e6279686649b481f","src/test_labels_names.rs":"c962c7aeac3d9ef2aca70c9e21983b231d4cf998cb06879374b0401e5149d1da","src/testing.rs":"60f85c6fb63fd4ab62e90dfa005920e79e0e1885795dc13a7a3c1980507925b1","src/utf_16.rs":"1d2c40857c946f6eecf724efc60a196865b4d84a59b08b42fbe4576fa8308fd0","src/utf_8.rs":"f9b9ea26212598607b925d231d424abdc9d16e9f6b66345ea155406953894b0e","src/utf_8_core.rs":"bbc010dbdfed0f5e7c48a1ab0772eaf2e27711b789bb82f71a678f2240158a65","src/variant.rs":"93dfec2dcfc9fd9711bb55d48177f4a0e9479c7fbd055f08db3853338569da83","src/x_user_defined.rs":"420fae797ea94e7a51eb005b97621ab32d68a8532c565afc60ecce6bdd84b6bd"},"package":"6f0a39f0e2f497d3c2e6a5529a0ec4fc640084fa401493c640421673471f8b72"}
\ No newline at end of file
--- a/third_party/rust/encoding_rs/.travis.yml
+++ b/third_party/rust/encoding_rs/.travis.yml
@@ -1,8 +1,16 @@
language: rust
rust:
+ - 1.15.0 # Earliest that's known to work
+ - 1.19.0 # Firefox requirement
- stable
- beta
- nightly
-matrix:
- allow_failures:
- - rust: nightly
+env:
+ - RUST_BACKTRACE=1
+ - RUST_BACKTRACE=1 RUSTC_BOOTSTRAP=1
+script:
+ - if [[ $RUSTC_BOOTSTRAP ]]; then cargo test --verbose --features 'simd-accel no-static-ideograph-encoder-tables serde'; fi
+ - if [[ -z $RUSTC_BOOTSTRAP ]]; then cargo test --verbose; fi
+#matrix:
+# allow_failures:
+# - rust: nightly
--- a/third_party/rust/encoding_rs/Cargo.toml
+++ b/third_party/rust/encoding_rs/Cargo.toml
@@ -1,28 +1,50 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+#
+# When uploading crates to the registry Cargo will automatically
+# "normalize" Cargo.toml files for maximal compatibility
+# with all versions of Cargo and also rewrite `path` dependencies
+# to registry (e.g. crates.io) dependencies
+#
+# If you believe there's an error in this file please file an
+# issue against the rust-lang/cargo repository. If you're
+# editing this file be aware that the upstream Cargo.toml
+# will likely look very different (and much more reasonable)
+
[package]
name = "encoding_rs"
-description = "A Gecko-oriented implementation of the Encoding Standard"
-version = "0.6.11" # Remember to keep html_root_url in lib.rs in sync!
+version = "0.7.0"
authors = ["Henri Sivonen <hsivonen@hsivonen.fi>"]
-license = "MIT/Apache-2.0"
-readme = "README.md"
+description = "A Gecko-oriented implementation of the Encoding Standard"
+homepage = "https://docs.rs/encoding_rs/"
documentation = "https://docs.rs/encoding_rs/"
-homepage = "https://docs.rs/encoding_rs/"
-repository = "https://github.com/hsivonen/encoding_rs"
+readme = "README.md"
keywords = ["encoding", "web", "unicode", "charset"]
categories = ["text-processing", "encoding", "web-programming", "email"]
+license = "MIT/Apache-2.0"
+repository = "https://github.com/hsivonen/encoding_rs"
+[profile.release]
+lto = true
+[dependencies.cfg-if]
+version = "0.1.0"
-[badges]
-travis-ci = { repository = "hsivonen/encoding_rs" }
+[dependencies.simd]
+version = "0.2.0"
+optional = true
+
+[dependencies.serde]
+version = "1.0"
+optional = true
+[dev-dependencies.serde_derive]
+version = "1.0"
+
+[dev-dependencies.bincode]
+version = "0.8"
+
+[dev-dependencies.serde_json]
+version = "1.0"
[features]
+no-static-ideograph-encoder-tables = []
simd-accel = ["simd"]
-no-static-ideograph-encoder-tables = []
-parallel-utf8 = ["rayon"]
-
-[dependencies]
-cfg-if = "0.1.0"
-simd = { version = "0.2.0", optional = true }
-rayon = { version = "0.7.0", optional = true }
-
-[profile.release]
-lto = true
+[badges.travis-ci]
+repository = "hsivonen/encoding_rs"
--- a/third_party/rust/encoding_rs/README.md
+++ b/third_party/rust/encoding_rs/README.md
@@ -1,76 +1,167 @@
# encoding_rs
[](https://travis-ci.org/hsivonen/encoding_rs)
[](https://crates.io/crates/encoding_rs)
[](https://docs.rs/encoding_rs/)
[](https://github.com/hsivonen/encoding_rs/blob/master/COPYRIGHT)
-encoding_rs aspires to become an implementation of the
-[Encoding Standard](https://encoding.spec.whatwg.org/) that
+encoding_rs an implementation of the (non-JavaScript parts of) the
+[Encoding Standard](https://encoding.spec.whatwg.org/) written in Rust and
+used in Gecko (starting with Firefox 56).
+
+## Functionality
+
+Due to the Gecko use case, encoding_rs supports decoding to and encoding from
+UTF-16 in addition to supporting the usual Rust use case of decoding to and
+encoding from UTF-8. Additionally, the API has been designed to be FFI-friendly
+to accommodate the C++ side of Gecko.
+
+Specifically, encoding_rs does the following:
-1. Is written in Rust.
-2. Is suitable for use in Gecko as a replacement of uconv. (I.e. supports
- decoding to UTF-16 and encoding from UTF-16.)
-3. Is suitable for use in Rust code (both in Gecko and independently of Gecko).
- (I.e. supports decoding to UTF-8 and encoding from UTF-8 and provides an API
- compatible with at least the most common ways of using
- [rust-encoding](https://github.com/lifthrasiir/rust-encoding/).)
+* Decodes a stream of bytes in an Encoding Standard-defined character encoding
+ into valid aligned native-endian in-RAM UTF-16 (units of `u16` / `char16_t`).
+* Encodes a stream of potentially-invalid aligned native-endian in-RAM UTF-16
+ (units of `u16` / `char16_t`) into a sequence of bytes in an Encoding
+ Standard-defined character encoding as if the lone surrogates had been
+ replaced with the REPLACEMENT CHARACTER before performing the encode.
+ (Gecko's UTF-16 is potentially invalid.)
+* Decodes a stream of bytes in an Encoding Standard-defined character
+ encoding into valid UTF-8.
+* Encodes a stream of valid UTF-8 into a sequence of bytes in an Encoding
+ Standard-defined character encoding. (Rust's UTF-8 is guaranteed-valid.)
+* Does the above in streaming (input and output split across multiple
+ buffers) and non-streaming (whole input in a single buffer and whole
+ output in a single buffer) variants.
+* Avoids copying (borrows) when possible in the non-streaming cases when
+ decoding to or encoding from UTF-8.
+* Resolves textual labels that identify character encodings in
+ protocol text into type-safe objects representing the those encodings
+ conceptually.
+* Maps the type-safe encoding objects onto strings suitable for
+ returning from `document.characterSet`.
+* Validates UTF-8 (in common instruction set scenarios a bit faster for Web
+ workloads than the standard library; hopefully will get upstreamed some
+ day) and ASCII.
## Licensing
Please see the file named
[COPYRIGHT](https://github.com/hsivonen/encoding_rs/blob/master/COPYRIGHT).
## API Documentation
Generated [API documentation](https://docs.rs/encoding_rs/) is available
online.
-## Design
+## C and C++ bindings
+
+An FFI layer for encoding_rs is available as a
+[separate crate](https://github.com/hsivonen/encoding_c). The crate comes
+with a [demo C++ wrapper](https://github.com/hsivonen/encoding_c/blob/master/include/encoding_rs_cpp.h)
+using the C++ standard library and [GSL](https://github.com/Microsoft/GSL/) types.
+
+For the Gecko context, there's a
+[C++ wrapper using the MFBT/XPCOM types](https://searchfox.org/mozilla-central/source/intl/Encoding.h#100).
+
+## Optional features
+
+There are currently three optional cargo features:
+
+### `simd-accel`
+
+Enables SSE2 acceleration on x86, x86_64 and Aarch64. Requires nightly Rust.
+_Enabling this cargo feature is recommended when building for x86, x86_64 or
+Aarch64 on nightly Rust._ The intention is for the functionality enabled by
+this feature to become the normal on-by-default behavior once explicit SIMD
+becames available on all Rust release channels.
-For design considerations, please see the associated [technical proposal to
-rewrite uconv in Rust](https://docs.google.com/document/d/13GCbdvKi83a77ZcKOxaEteXp1SOGZ_9Fmztb9iX22v0/edit#).
+Enabling this feature breaks the build unless the target is x86 with SSE2
+(Rust's default 32-bit x86 target, `i686`, has SSE2, but Linux distros may
+use an x86 target without SSE2, i.e. `i586` in `rustup` terms), x86_64 or
+Aarch64.
+
+### `serde`
+
+Enables support for serializing and deserializing `&'static Encoding`-typed
+struct fields using [Serde][1].
+
+[1]: https://serde.rs/
+
+### `no-static-ideograph-encoder-tables`
+
+Makes the binary size smaller at the expense of ideograph _encode_ speed for
+Chinese and Japanese legacy encodings. (Does _not_ affect decode speed.)
+
+The speed resulting from enabling this feature is believed to be acceptable
+for Web browser-exposed encoder use cases. However, the result is likely
+unacceptable for other applications that need to produce output in Chinese or
+Japanese legacy encodings. (But applications really should always be using
+UTF-8 for output.)
## Performance goals
For decoding to UTF-16, the goal is to perform at least as well as Gecko's old
uconv. For decoding to UTF-8, the goal is to perform at least as well as
rust-encoding.
Encoding to UTF-8 should be fast. (UTF-8 to UTF-8 encode should be equivalent
to `memcpy` and UTF-16 to UTF-8 should be fast.)
Speed is a non-goal when encoding to legacy encodings. Encoding to legacy
encodings should not be optimized for speed at the expense of code size as long
as form submission and URL parsing in Gecko don't become noticeably too slow
in real-world use.
+Currently, by default, encoding_rs builds with limited encoder-specific
+accelation tables for GB2312 Level 1 Hanzi, Big5 Level 1 Hanzi and JIS X
+0208 Level 1 Kanji. These tables use binary search and strike a balance
+between not having encoder-specific tables at all (doing linear search
+over the decode-optimized tables) and having larger directly-indexable
+encoder-side tables. It is not clear that anyone wants this in-between
+approach, and it may be changed in the future.
+
+In the interest of binary size, Firefox builds with the
+`no-static-ideograph-encoder-tables` cargo feature, which omits
+the encoder-specific tables and performs linear search over the
+decode-optimized tables. With realistic work loads, this seemed fast enough
+not to be user-visibly slow on Raspberry Pi 3 (which stood in for a phone
+for testing) in the Web-exposed encoder use cases.
+
A framework for measuring performance is [available separately][1].
[1]: https://github.com/hsivonen/encoding_bench/
-## C binding
+## Rust Version Compatibility
+
+It is a goal to support the latest stable Rust, the latest nightly Rust and
+the version of Rust that's used for Firefox Nightly (currently 1.19.0).
+These are tested on Travis.
-An FFI layer for encoding_rs is available as a
-[separate crate](https://github.com/hsivonen/encoding_c).
+Additionally, beta and the oldest known to work Rust version (currently
+1.15.0) are tested on Travis. The oldest Rust known to work is tested as
+a canary so that when the oldest known to work no longer works, the change
+can be documented here. At this time, there is no firm commitment to support
+a version older than what's required by Firefox, but there isn't an active
+plan to make changes that would make 1.15.0 no longer work, either.
## Compatibility with rust-encoding
A compatibility layer that implements the rust-encoding API on top of
encoding_rs is
[provided as a separate crate](https://github.com/hsivonen/encoding_rs_compat)
-(cannot be uploaded to crates.io).
+(cannot be uploaded to crates.io). The compatibility layer was originally
+written with the assuption that Firefox would need it, but it is not currently
+used in Firefox.
## Roadmap
- [x] Design the low-level API.
-- [x] Provide Rust-only convenience features (some BOM sniffing variants still
- TODO).
+- [x] Provide Rust-only convenience features.
- [x] Provide an stl/gsl-flavored C++ API.
- [x] Implement all decoders and encoders.
- [x] Add unit tests for all decoders and encoders.
- [x] Finish BOM sniffing variants in Rust-only convenience features.
- [x] Document the API.
- [x] Publish the crate on crates.io.
- [x] Create a solution for measuring performance.
- [x] Accelerate ASCII conversions using SSE2 on x86.
@@ -78,29 +169,41 @@ encoding_rs is
non-x86 architectures (process an `usize` instead of `u8` at a time).
- [x] Split FFI into a separate crate so that the FFI doesn't interfere with
LTO in pure-Rust usage.
- [x] Compress CJK indices by making use of sequential code points as well
as Unicode-ordered parts of indices.
- [x] Make lookups by label or name use binary search that searches from the
end of the label/name to the start.
- [x] Make labels with non-ASCII bytes fail fast.
-- [x] Parallelize UTF-8 validation using [Rayon](https://github.com/nikomatsakis/rayon).
+- [ ] Parallelize UTF-8 validation using [Rayon](https://github.com/nikomatsakis/rayon).
- [x] Provide an XPCOM/MFBT-flavored C++ API.
- [ ] Investigate accelerating single-byte encode with a single fast-tracked
range per encoding.
-- [ ] Replace uconv with encoding_rs in Gecko.
+- [x] Replace uconv with encoding_rs in Gecko.
- [x] Implement the rust-encoding API in terms of encoding_rs.
-- [ ] Investigate the use of NEON on newer ARM CPUs that have a lesser penalty
- on data flow from NEON to ALU registers.
+- [ ] Add SIMD acceleration for Aarch64.
+- [ ] Investigate the use of NEON on 32-bit ARM.
- [ ] Investigate Björn Höhrmann's lookup table acceleration for UTF-8 as
adapted to Rust in rust-encoding.
## Release Notes
+### 0.7.0
+
+* [Make `replacement` a label of the replacement
+ encoding.](https://github.com/whatwg/encoding/issues/70) (Spec change.)
+* Remove `Encoding::for_name()`. (`Encoding::for_label(foo).unwrap()` is
+ now close enough after the above label change.)
+* Remove the `parallel-utf8` cargo feature.
+* Add optional Serde support for `&'static Encoding`.
+* Performance tweaks for ASCII handling.
+* Performance tweaks for UTF-8 validation.
+* SIMD support on aarch64.
+
### 0.6.11
* Make `Encoder::has_pending_state()` public.
* Update the `simd` crate dependency to 0.2.0.
### 0.6.10
* Reserve enough space for NCRs when encoding to ISO-2022-JP.
--- a/third_party/rust/encoding_rs/generate-encoding-data.py
+++ b/third_party/rust/encoding_rs/generate-encoding-data.py
@@ -199,26 +199,17 @@ pub static %s_INIT: Encoding = Encoding
/// This will change from `static` to `const` if Rust changes
/// to make the referent of `pub const FOO: &'static Encoding`
/// unique cross-crate, so don't take the address of this
/// `static`.
pub static %s: &'static Encoding = &%s_INIT;
''' % (to_dom_name(name), to_constant_name(name), to_dom_name(name), variant, to_dom_name(name), to_constant_name(name), to_constant_name(name)))
-label_file.write("""static ENCODINGS_SORTED_BY_NAME: [&'static Encoding; %d] = [
-""" % (len(dom) - 1))
-
-for dom_name in dom:
- if dom_name != "UTF-8":
- label_file.write("&%s_INIT,\n" % to_constant_name(dom_name))
-
-label_file.write("""];
-
-static LABELS_SORTED: [&'static str; %d] = [
+label_file.write("""static LABELS_SORTED: [&'static str; %d] = [
""" % len(labels))
for label in labels:
label_file.write('''"%s",\n''' % label.label)
label_file.write("""];
static ENCODINGS_IN_LABEL_SORT: [&'static Encoding; %d] = [
@@ -245,25 +236,16 @@ use super::*;
#[test]
fn test_all_labels() {
''')
for label in labels:
label_test_file.write('''assert_eq!(Encoding::for_label(b"%s"), Some(%s));\n''' % (label.label, to_constant_name(label.preferred)))
label_test_file.write('''}
-
-#[test]
-fn test_all_names() {
-''')
-
-for dom_name in dom:
- label_test_file.write('''assert_eq!(Encoding::for_name(b"%s"), %s);\n''' % (dom_name, to_constant_name(dom_name)))
-
-label_test_file.write('''}
''')
label_test_file.close()
def null_to_zero(code_point):
if not code_point:
code_point = 0
return code_point
@@ -1642,16 +1624,20 @@ static_file.write("""// Copyright 2016 M
#ifndef ENCODING_RS_ENCODING
#define ENCODING_RS_ENCODING Encoding
#ifndef __cplusplus
typedef struct Encoding_ Encoding;
#endif
#endif
+#ifndef ENCODING_RS_NOT_NULL_CONST_ENCODING_PTR
+#define ENCODING_RS_NOT_NULL_CONST_ENCODING_PTR const ENCODING_RS_ENCODING*
+#endif
+
#ifndef ENCODING_RS_ENCODER
#define ENCODING_RS_ENCODER Encoder
#ifndef __cplusplus
typedef struct Encoder_ Encoder;
#endif
#endif
#ifndef ENCODING_RS_DECODER
@@ -1667,17 +1653,17 @@ typedef struct Decoder_ Decoder;
// %s
#define ENCODING_NAME_MAX_LENGTH %d
""" % (longest_name, longest_name_length))
for name in preferred:
static_file.write('''/// The %s encoding.
-extern const ENCODING_RS_ENCODING* const %s_ENCODING;
+extern ENCODING_RS_NOT_NULL_CONST_ENCODING_PTR const %s_ENCODING;
''' % (to_dom_name(name), to_constant_name(name)))
static_file.write("""#endif // encoding_rs_statics_h_
""")
static_file.close()
(utf_8_rs_begin, utf_8_rs_end) = read_non_generated("src/utf_8.rs")
--- a/third_party/rust/encoding_rs/src/ascii.rs
+++ b/third_party/rust/encoding_rs/src/ascii.rs
@@ -16,19 +16,20 @@
// numbers of the actual ARMv7 CPU I have access to, because (thermal?)
// throttling kept interfering. Since Raspberry Pi 3 (ARMv8 core but running
// ARMv7 code) produced reproducible performance numbers, that's the ARM
// computer that this code ended up being optimized for in the ALU case.
// Less popular CPU architectures simply get the approach that was chosen based
// on Raspberry Pi 3 measurements. The UTF-16 and UTF-8 ALU cases take
// different approaches based on benchmarking on Raspberry Pi 3.
-#[cfg(feature = "simd-accel")]
+#[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))]
use simd_funcs::*;
+#[allow(unused_macros)]
macro_rules! ascii_naive {
($name:ident,
$src_unit:ty,
$dst_unit:ty) => (
#[inline(always)]
pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) -> Option<($src_unit, usize)> {
// Yes, manually omitting the bound check here matters
// a lot for perf.
@@ -38,16 +39,17 @@ macro_rules! ascii_naive {
return Some((code_unit, i));
}
*(dst.offset(i as isize)) = code_unit as $dst_unit;
}
return None;
});
}
+#[allow(unused_macros)]
macro_rules! ascii_alu {
($name:ident,
$src_unit:ty,
$dst_unit:ty,
$stride_fn:ident) => (
#[cfg_attr(feature = "cargo-clippy", allow(never_loop))]
#[inline(always)]
pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) -> Option<($src_unit, usize)> {
@@ -94,24 +96,25 @@ macro_rules! ascii_alu {
let code_unit = *(src.offset(offset as isize));
if code_unit > 127 {
return Some((code_unit, offset));
}
*(dst.offset(offset as isize)) = code_unit as $dst_unit;
offset += 1;
until_alignment -= 1;
}
+ let len_minus_stride = len - STRIDE_SIZE;
loop {
if let Some(num_ascii) = $stride_fn(src.offset(offset as isize) as *const usize,
dst.offset(offset as isize) as *mut usize) {
offset += num_ascii;
return Some((*(src.offset(offset as isize)), offset));
}
offset += STRIDE_SIZE;
- if offset + STRIDE_SIZE > len {
+ if offset > len_minus_stride {
break;
}
}
}
break;
}
while offset < len {
let code_unit = *(src.offset(offset as isize));
@@ -120,16 +123,17 @@ macro_rules! ascii_alu {
}
*(dst.offset(offset as isize)) = code_unit as $dst_unit;
offset += 1;
}
None
});
}
+#[allow(unused_macros)]
macro_rules! basic_latin_alu {
($name:ident,
$src_unit:ty,
$dst_unit:ty,
$stride_fn:ident) => (
#[cfg_attr(feature = "cargo-clippy", allow(never_loop))]
#[inline(always)]
pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) -> Option<($src_unit, usize)> {
@@ -177,23 +181,24 @@ macro_rules! basic_latin_alu {
let code_unit = *(src.offset(offset as isize));
if code_unit > 127 {
return Some((code_unit, offset));
}
*(dst.offset(offset as isize)) = code_unit as $dst_unit;
offset += 1;
until_alignment -= 1;
}
+ let len_minus_stride = len - STRIDE_SIZE;
loop {
if !$stride_fn(src.offset(offset as isize) as *const usize,
dst.offset(offset as isize) as *mut usize) {
break;
}
offset += STRIDE_SIZE;
- if offset + STRIDE_SIZE > len {
+ if offset > len_minus_stride {
break;
}
}
}
break;
}
while offset < len {
let code_unit = *(src.offset(offset as isize));
@@ -202,81 +207,80 @@ macro_rules! basic_latin_alu {
}
*(dst.offset(offset as isize)) = code_unit as $dst_unit;
offset += 1;
}
None
});
}
-macro_rules! ascii_simd {
+#[allow(unused_macros)]
+macro_rules! ascii_simd_check_align {
($name:ident,
$src_unit:ty,
$dst_unit:ty,
$stride_both_aligned:ident,
$stride_src_aligned:ident,
$stride_dst_aligned:ident,
$stride_neither_aligned:ident) => (
#[inline(always)]
pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) -> Option<($src_unit, usize)> {
let mut offset = 0usize;
- // XXX should we have more branchy code to move the pointers to
- // alignment if they aren't aligned but could align after
- // processing a few code units?
if STRIDE_SIZE <= len {
+ let len_minus_stride = len - STRIDE_SIZE;
// XXX Should we first process one stride unconditinoally as unaligned to
// avoid the cost of the branchiness below if the first stride fails anyway?
// XXX Should we just use unaligned SSE2 access unconditionally? It seems that
// on Haswell, it would make sense to just use unaligned and not bother
// checking. Need to benchmark older architectures before deciding.
let dst_masked = (dst as usize) & ALIGNMENT_MASK;
if ((src as usize) & ALIGNMENT_MASK) == 0 {
if dst_masked == 0 {
loop {
if !$stride_both_aligned(src.offset(offset as isize),
dst.offset(offset as isize)) {
break;
}
offset += STRIDE_SIZE;
- if offset + STRIDE_SIZE > len {
+ if offset > len_minus_stride {
break;
}
}
} else {
loop {
if !$stride_src_aligned(src.offset(offset as isize),
dst.offset(offset as isize)) {
break;
}
offset += STRIDE_SIZE;
- if offset + STRIDE_SIZE > len {
+ if offset > len_minus_stride {
break;
}
}
}
} else {
if dst_masked == 0 {
loop {
if !$stride_dst_aligned(src.offset(offset as isize),
dst.offset(offset as isize)) {
break;
}
offset += STRIDE_SIZE;
- if offset + STRIDE_SIZE > len {
+ if offset > len_minus_stride {
break;
}
}
} else {
loop {
if !$stride_neither_aligned(src.offset(offset as isize),
dst.offset(offset as isize)) {
break;
}
offset += STRIDE_SIZE;
- if offset + STRIDE_SIZE > len {
+ if offset > len_minus_stride {
break;
}
}
}
}
}
while offset < len {
let code_unit = *(src.offset(offset as isize));
@@ -285,31 +289,67 @@ macro_rules! ascii_simd {
}
*(dst.offset(offset as isize)) = code_unit as $dst_unit;
offset += 1;
}
None
});
}
+#[allow(unused_macros)]
+macro_rules! ascii_simd_unalign {
+ ($name:ident,
+ $src_unit:ty,
+ $dst_unit:ty,
+ $stride_neither_aligned:ident) => (
+ #[inline(always)]
+ pub unsafe fn $name(src: *const $src_unit, dst: *mut $dst_unit, len: usize) -> Option<($src_unit, usize)> {
+ let mut offset = 0usize;
+ if STRIDE_SIZE <= len {
+ let len_minus_stride = len - STRIDE_SIZE;
+ loop {
+ if !$stride_neither_aligned(src.offset(offset as isize),
+ dst.offset(offset as isize)) {
+ break;
+ }
+ offset += STRIDE_SIZE;
+ if offset > len_minus_stride {
+ break;
+ }
+ }
+ }
+ while offset < len {
+ let code_unit = *(src.offset(offset as isize));
+ if code_unit > 127 {
+ return Some((code_unit, offset));
+ }
+ *(dst.offset(offset as isize)) = code_unit as $dst_unit;
+ offset += 1;
+ }
+ None
+ });
+}
+
+#[allow(unused_macros)]
macro_rules! ascii_to_ascii_simd_stride {
($name:ident,
$load:ident,
$store:ident) => (
#[inline(always)]
pub unsafe fn $name(src: *const u8, dst: *mut u8) -> bool {
let simd = $load(src);
if !is_ascii(simd) {
return false;
}
$store(dst, simd);
true
});
}
+#[allow(unused_macros)]
macro_rules! ascii_to_basic_latin_simd_stride {
($name:ident,
$load:ident,
$store:ident) => (
#[inline(always)]
pub unsafe fn $name(src: *const u8, dst: *mut u16) -> bool {
let simd = $load(src);
if !is_ascii(simd) {
@@ -317,16 +357,17 @@ macro_rules! ascii_to_basic_latin_simd_s
}
let (first, second) = unpack(simd);
$store(dst, first);
$store(dst.offset(8), second);
true
});
}
+#[allow(unused_macros)]
macro_rules! basic_latin_to_ascii_simd_stride {
($name:ident,
$load:ident,
$store:ident) => (
#[inline(always)]
pub unsafe fn $name(src: *const u16, dst: *mut u8) -> bool {
let first = $load(src);
let second = $load(src.offset(8));
@@ -336,18 +377,38 @@ macro_rules! basic_latin_to_ascii_simd_s
true
},
None => false,
}
});
}
cfg_if! {
- if #[cfg(all(feature = "simd-accel", target_feature = "sse2"))] {
- // SIMD
+ if #[cfg(all(feature = "simd-accel", target_endian = "little", target_arch = "aarch64"))] {
+ // SIMD with the same instructions for aligned and unaligned loads and stores
+
+ pub const STRIDE_SIZE: usize = 16;
+
+ const ALIGNMENT: usize = 8;
+
+ ascii_to_ascii_simd_stride!(ascii_to_ascii_stride_neither_aligned, load16_unaligned, store16_unaligned);
+
+ ascii_to_basic_latin_simd_stride!(ascii_to_basic_latin_stride_neither_aligned, load16_unaligned, store8_unaligned);
+
+ basic_latin_to_ascii_simd_stride!(basic_latin_to_ascii_stride_neither_aligned, load8_unaligned, store16_unaligned);
+
+ ascii_simd_unalign!(ascii_to_ascii, u8, u8, ascii_to_ascii_stride_neither_aligned);
+ ascii_simd_unalign!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_neither_aligned);
+ ascii_simd_unalign!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_neither_aligned);
+ } else if #[cfg(all(feature = "simd-accel", target_feature = "sse2"))] {
+ // SIMD with different instructions for aligned and unaligned loads and stores.
+ //
+ // Newer microarchitectures are not supposed to have a performance difference between
+ // aligned and unaligned SSE2 loads and stores when the address is actually aligned,
+ // but the benchmark results I see don't agree.
pub const STRIDE_SIZE: usize = 16;
const ALIGNMENT_MASK: usize = 15;
ascii_to_ascii_simd_stride!(ascii_to_ascii_stride_both_aligned, load16_aligned, store16_aligned);
ascii_to_ascii_simd_stride!(ascii_to_ascii_stride_src_aligned, load16_aligned, store16_unaligned);
ascii_to_ascii_simd_stride!(ascii_to_ascii_stride_dst_aligned, load16_unaligned, store16_aligned);
@@ -358,19 +419,19 @@ cfg_if! {
ascii_to_basic_latin_simd_stride!(ascii_to_basic_latin_stride_dst_aligned, load16_unaligned, store8_aligned);
ascii_to_basic_latin_simd_stride!(ascii_to_basic_latin_stride_neither_aligned, load16_unaligned, store8_unaligned);
basic_latin_to_ascii_simd_stride!(basic_latin_to_ascii_stride_both_aligned, load8_aligned, store16_aligned);
basic_latin_to_ascii_simd_stride!(basic_latin_to_ascii_stride_src_aligned, load8_aligned, store16_unaligned);
basic_latin_to_ascii_simd_stride!(basic_latin_to_ascii_stride_dst_aligned, load8_unaligned, store16_aligned);
basic_latin_to_ascii_simd_stride!(basic_latin_to_ascii_stride_neither_aligned, load8_unaligned, store16_unaligned);
- ascii_simd!(ascii_to_ascii, u8, u8, ascii_to_ascii_stride_both_aligned, ascii_to_ascii_stride_src_aligned, ascii_to_ascii_stride_dst_aligned, ascii_to_ascii_stride_neither_aligned);
- ascii_simd!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_both_aligned, ascii_to_basic_latin_stride_src_aligned, ascii_to_basic_latin_stride_dst_aligned, ascii_to_basic_latin_stride_neither_aligned);
- ascii_simd!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_both_aligned, basic_latin_to_ascii_stride_src_aligned, basic_latin_to_ascii_stride_dst_aligned, basic_latin_to_ascii_stride_neither_aligned);
+ ascii_simd_check_align!(ascii_to_ascii, u8, u8, ascii_to_ascii_stride_both_aligned, ascii_to_ascii_stride_src_aligned, ascii_to_ascii_stride_dst_aligned, ascii_to_ascii_stride_neither_aligned);
+ ascii_simd_check_align!(ascii_to_basic_latin, u8, u16, ascii_to_basic_latin_stride_both_aligned, ascii_to_basic_latin_stride_src_aligned, ascii_to_basic_latin_stride_dst_aligned, ascii_to_basic_latin_stride_neither_aligned);
+ ascii_simd_check_align!(basic_latin_to_ascii, u16, u8, basic_latin_to_ascii_stride_both_aligned, basic_latin_to_ascii_stride_src_aligned, basic_latin_to_ascii_stride_dst_aligned, basic_latin_to_ascii_stride_neither_aligned);
} else if #[cfg(all(target_endian = "little", target_pointer_width = "64"))] {
// Aligned ALU word, little-endian, 64-bit
pub const STRIDE_SIZE: usize = 16;
const ALIGNMENT: usize = 8;
const ALIGNMENT_MASK: usize = 7;
@@ -625,66 +686,98 @@ cfg_if! {
} else {
ascii_naive!(ascii_to_ascii, u8, u8);
ascii_naive!(ascii_to_basic_latin, u8, u16);
ascii_naive!(basic_latin_to_ascii, u16, u8);
}
}
cfg_if! {
- if #[cfg(all(feature = "simd-accel", target_feature = "sse2"))] {
- } else if #[cfg(target_endian = "little")] {
+ if #[cfg(target_endian = "little")] {
+ #[allow(dead_code)]
#[inline(always)]
fn count_zeros(word: usize) -> u32 {
word.trailing_zeros()
}
} else {
+ #[allow(dead_code)]
#[inline(always)]
fn count_zeros(word: usize) -> u32 {
word.leading_zeros()
}
}
}
cfg_if! {
- if #[cfg(all(feature = "simd-accel", target_feature = "sse2"))] {
+ if #[cfg(all(feature = "simd-accel", target_endian = "little", target_arch = "aarch64"))] {
#[inline(always)]
pub fn validate_ascii(slice: &[u8]) -> Option<(u8, usize)> {
let src = slice.as_ptr();
let len = slice.len();
let mut offset = 0usize;
if STRIDE_SIZE <= len {
+ let len_minus_stride = len - STRIDE_SIZE;
+ loop {
+ let simd = unsafe { load16_unaligned(src.offset(offset as isize)) };
+ if !is_ascii(simd) {
+ break;
+ }
+ offset += STRIDE_SIZE;
+ if offset > len_minus_stride {
+ break;
+ }
+ }
+ }
+ while offset < len {
+ let code_unit = slice[offset];
+ if code_unit > 127 {
+ return Some((code_unit, offset));
+ }
+ offset += 1;
+ }
+ None
+ }
+ } else if #[cfg(all(feature = "simd-accel", target_feature = "sse2"))] {
+ #[inline(always)]
+ pub fn validate_ascii(slice: &[u8]) -> Option<(u8, usize)> {
+ let src = slice.as_ptr();
+ let len = slice.len();
+ let mut offset = 0usize;
+ if STRIDE_SIZE <= len {
+ let len_minus_stride = len - STRIDE_SIZE;
// XXX Should we first process one stride unconditionally as unaligned to
// avoid the cost of the branchiness below if the first stride fails anyway?
// XXX Should we just use unaligned SSE2 access unconditionally? It seems that
// on Haswell, it would make sense to just use unaligned and not bother
// checking. Need to benchmark older architectures before deciding.
if ((src as usize) & ALIGNMENT_MASK) == 0 {
loop {
let simd = unsafe { load16_aligned(src.offset(offset as isize)) };
- if let Some(consumed) = check_ascii(simd) {
- offset += consumed;
+ let mask = mask_ascii(simd);
+ if mask != 0 {
+ offset += mask.trailing_zeros() as usize;
let non_ascii = unsafe { *src.offset(offset as isize) };
return Some((non_ascii, offset));
}
offset += STRIDE_SIZE;
- if offset + STRIDE_SIZE > len {
+ if offset > len_minus_stride {
break;
}
}
} else {
loop {
let simd = unsafe { load16_unaligned(src.offset(offset as isize)) };
- if let Some(consumed) = check_ascii(simd) {
- offset += consumed;
+ let mask = mask_ascii(simd);
+ if mask != 0 {
+ offset += mask.trailing_zeros() as usize;
let non_ascii = unsafe { *src.offset(offset as isize) };
return Some((non_ascii, offset));
}
offset += STRIDE_SIZE;
- if offset + STRIDE_SIZE > len {
+ if offset > len_minus_stride {
break;
}
}
}
}
while offset < len {
let code_unit = slice[offset];
if code_unit > 127 {
@@ -735,51 +828,53 @@ cfg_if! {
// `zeros` now contains 7 (for the seven bits of non-ASCII)
// plus 8 times the number of ASCII in text order before the
// non-ASCII byte in the little-endian case or 8 times the number of ASCII in
// text order before the non-ASCII byte in the big-endian case.
let num_ascii = (zeros >> 3) as usize;
Some(ALIGNMENT + num_ascii)
}
+ #[cfg(not(all(feature = "simd-accel", target_endian = "little", target_arch = "aarch64")))]
ascii_alu!(ascii_to_ascii, u8, u8, ascii_to_ascii_stride);
#[inline(always)]
pub fn validate_ascii(slice: &[u8]) -> Option<(u8, usize)> {
- let src = slice.as_ptr();
- let len = slice.len();
- let mut offset = 0usize;
- let mut until_alignment = (ALIGNMENT - ((src as usize) & ALIGNMENT_MASK)) & ALIGNMENT_MASK;
- if until_alignment + STRIDE_SIZE <= len {
- while until_alignment != 0 {
- let code_unit = slice[offset];
- if code_unit > 127 {
- return Some((code_unit, offset));
- }
- offset += 1;
- until_alignment -= 1;
- }
- loop {
- let ptr = unsafe { src.offset(offset as isize) as *const usize };
- if let Some(num_ascii) = unsafe { validate_ascii_stride(ptr) } {
- offset += num_ascii;
- return Some((unsafe { *(src.offset(offset as isize)) }, offset));
- }
- offset += STRIDE_SIZE;
- if offset + STRIDE_SIZE > len {
- break;
- }
- }
- }
- while offset < len {
- let code_unit = slice[offset];
- if code_unit > 127 {
- return Some((code_unit, offset));
- }
- offset += 1;
+ let src = slice.as_ptr();
+ let len = slice.len();
+ let mut offset = 0usize;
+ let mut until_alignment = (ALIGNMENT - ((src as usize) & ALIGNMENT_MASK)) & ALIGNMENT_MASK;
+ if until_alignment + STRIDE_SIZE <= len {
+ while until_alignment != 0 {
+ let code_unit = slice[offset];
+ if code_unit > 127 {
+ return Some((code_unit, offset));
+ }
+ offset += 1;
+ until_alignment -= 1;
+ }
+ let len_minus_stride = len - STRIDE_SIZE;
+ loop {
+ let ptr = unsafe { src.offset(offset as isize) as *const usize };
+ if let Some(num_ascii) = unsafe { validate_ascii_stride(ptr) } {
+ offset += num_ascii;
+ return Some((unsafe { *(src.offset(offset as isize)) }, offset));
+ }
+ offset += STRIDE_SIZE;
+ if offset > len_minus_stride {
+ break;
+ }
+ }
+ }
+ while offset < len {
+ let code_unit = slice[offset];
+ if code_unit > 127 {
+ return Some((code_unit, offset));
+ }
+ offset += 1;
}
None
}
}
}
pub fn ascii_valid_up_to(bytes: &[u8]) -> usize {
--- a/third_party/rust/encoding_rs/src/lib.rs
+++ b/third_party/rust/encoding_rs/src/lib.rs
@@ -3,17 +3,17 @@
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#![cfg_attr(feature = "cargo-clippy", allow(doc_markdown, inline_always, new_ret_no_self))]
-#![doc(html_root_url = "https://docs.rs/encoding_rs/0.6.11")]
+#![doc(html_root_url = "https://docs.rs/encoding_rs/0.7.0")]
//! encoding_rs is a Gecko-oriented Free Software / Open Source implementation
//! of the [Encoding Standard](https://encoding.spec.whatwg.org/) in Rust.
//! Gecko-oriented means that converting to and from UTF-16 is supported in
//! addition to converting to and from UTF-8, that the performance and
//! streamability goals are browser-oriented, and that FFI-friendliness is a
//! goal.
//!
@@ -491,31 +491,44 @@
//! </tbody>
//! </table>
#![cfg_attr(feature = "simd-accel", feature(cfg_target_feature, platform_intrinsics))]
#[macro_use]
extern crate cfg_if;
-#[cfg(feature = "simd-accel")]
+#[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))]
extern crate simd;
+#[cfg(feature = "serde")]
+extern crate serde;
+
+#[cfg(all(test,feature = "serde"))]
+extern crate serde_json;
+#[cfg(all(test,feature = "serde"))]
+extern crate bincode;
+#[cfg(all(test,feature = "serde"))]
+#[macro_use]
+extern crate serde_derive;
+
#[macro_use]
mod macros;
-#[cfg(feature = "simd-accel")]
+#[cfg(all(feature = "simd-accel", any(target_feature = "sse2", all(target_endian = "little", target_arch = "aarch64"))))]
mod simd_funcs;
+#[cfg(any(all(feature = "simd-accel", target_feature = "sse2"), all(target_endian = "little", target_arch = "aarch64"), all(target_endian = "little", target_arch = "arm")))]
+mod utf_8_core;
+
#[cfg(test)]
mod testing;
mod single_byte;
mod utf_8;
-mod utf_8_core;
mod gb18030;
mod big5;
mod euc_jp;
mod iso_2022_jp;
mod shift_jis;
mod euc_kr;
mod replacement;
mod x_user_defined;
@@ -531,16 +544,21 @@ use utf_8::utf8_valid_up_to;
use ascii::ascii_valid_up_to;
use ascii::iso_2022_jp_ascii_valid_up_to;
use std::borrow::Cow;
use std::cmp::Ordering;
use std::hash::Hash;
use std::hash::Hasher;
+#[cfg(feature = "serde")]
+use serde::{Serialize, Serializer, Deserialize, Deserializer};
+#[cfg(feature = "serde")]
+use serde::de::Visitor;
+
/// This has to be the max length of an NCR instead of max
/// minus one, because we can't rely on getting the minus
/// one from the space reserved for the current unmappable,
/// because the ISO-2022-JP encoder can fill up that space
/// with a state transition escape.
const NCR_EXTRA: usize = 10; // 
// BEGIN GENERATED CODE. PLEASE DO NOT EDIT.
@@ -1543,57 +1561,17 @@ pub static X_USER_DEFINED_INIT: Encoding
/// The x-user-defined encoding.
///
/// This will change from `static` to `const` if Rust changes
/// to make the referent of `pub const FOO: &'static Encoding`
/// unique cross-crate, so don't take the address of this
/// `static`.
pub static X_USER_DEFINED: &'static Encoding = &X_USER_DEFINED_INIT;
-static ENCODINGS_SORTED_BY_NAME: [&'static Encoding; 39] = [&GBK_INIT,
- &BIG5_INIT,
- &IBM866_INIT,
- &EUC_JP_INIT,
- &KOI8_R_INIT,
- &EUC_KR_INIT,
- &KOI8_U_INIT,
- &GB18030_INIT,
- &UTF_16BE_INIT,
- &UTF_16LE_INIT,
- &SHIFT_JIS_INIT,
- &MACINTOSH_INIT,
- &ISO_8859_2_INIT,
- &ISO_8859_3_INIT,
- &ISO_8859_4_INIT,
- &ISO_8859_5_INIT,
- &ISO_8859_6_INIT,
- &ISO_8859_7_INIT,
- &ISO_8859_8_INIT,
- &ISO_8859_10_INIT,
- &ISO_8859_13_INIT,
- &ISO_8859_14_INIT,
- &WINDOWS_874_INIT,
- &ISO_8859_15_INIT,
- &ISO_8859_16_INIT,
- &ISO_2022_JP_INIT,
- &REPLACEMENT_INIT,
- &WINDOWS_1250_INIT,
- &WINDOWS_1251_INIT,
- &WINDOWS_1252_INIT,
- &WINDOWS_1253_INIT,
- &WINDOWS_1254_INIT,
- &WINDOWS_1255_INIT,
- &WINDOWS_1256_INIT,
- &WINDOWS_1257_INIT,
- &WINDOWS_1258_INIT,
- &ISO_8859_8_I_INIT,
- &X_MAC_CYRILLIC_INIT,
- &X_USER_DEFINED_INIT];
-
-static LABELS_SORTED: [&'static str; 218] = ["l1",
+static LABELS_SORTED: [&'static str; 219] = ["l1",
"l2",
"l3",
"l4",
"l5",
"l6",
"l9",
"866",
"mac",
@@ -1763,16 +1741,17 @@ static LABELS_SORTED: [&'static str; 218
"csiso88598i",
"windows-31j",
"x-mac-roman",
"iso-2022-cn",
"iso-2022-jp",
"csiso2022jp",
"iso-2022-kr",
"csiso2022kr",
+ "replacement",
"windows-1250",
"windows-1251",
"windows-1252",
"windows-1253",
"windows-1254",
"windows-1255",
"windows-1256",
"windows-1257",
@@ -1802,17 +1781,17 @@ static LABELS_SORTED: [&'static str; 218
"x-mac-ukrainian",
"iso-2022-cn-ext",
"csisolatinarabic",
"csisolatinhebrew",
"unicode-1-1-utf-8",
"csisolatincyrillic",
"cseucpkdfmtjapanese"];
-static ENCODINGS_IN_LABEL_SORT: [&'static Encoding; 218] = [&WINDOWS_1252_INIT,
+static ENCODINGS_IN_LABEL_SORT: [&'static Encoding; 219] = [&WINDOWS_1252_INIT,
&ISO_8859_2_INIT,
&ISO_8859_3_INIT,
&ISO_8859_4_INIT,
&WINDOWS_1254_INIT,
&ISO_8859_10_INIT,
&ISO_8859_15_INIT,
&IBM866_INIT,
&MACINTOSH_INIT,
@@ -1982,16 +1961,17 @@ static ENCODINGS_IN_LABEL_SORT: [&'stati
&ISO_8859_8_I_INIT,
&SHIFT_JIS_INIT,
&MACINTOSH_INIT,
&REPLACEMENT_INIT,
&ISO_2022_JP_INIT,
&ISO_2022_JP_INIT,
&REPLACEMENT_INIT,
&REPLACEMENT_INIT,
+ &REPLACEMENT_INIT,
&WINDOWS_1250_INIT,
&WINDOWS_1251_INIT,
&WINDOWS_1252_INIT,
&WINDOWS_1253_INIT,
&WINDOWS_1254_INIT,
&WINDOWS_1255_INIT,
&WINDOWS_1256_INIT,
&WINDOWS_1257_INIT,
@@ -2032,18 +2012,17 @@ static ENCODINGS_IN_LABEL_SORT: [&'stati
///
/// An _encoding_ defines a mapping from a `u8` sequence to a `char` sequence
/// and, in most cases, vice versa. Each encoding has a name, an output
/// encoding, and one or more labels.
///
/// _Labels_ are ASCII-case-insensitive strings that are used to identify an
/// encoding in formats and protocols. The _name_ of the encoding is the
/// preferred label in the case appropriate for returning from the
-/// [`characterSet`][2] property of the `Document` DOM interface, except for
-/// the replacement encoding whose name is not one of its labels.
+/// [`characterSet`][2] property of the `Document` DOM interface.
///
/// The _output encoding_ is the encoding used for form submission and URL
/// parsing on Web pages in the encoding. This is UTF-8 for the replacement,
/// UTF-16LE and UTF-16BE encodings and the encoding itself for other
/// encodings.
///
/// [1]: https://encoding.spec.whatwg.org/
/// [2]: https://dom.spec.whatwg.org/#dom-document-characterset
@@ -2274,55 +2253,16 @@ impl Encoding {
Some((UTF_16LE, 2))
} else if buffer.starts_with(b"\xFE\xFF") {
Some((UTF_16BE, 2))
} else {
None
}
}
- /// If the argument matches exactly (case-sensitively; no whitespace
- /// removal performed) the name of an encoding, returns
- /// `&'static Encoding` representing that encoding. Otherwise panics.
- ///
- /// The motivating use case for this method is interoperability with
- /// legacy Gecko code that represents encodings as name string instead of
- /// type-safe `Encoding` objects. Using this method for other purposes is
- /// most likely the wrong thing to do.
- ///
- /// Available via the C wrapper.
- ///
- /// # Panics
- ///
- /// Panics if the argument is not the name of an encoding.
- #[cfg_attr(feature = "cargo-clippy", allow(match_wild_err_arm))]
- pub fn for_name(name: &[u8]) -> &'static Encoding {
- // The length of `"UTF-8"` is unique, so it's easy to check the most
- // common case first.
- if name.len() == 5 {
- assert_eq!(name, b"UTF-8", "Bogus encoding name");
- return UTF_8;
- }
- match ENCODINGS_SORTED_BY_NAME.binary_search_by(
- |probe| {
- let bytes = probe.name().as_bytes();
- let c = bytes.len().cmp(&name.len());
- if c != Ordering::Equal {
- return c;
- }
- let probe_iter = bytes.iter().rev();
- let candidate_iter = name.iter().rev();
- probe_iter.cmp(candidate_iter)
- }
- ) {
- Ok(i) => ENCODINGS_SORTED_BY_NAME[i],
- Err(_) => panic!("Bogus encoding name"),
- }
- }
-
/// Returns the name of this encoding.
///
/// This name is appropriate to return as-is from the DOM
/// `document.characterSet` property.
///
/// Available via the C wrapper.
pub fn name(&'static self) -> &'static str {
self.name
@@ -2841,16 +2781,56 @@ impl Hash for Encoding {
}
impl std::fmt::Debug for Encoding {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "Encoding {{ {} }}", self.name)
}
}
+#[cfg(feature = "serde")]
+impl Serialize for Encoding {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where S: Serializer
+ {
+ serializer.serialize_str(self.name)
+ }
+}
+
+#[cfg(feature = "serde")]
+struct EncodingVisitor;
+
+#[cfg(feature = "serde")]
+impl<'de> Visitor<'de> for EncodingVisitor {
+ type Value = &'static Encoding;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+ formatter.write_str("a valid encoding label")
+ }
+
+ fn visit_str<E>(self, value: &str) -> Result<&'static Encoding, E>
+ where E: serde::de::Error
+ {
+ if let Some(enc) = Encoding::for_label(value.as_bytes()) {
+ Ok(enc)
+ } else {
+ Err(E::custom(format!("invalid encoding label: {}", value)))
+ }
+ }
+}
+
+#[cfg(feature = "serde")]
+impl<'de> Deserialize<'de> for &'static Encoding {
+ fn deserialize<D>(deserializer: D) -> Result<&'static Encoding, D::Error>
+ where D: Deserializer<'de>
+ {
+ deserializer.deserialize_str(EncodingVisitor)
+ }
+}
+
/// Tracks the life cycle of a decoder from BOM sniffing to conversion to end.
#[derive(PartialEq, Debug)]
enum DecoderLifeCycle {
/// The decoder has seen no input yet.
AtStart,
/// The decoder has seen no input yet but expects UTF-8.
AtUtf8Start,
/// The decoder has seen no input yet but expects UTF-16BE.
@@ -4197,16 +4177,24 @@ fn checked_min(one: Option<usize>, other
}
} else {
other
}
}
// ############## TESTS ###############
+#[cfg(all(test, feature = "serde"))]
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
+struct Demo {
+ num: u32,
+ name: String,
+ enc: &'static Encoding,
+}
+
#[cfg(test)]
mod test_labels_names;
#[cfg(test)]
mod tests {
use super::*;
use std::borrow::Cow;
@@ -4435,40 +4423,16 @@ mod tests {
Some(UTF_8)
);
assert_eq!(Encoding::for_label(b"utf-8 _"), None);
assert_eq!(Encoding::for_label(b"bogus"), None);
assert_eq!(Encoding::for_label(b"bogusbogusbogusbogus"), None);
}
#[test]
- #[should_panic]
- fn test_bogus_name_utf_8_case() {
- Encoding::for_name(b"utf-8");
- }
-
- #[test]
- #[should_panic]
- fn test_bogus_name() {
- Encoding::for_name(b"ISO-8859-1");
- }
-
- #[test]
- #[should_panic]
- fn test_bogus_name_gbk() {
- Encoding::for_name(b"gbk");
- }
-
- #[test]
- #[should_panic]
- fn test_bogus_name_spaces() {
- Encoding::for_name(b" UTF-8 ");
- }
-
- #[test]
fn test_decode_valid_windows_1257_to_cow() {
let (cow, encoding, had_errors) = WINDOWS_1257.decode(b"abc\x80\xE4");
match cow {
Cow::Borrowed(_) => unreachable!(),
Cow::Owned(s) => {
assert_eq!(s, "abc\u{20AC}\u{00E4}");
}
}
@@ -5026,9 +4990,29 @@ mod tests {
let (result, read, written) =
decoder.decode_to_utf8_without_replacement(input, &mut output[..needed], true);
assert_eq!(result, DecoderResult::InputEmpty);
assert_eq!(read, input.len());
assert_eq!(written, 1);
assert_eq!(output[0], 0x41);
}
}
+
+ #[cfg(feature = "serde")]
+ #[test]
+ fn test_serde() {
+ let demo = Demo {
+ num: 42,
+ name: "foo".into(),
+ enc: UTF_8,
+ };
+
+ let serialized = serde_json::to_string(&demo).unwrap();
+
+ let deserialized: Demo = serde_json::from_str(&serialized).unwrap();
+ assert_eq!(deserialized, demo);
+
+ let bincoded = bincode::serialize(&demo, bincode::Infinite).unwrap();
+ let debincoded: Demo = bincode::deserialize(&bincoded[..]).unwrap();
+ assert_eq!(debincoded, demo);
+ }
+
}
--- a/third_party/rust/encoding_rs/src/simd_funcs.rs
+++ b/third_party/rust/encoding_rs/src/simd_funcs.rs
@@ -3,27 +3,19 @@
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use simd::u8x16;
-use simd::i8x16;
use simd::u16x8;
-use simd::i16x8;
use simd::Simd;
-extern "platform-intrinsic" {
- fn simd_shuffle16<T: Simd, U: Simd<Elem = T::Elem>>(x: T, y: T, idx: [u32; 16]) -> U;
- fn x86_mm_packus_epi16(x: i16x8, y: i16x8) -> u8x16;
- fn x86_mm_movemask_epi8(x: i8x16) -> i32;
-}
-
// TODO: Migrate unaligned access to stdlib code if/when the RFC
// https://github.com/rust-lang/rfcs/pull/1725 is implemented.
#[inline(always)]
pub unsafe fn load16_unaligned(ptr: *const u8) -> u8x16 {
let mut simd = ::std::mem::uninitialized();
::std::ptr::copy_nonoverlapping(ptr, &mut simd as *mut u8x16 as *mut u8, 16);
simd
@@ -61,45 +53,125 @@ pub unsafe fn store8_unaligned(ptr: *mut
::std::ptr::copy_nonoverlapping(&s as *const u16x8 as *const u8, ptr as *mut u8, 16);
}
#[inline(always)]
pub unsafe fn store8_aligned(ptr: *mut u16, s: u16x8) {
*(ptr as *mut u16x8) = s;
}
-/// _mm_movemask_epi8 in SSE2. vec_all_lt in AltiVec.
-#[inline(always)]
-pub fn is_ascii(s: u8x16) -> bool {
- unsafe {
- let signed: i8x16 = ::std::mem::transmute_copy(&s);
- x86_mm_movemask_epi8(signed) == 0
+extern "platform-intrinsic" {
+ fn simd_shuffle16<T: Simd, U: Simd<Elem = T::Elem>>(x: T, y: T, idx: [u32; 16]) -> U;
+}
+
+cfg_if! {
+ if #[cfg(target_feature = "sse2")] {
+
+ use simd::i16x8;
+ use simd::i8x16;
+ extern "platform-intrinsic" {
+ fn x86_mm_packus_epi16(x: i16x8, y: i16x8) -> u8x16;
+ fn x86_mm_movemask_epi8(x: i8x16) -> i32;
+ }
+
+ #[inline(always)]
+ pub fn is_ascii(s: u8x16) -> bool {
+ unsafe {
+ let signed: i8x16 = ::std::mem::transmute_copy(&s);
+ x86_mm_movemask_epi8(signed) == 0
+ }
+ }
+
+ // Expose low-level mask instead of higher-level conclusion,
+ // because the non-ASCII case would perform less well otherwise.
+ #[inline(always)]
+ pub fn mask_ascii(s: u8x16) -> i32 {
+ unsafe {
+ let signed: i8x16 = ::std::mem::transmute_copy(&s);
+ x86_mm_movemask_epi8(signed)
+ }
+ }
+
+ #[inline(always)]
+ pub unsafe fn pack_basic_latin(a: u16x8, b: u16x8) -> Option<u8x16> {
+ // If the 16-bit lane is out of range positive, the 8-bit lane becomes 0xFF
+ // when packing, which would allow us to pack first and then check for
+ // ASCII, but if the 16-bit lane is negative, the 8-bit lane becomes 0x00.
+ // Sigh. Hence, check first.
+ let highest_ascii = u16x8::splat(0x7F);
+ let combined = a | b;
+ if combined.gt(highest_ascii).any() {
+ None
+ } else {
+ let first: i16x8 = ::std::mem::transmute_copy(&a);
+ let second: i16x8 = ::std::mem::transmute_copy(&b);
+ Some(x86_mm_packus_epi16(first, second))
+ }
+ }
+ } else if #[cfg(target_arch = "aarch64")]{
+
+ extern "platform-intrinsic" {
+ fn aarch64_vmaxvq_u8(x: u8x16) -> u8;
+ fn aarch64_vmaxvq_u16(x: u16x8) -> u16;
+ }
+
+ #[inline(always)]
+ pub fn is_ascii(s: u8x16) -> bool {
+ unsafe {
+ aarch64_vmaxvq_u8(s) < 0x80
+ }
+ }
+
+ #[inline(always)]
+ pub unsafe fn pack_basic_latin(a: u16x8, b: u16x8) -> Option<u8x16> {
+ let combined = a | b;
+ if aarch64_vmaxvq_u16(combined) < 0x80 {
+ let first: u8x16 = ::std::mem::transmute_copy(&a);
+ let second: u8x16 = ::std::mem::transmute_copy(&b);
+ let lower: u8x16 = simd_shuffle16(
+ first,
+ second,
+ [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30],
+ );
+ Some(lower)
+ } else {
+ None
+ }
+ }
+
+
+ } else {
+
+ #[inline(always)]
+ pub fn is_ascii(s: u8x16) -> bool {
+ let highest_ascii = u8x16::splat(0x7F);
+ !s.gt(highest_ascii).any()
+ }
+
+ #[inline(always)]
+ pub unsafe fn pack_basic_latin(a: u16x8, b: u16x8) -> Option<u8x16> {
+ let highest_ascii = u16x8::splat(0x7F);
+ let combined = a | b;
+ if combined.gt(highest_ascii).any() {
+ None
+ } else {
+ let first: u8x16 = ::std::mem::transmute_copy(&a);
+ let second: u8x16 = ::std::mem::transmute_copy(&b);
+ let lower: u8x16 = simd_shuffle16(
+ first,
+ second,
+ [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30],
+ );
+ Some(lower)
+ }
+ }
+
}
}
-
-/// _mm_movemask_epi8 in SSE2.
-#[inline(always)]
-pub fn check_ascii(s: u8x16) -> Option<usize> {
- let mask = unsafe {
- let signed: i8x16 = ::std::mem::transmute_copy(&s);
- x86_mm_movemask_epi8(signed)
- };
- if mask == 0 {
- return None;
- }
- // We don't extract the non-ascii byte from the SIMD register, because
- // at least on Haswell, it seems faster to let the caller re-read it from
- // memory.
- Some(mask.trailing_zeros() as usize)
-}
-
-/// vzipq_u8 in NEON. _mm_unpacklo_epi8 and
-/// _mm_unpackhi_epi8 in SSE2. vec_mergeh and vec_mergel or vec_unpackh and
-/// vec_unpackl in AltiVec.
#[inline(always)]
pub fn unpack(s: u8x16) -> (u16x8, u16x8) {
unsafe {
let first: u8x16 = simd_shuffle16(
s,
u8x16::splat(0),
[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23],
);
@@ -107,34 +179,16 @@ pub fn unpack(s: u8x16) -> (u16x8, u16x8
s,
u8x16::splat(0),
[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31],
);
(::std::mem::transmute_copy(&first), ::std::mem::transmute_copy(&second))
}
}
-/// vuzpq_u8 in NEON. _mm_packus_epi16 in SSE2. vec_packsu *followed* by ASCII
-/// check in AltiVec.
-#[inline(always)]
-pub unsafe fn pack_basic_latin(a: u16x8, b: u16x8) -> Option<u8x16> {
- // If the 16-bit lane is out of range positive, the 8-bit lane becomes 0xFF
- // when packing, which would allow us to pack later and then check for
- // ASCII, but if the 16-bit lane is negative, the 8-bit lane becomes 0x00.
- // Sigh. Hence, check first.
- let above_ascii = u16x8::splat(0x80);
- if a.lt(above_ascii).all() && b.lt(above_ascii).all() {
- let first: i16x8 = ::std::mem::transmute_copy(&a);
- let second: i16x8 = ::std::mem::transmute_copy(&b);
- Some(x86_mm_packus_epi16(first, second))
- } else {
- None
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_unpack() {
let ascii: [u8; 16] = [0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
0x72, 0x73, 0x74, 0x75, 0x76];
@@ -214,27 +268,25 @@ mod tests {
#[test]
fn test_is_ascii_failure() {
let input: [u8; 16] = [0x61, 0x62, 0x63, 0x64, 0x81, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
0x72, 0x73, 0x74, 0x75, 0x76];
let simd = unsafe { load16_unaligned(input.as_ptr()) };
assert!(!is_ascii(simd));
}
+ #[cfg(target_feature = "sse2")]
#[test]
fn test_check_ascii() {
let input: [u8; 16] = [0x61, 0x62, 0x63, 0x64, 0x81, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
0x72, 0x73, 0x74, 0x75, 0x76];
let simd = unsafe { load16_unaligned(input.as_ptr()) };
- match check_ascii(simd) {
- None => unreachable!(),
- Some(consumed) => {
- assert_eq!(consumed, 4);
- }
- }
+ let mask = mask_ascii(simd);
+ assert_ne!(mask, 0);
+ assert_eq!(mask.trailing_zeros(), 4);
}
#[test]
fn test_alu() {
let input: [u8; 16] = [0x61, 0x62, 0x63, 0x64, 0x81, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
0x72, 0x73, 0x74, 0x75, 0x76];
let mut alu = 0u64;
unsafe {
--- a/third_party/rust/encoding_rs/src/test_labels_names.rs
+++ b/third_party/rust/encoding_rs/src/test_labels_names.rs
@@ -183,16 +183,17 @@ fn test_all_labels() {
assert_eq!(Encoding::for_label(b"csiso88598i"), Some(ISO_8859_8_I));
assert_eq!(Encoding::for_label(b"windows-31j"), Some(SHIFT_JIS));
assert_eq!(Encoding::for_label(b"x-mac-roman"), Some(MACINTOSH));
assert_eq!(Encoding::for_label(b"iso-2022-cn"), Some(REPLACEMENT));
assert_eq!(Encoding::for_label(b"iso-2022-jp"), Some(ISO_2022_JP));
assert_eq!(Encoding::for_label(b"csiso2022jp"), Some(ISO_2022_JP));
assert_eq!(Encoding::for_label(b"iso-2022-kr"), Some(REPLACEMENT));
assert_eq!(Encoding::for_label(b"csiso2022kr"), Some(REPLACEMENT));
+ assert_eq!(Encoding::for_label(b"replacement"), Some(REPLACEMENT));
assert_eq!(Encoding::for_label(b"windows-1250"), Some(WINDOWS_1250));
assert_eq!(Encoding::for_label(b"windows-1251"), Some(WINDOWS_1251));
assert_eq!(Encoding::for_label(b"windows-1252"), Some(WINDOWS_1252));
assert_eq!(Encoding::for_label(b"windows-1253"), Some(WINDOWS_1253));
assert_eq!(Encoding::for_label(b"windows-1254"), Some(WINDOWS_1254));
assert_eq!(Encoding::for_label(b"windows-1255"), Some(WINDOWS_1255));
assert_eq!(Encoding::for_label(b"windows-1256"), Some(WINDOWS_1256));
assert_eq!(Encoding::for_label(b"windows-1257"), Some(WINDOWS_1257));
@@ -225,52 +226,8 @@ fn test_all_labels() {
);
assert_eq!(Encoding::for_label(b"iso-2022-cn-ext"), Some(REPLACEMENT));
assert_eq!(Encoding::for_label(b"csisolatinarabic"), Some(ISO_8859_6));
assert_eq!(Encoding::for_label(b"csisolatinhebrew"), Some(ISO_8859_8));
assert_eq!(Encoding::for_label(b"unicode-1-1-utf-8"), Some(UTF_8));
assert_eq!(Encoding::for_label(b"csisolatincyrillic"), Some(ISO_8859_5));
assert_eq!(Encoding::for_label(b"cseucpkdfmtjapanese"), Some(EUC_JP));
}
-
-#[test]
-fn test_all_names() {
- assert_eq!(Encoding::for_name(b"GBK"), GBK);
- assert_eq!(Encoding::for_name(b"Big5"), BIG5);
- assert_eq!(Encoding::for_name(b"UTF-8"), UTF_8);
- assert_eq!(Encoding::for_name(b"IBM866"), IBM866);
- assert_eq!(Encoding::for_name(b"EUC-JP"), EUC_JP);
- assert_eq!(Encoding::for_name(b"KOI8-R"), KOI8_R);
- assert_eq!(Encoding::for_name(b"EUC-KR"), EUC_KR);
- assert_eq!(Encoding::for_name(b"KOI8-U"), KOI8_U);
- assert_eq!(Encoding::for_name(b"gb18030"), GB18030);
- assert_eq!(Encoding::for_name(b"UTF-16BE"), UTF_16BE);
- assert_eq!(Encoding::for_name(b"UTF-16LE"), UTF_16LE);
- assert_eq!(Encoding::for_name(b"Shift_JIS"), SHIFT_JIS);
- assert_eq!(Encoding::for_name(b"macintosh"), MACINTOSH);
- assert_eq!(Encoding::for_name(b"ISO-8859-2"), ISO_8859_2);
- assert_eq!(Encoding::for_name(b"ISO-8859-3"), ISO_8859_3);
- assert_eq!(Encoding::for_name(b"ISO-8859-4"), ISO_8859_4);
- assert_eq!(Encoding::for_name(b"ISO-8859-5"), ISO_8859_5);
- assert_eq!(Encoding::for_name(b"ISO-8859-6"), ISO_8859_6);
- assert_eq!(Encoding::for_name(b"ISO-8859-7"), ISO_8859_7);
- assert_eq!(Encoding::for_name(b"ISO-8859-8"), ISO_8859_8);
- assert_eq!(Encoding::for_name(b"ISO-8859-10"), ISO_8859_10);
- assert_eq!(Encoding::for_name(b"ISO-8859-13"), ISO_8859_13);
- assert_eq!(Encoding::for_name(b"ISO-8859-14"), ISO_8859_14);
- assert_eq!(Encoding::for_name(b"windows-874"), WINDOWS_874);
- assert_eq!(Encoding::for_name(b"ISO-8859-15"), ISO_8859_15);
- assert_eq!(Encoding::for_name(b"ISO-8859-16"), ISO_8859_16);
- assert_eq!(Encoding::for_name(b"ISO-2022-JP"), ISO_2022_JP);
- assert_eq!(Encoding::for_name(b"replacement"), REPLACEMENT);
- assert_eq!(Encoding::for_name(b"windows-1250"), WINDOWS_1250);
- assert_eq!(Encoding::for_name(b"windows-1251"), WINDOWS_1251);
- assert_eq!(Encoding::for_name(b"windows-1252"), WINDOWS_1252);
- assert_eq!(Encoding::for_name(b"windows-1253"), WINDOWS_1253);
- assert_eq!(Encoding::for_name(b"windows-1254"), WINDOWS_1254);
- assert_eq!(Encoding::for_name(b"windows-1255"), WINDOWS_1255);
- assert_eq!(Encoding::for_name(b"windows-1256"), WINDOWS_1256);
- assert_eq!(Encoding::for_name(b"windows-1257"), WINDOWS_1257);
- assert_eq!(Encoding::for_name(b"windows-1258"), WINDOWS_1258);
- assert_eq!(Encoding::for_name(b"ISO-8859-8-I"), ISO_8859_8_I);
- assert_eq!(Encoding::for_name(b"x-mac-cyrillic"), X_MAC_CYRILLIC);
- assert_eq!(Encoding::for_name(b"x-user-defined"), X_USER_DEFINED);
-}
--- a/third_party/rust/encoding_rs/src/utf_16.rs
+++ b/third_party/rust/encoding_rs/src/utf_16.rs
@@ -25,17 +25,17 @@ impl Utf16Decoder {
lead_surrogate: 0,
lead_byte: None,
be: big_endian,
pending_bmp: false,
}
)
}
- pub fn additional_from_state(&self) -> usize() {
+ pub fn additional_from_state(&self) -> usize {
1 + if self.lead_byte.is_some() { 1 } else { 0 } +
if self.lead_surrogate == 0 { 0 } else { 2 }
}
pub fn max_utf16_buffer_length(&self, byte_length: usize) -> Option<usize> {
checked_add(
1,
checked_div(byte_length.checked_add(self.additional_from_state()), 2),
--- a/third_party/rust/encoding_rs/src/utf_8.rs
+++ b/third_party/rust/encoding_rs/src/utf_8.rs
@@ -10,17 +10,34 @@
#[cfg(feature = "parallel-utf8")]
extern crate rayon;
use handles::*;
use variant::*;
use super::*;
use ascii::ascii_to_basic_latin;
use ascii::basic_latin_to_ascii;
-use utf_8_core::run_utf8_validation;
+
+// Keep this cfg_if in sync with whether the utf_8_core module is defined in lib.rs.
+cfg_if! {
+ // When running 32-bit ARM code on Raspberry Pi 3, which has a 64-bit CPU,
+ // this is a pessimization for non-Latin, non-CJK scripts. However, this
+ // optimization seems to work across scripts when running 32-bit ARM code
+ // on a 32-bit ARM CPU (particularly good on Exynos 5) and when running
+ // 64-bit ARM code on a 64-bit ARM CPU.
+ if #[cfg(any(all(feature = "simd-accel", target_feature = "sse2"), all(target_endian = "little", target_arch = "aarch64"), all(target_endian = "little", target_arch = "arm")))] {
+ use utf_8_core::run_utf8_validation;
+ } else {
+ use ::std::str::Utf8Error;
+ #[inline(always)]
+ fn run_utf8_validation(v: &[u8]) -> Result<&str, Utf8Error> {
+ ::std::str::from_utf8(v)
+ }
+ }
+}
const UTF8_NORMAL_TRAIL: u8 = 1 << 3;
const UTF8_THREE_BYTE_SPECIAL_LOWER_BOUND_TRAIL: u8 = 1 << 4;
const UTF8_THREE_BYTE_SPECIAL_UPPER_BOUND_TRAIL: u8 = 1 << 5;
const UTF8_FOUR_BYTE_SPECIAL_LOWER_BOUND_TRAIL: u8 = 1 << 6;
@@ -58,17 +75,17 @@ pub fn utf8_valid_up_to(bytes: &[u8]) ->
// This magic number has been determined on i7-4770 with SSE2 enabled.
// It's very likely that the number should be different when different
// ISA is used for ASCII acceleration. The number has been chosen
// to optimize the all-ASCII case. With mostly non-ASCII, the number
// should be much smaller, but that would pessimize the all-ASCII case,
// which we are trying to optimize here.
if len < 290000 {
return match run_utf8_validation(&bytes[..len]) {
- Ok(()) => bytes.len(),
+ Ok(_) => bytes.len(),
Err(e) => e.valid_up_to(),
};
}
let mid = len >> 1;
let mut adjusted = mid;
let mut i = 0;
'inner: loop {
// No need to check for `adjusted` reaching `len` because we
@@ -92,17 +109,17 @@ pub fn utf8_valid_up_to(bytes: &[u8]) ->
}
return head_valid_up_to;
}
}
#[cfg(not(feature = "parallel-utf8"))]
pub fn utf8_valid_up_to(bytes: &[u8]) -> usize {
match run_utf8_validation(bytes) {
- Ok(()) => bytes.len(),
+ Ok(_) => bytes.len(),
Err(e) => e.valid_up_to(),
}
}
#[cfg_attr(feature = "cargo-clippy", allow(never_loop, cyclomatic_complexity))]
pub fn convert_utf8_to_utf16_up_to_invalid(src: &[u8], dst: &mut [u16]) -> (usize, usize) {
// This algorithm differs from the UTF-8 validation algorithm, but making
// this one consistent with that one makes this slower for reasons I don't
--- a/third_party/rust/encoding_rs/src/utf_8_core.rs
+++ b/third_party/rust/encoding_rs/src/utf_8_core.rs
@@ -6,17 +6,18 @@
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// The initial revision of this file was extracted from the "UTF-8 validation"
// section of the file src/libcore/str/mod.rs from Rust project at revision
// 7ad7232422f7e5bbfa0e52dabe36c12677df19e2. The Utf8Error struct also comes
-// from that file.
+// from that file. Subsequently, changes from the mentioned file at revision
+// 85eadf84f3945dc431643ea43d34f15193fdafb4 were merged into this file.
use ascii::validate_ascii;
/// Errors which can occur when attempting to interpret a sequence of `u8`
/// as a string.
///
/// As such, the `from_utf8` family of functions and methods for both `String`s
/// and `&str`s make use of this error, for example.
@@ -51,53 +52,53 @@ impl Utf8Error {
pub fn valid_up_to(&self) -> usize {
self.valid_up_to
}
}
#[cfg_attr(feature = "cargo-clippy", allow(eval_order_dependence))]
#[inline(always)]
pub fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> {
- let mut offset = 0;
+ let mut index = 0;
let len = v.len();
+
'outer: loop {
let mut first = {
- let remaining = &v[offset..];
+ let remaining = &v[index..];
match validate_ascii(remaining) {
None => {
// offset += remaining.len();
break 'outer;
}
Some((non_ascii, consumed)) => {
- offset += consumed;
+ index += consumed;
non_ascii
}
}
};
- let old_offset = offset;
- macro_rules! err { () => {{
- return Err(Utf8Error {
- valid_up_to: old_offset
- })
- }}}
+ let old_offset = index;
+ macro_rules! err {
+ ($error_len: expr) => {
+ return Err(Utf8Error {
+ valid_up_to: old_offset,
+ })
+ }
+ }
macro_rules! next { () => {{
- offset += 1;
+ index += 1;
// we needed data, but there was none: error!
- if offset >= len {
- err!()
+ if index >= len {
+ err!(None)
}
- v[offset]
+ v[index]
}}}
+
'inner: loop {
- // Intuitively, it would make sense to check availability for
- // a four-byte sequence here, not check per byte and handle the
- // end of the buffer as a special case. For some reason, that
- // disturbs something in a way that would make things slower.
- let second = next!();
+ let w = UTF8_CHAR_WIDTH[first as usize];
// 2-byte encoding is for codepoints \u{0080} to \u{07ff}
// first C2 80 last DF BF
// 3-byte encoding is for codepoints \u{0800} to \u{ffff}
// first E0 A0 80 last EF BF BF
// excluding surrogates codepoints \u{d800} to \u{dfff}
// ED A0 80 to ED BF BF
// 4-byte encoding is for codepoints \u{1000}0 to \u{10ff}ff
// first F0 90 80 80 last F4 8F BF BF
@@ -106,75 +107,84 @@ pub fn run_utf8_validation(v: &[u8]) ->
//
// https://tools.ietf.org/html/rfc3629
// UTF8-1 = %x00-7F
// UTF8-2 = %xC2-DF UTF8-tail
// UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
// %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
// %xF4 %x80-8F 2( UTF8-tail )
- match first {
- 0xC2...0xDF => {
- if second & !CONT_MASK != TAG_CONT_U8 {
- err!()
+ match w {
+ 2 => {
+ if next!() & !CONT_MASK != TAG_CONT_U8 {
+ err!(Some(1))
}
}
- 0xE0 => {
- match (second, next!() & !CONT_MASK) {
- (0xA0...0xBF, TAG_CONT_U8) => {}
- _ => err!(),
+ 3 => {
+ match (first, next!()) {
+ (0xE0, 0xA0...0xBF) |
+ (0xE1...0xEC, 0x80...0xBF) |
+ (0xED, 0x80...0x9F) |
+ (0xEE...0xEF, 0x80...0xBF) => {}
+ _ => err!(Some(1)),
}
- }
- 0xE1...0xEC | 0xEE...0xEF => {
- match (second & !CONT_MASK, next!() & !CONT_MASK) {
- (TAG_CONT_U8, TAG_CONT_U8) => {}
- _ => err!(),
- }
- }
- 0xED => {
- match (second, next!() & !CONT_MASK) {
- (0x80...0x9F, TAG_CONT_U8) => {}
- _ => err!(),
+ if next!() & !CONT_MASK != TAG_CONT_U8 {
+ err!(Some(2))
}
}
- 0xF0 => {
- match (second, next!() & !CONT_MASK, next!() & !CONT_MASK) {
- (0x90...0xBF, TAG_CONT_U8, TAG_CONT_U8) => {}
- _ => err!(),
+ 4 => {
+ match (first, next!()) {
+ (0xF0, 0x90...0xBF) |
+ (0xF1...0xF3, 0x80...0xBF) |
+ (0xF4, 0x80...0x8F) => {}
+ _ => err!(Some(1)),
}
- }
- 0xF1...0xF3 => {
- match (second & !CONT_MASK, next!() & !CONT_MASK, next!() & !CONT_MASK) {
- (TAG_CONT_U8, TAG_CONT_U8, TAG_CONT_U8) => {}
- _ => err!(),
+ if next!() & !CONT_MASK != TAG_CONT_U8 {
+ err!(Some(2))
+ }
+ if next!() & !CONT_MASK != TAG_CONT_U8 {
+ err!(Some(3))
}
}
- 0xF4 => {
- match (second, next!() & !CONT_MASK, next!() & !CONT_MASK) {
- (0x80...0x8F, TAG_CONT_U8, TAG_CONT_U8) => {}
- _ => err!(),
- }
- }
- _ => err!(),
+ _ => err!(Some(1)),
}
- offset += 1;
- if offset == len {
+ index += 1;
+ if index == len {
break 'outer;
}
- first = v[offset];
+ first = v[index];
// This check is separate from the above `match`, because merging
- // this check into it causes a spectacular performance drop
- // (over twice as slow).
+ // this check into it causes a notable performance drop.
if first < 0x80 {
- offset += 1;
+ index += 1;
continue 'outer;
}
continue 'inner;
}
}
Ok(())
}
+// https://tools.ietf.org/html/rfc3629
+static UTF8_CHAR_WIDTH: [u8; 256] = [
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x1F
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x3F
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x5F
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x7F
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0x9F
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0xBF
+0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // 0xDF
+3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, // 0xEF
+4,4,4,4,4,0,0,0,0,0,0,0,0,0,0,0, // 0xFF
+];
+
/// Mask of the value bits of a continuation byte
const CONT_MASK: u8 = 0b0011_1111;
/// Value of the tag bits (tag mask is !CONT_MASK) of a continuation byte
const TAG_CONT_U8: u8 = 0b1000_0000;
--- a/toolkit/library/gtest/rust/Cargo.lock
+++ b/toolkit/library/gtest/rust/Cargo.lock
@@ -427,34 +427,34 @@ dependencies = [
[[package]]
name = "either"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "encoding_c"
-version = "0.7.6"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
- "encoding_rs 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "encoding_glue"
version = "0.1.0"
dependencies = [
- "encoding_rs 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"nserror 0.1.0",
"nsstring 0.1.0",
]
[[package]]
name = "encoding_rs"
-version = "0.6.11"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"simd 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "env_logger"
@@ -564,17 +564,17 @@ name = "gkrust-shared"
version = "0.1.0"
dependencies = [
"audioipc-client 0.1.0",
"audioipc-server 0.1.0",
"cubeb 0.3.0",
"cubeb-backend 0.2.0",
"cubeb-core 0.1.0",
"cubeb-pulse 0.0.1",
- "encoding_c 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_c 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"encoding_glue 0.1.0",
"geckoservo 0.0.1",
"mp4parse_capi 0.8.0",
"nserror 0.1.0",
"nsstring 0.1.0",
"rust_url_capi 0.0.1",
"webrender_bindings 0.1.0",
]
@@ -1573,18 +1573,18 @@ dependencies = [
"checksum darling 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9861a8495606435477df581bc858ccf15a3469747edf175b94a4704fd9aaedac"
"checksum darling_core 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1486a8b00b45062c997f767738178b43219133dd0c8c826cb811e60563810821"
"checksum darling_macro 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8a86ec160aa0c3dd492dd4a14ec8104ad8f1a9400a820624db857998cc1f80f9"
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab"
"checksum dtoa-short 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fe6f727b406462fd57c95fed84d1b0dbfb5f0136fcac005adba9ea0367c05cc8"
"checksum dwrote 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "36e3b27cd0b8a68e00f07e8d8e1e4f4d8a6b8b873290a734f63bd56d792d23e1"
"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a"
-"checksum encoding_c 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1e563195f16d7439103c49281b8b00bd0f223c16e19f0d500448239a27712570"
-"checksum encoding_rs 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "e00a1b1e95eb46988805ceee6f34cd95c46a6753e290cb3ff0486931989d4a4c"
+"checksum encoding_c 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "93ec52324ca72f423237a413ca0e1c60654c8b3d0934fcd5fd888508dfcc4ba7"
+"checksum encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6f0a39f0e2f497d3c2e6a5529a0ec4fc640084fa401493c640421673471f8b72"
"checksum env_logger 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ed39959122ea027670b704fb70539f4286ddf4a49eefede23bf0b4b2a069ec03"
"checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8"
"checksum euclid 0.15.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7be9fcb1ce77782eb620253eb02bc1f000545f3c360841a26cda572f10fad4ff"
"checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344"
"checksum freetype 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "398b8a11884898184d55aca9806f002b3cf68f0e860e0cbb4586f834ee39b0e7"
"checksum fs2 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ab76cfd2aaa59b7bf6688ad9ba15bbae64bff97f04ea02144cfd3443e5c2866"
"checksum futures 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "55f0008e13fc853f79ea8fc86e931486860d4c4c156cdffb59fa5f7fa833660a"
"checksum fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
--- a/toolkit/library/gtest/rust/Cargo.toml
+++ b/toolkit/library/gtest/rust/Cargo.toml
@@ -9,17 +9,16 @@ description = "Testing code for libgkrus
bindgen = ["gkrust-shared/bindgen"]
servo = ["gkrust-shared/servo"]
quantum_render = ["gkrust-shared/quantum_render"]
cubeb-remoting = ["gkrust-shared/cubeb-remoting"]
cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
gecko_debug = ["gkrust-shared/gecko_debug"]
simd-accel = ["gkrust-shared/simd-accel"]
no-static-ideograph-encoder-tables = ["gkrust-shared/no-static-ideograph-encoder-tables"]
-# parallel-utf8 = ["gkrust-shared/parallel-utf8"]
[dependencies]
mp4parse-gtest = { path = "../../../../dom/media/gtest" }
nsstring-gtest = { path = "../../../../xpcom/rust/nsstring/gtest" }
gkrust-shared = { path = "../../rust/shared" }
[lib]
path = "lib.rs"
--- a/toolkit/library/rust/Cargo.lock
+++ b/toolkit/library/rust/Cargo.lock
@@ -425,34 +425,34 @@ dependencies = [
[[package]]
name = "either"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "encoding_c"
-version = "0.7.6"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
- "encoding_rs 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "encoding_glue"
version = "0.1.0"
dependencies = [
- "encoding_rs 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"nserror 0.1.0",
"nsstring 0.1.0",
]
[[package]]
name = "encoding_rs"
-version = "0.6.11"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"simd 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "env_logger"
@@ -562,17 +562,17 @@ name = "gkrust-shared"
version = "0.1.0"
dependencies = [
"audioipc-client 0.1.0",
"audioipc-server 0.1.0",
"cubeb 0.3.0",
"cubeb-backend 0.2.0",
"cubeb-core 0.1.0",
"cubeb-pulse 0.0.1",
- "encoding_c 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_c 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"encoding_glue 0.1.0",
"geckoservo 0.0.1",
"mp4parse_capi 0.8.0",
"nserror 0.1.0",
"nsstring 0.1.0",
"rust_url_capi 0.0.1",
"webrender_bindings 0.1.0",
]
@@ -1560,18 +1560,18 @@ dependencies = [
"checksum darling 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9861a8495606435477df581bc858ccf15a3469747edf175b94a4704fd9aaedac"
"checksum darling_core 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1486a8b00b45062c997f767738178b43219133dd0c8c826cb811e60563810821"
"checksum darling_macro 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8a86ec160aa0c3dd492dd4a14ec8104ad8f1a9400a820624db857998cc1f80f9"
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab"
"checksum dtoa-short 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fe6f727b406462fd57c95fed84d1b0dbfb5f0136fcac005adba9ea0367c05cc8"
"checksum dwrote 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "36e3b27cd0b8a68e00f07e8d8e1e4f4d8a6b8b873290a734f63bd56d792d23e1"
"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a"
-"checksum encoding_c 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1e563195f16d7439103c49281b8b00bd0f223c16e19f0d500448239a27712570"
-"checksum encoding_rs 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "e00a1b1e95eb46988805ceee6f34cd95c46a6753e290cb3ff0486931989d4a4c"
+"checksum encoding_c 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "93ec52324ca72f423237a413ca0e1c60654c8b3d0934fcd5fd888508dfcc4ba7"
+"checksum encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6f0a39f0e2f497d3c2e6a5529a0ec4fc640084fa401493c640421673471f8b72"
"checksum env_logger 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ed39959122ea027670b704fb70539f4286ddf4a49eefede23bf0b4b2a069ec03"
"checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8"
"checksum euclid 0.15.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7be9fcb1ce77782eb620253eb02bc1f000545f3c360841a26cda572f10fad4ff"
"checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344"
"checksum freetype 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "398b8a11884898184d55aca9806f002b3cf68f0e860e0cbb4586f834ee39b0e7"
"checksum fs2 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ab76cfd2aaa59b7bf6688ad9ba15bbae64bff97f04ea02144cfd3443e5c2866"
"checksum futures 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "55f0008e13fc853f79ea8fc86e931486860d4c4c156cdffb59fa5f7fa833660a"
"checksum fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
--- a/toolkit/library/rust/Cargo.toml
+++ b/toolkit/library/rust/Cargo.toml
@@ -9,17 +9,16 @@ description = "Rust code for libxul"
bindgen = ["gkrust-shared/bindgen"]
servo = ["gkrust-shared/servo"]
quantum_render = ["gkrust-shared/quantum_render"]
cubeb-remoting = ["gkrust-shared/cubeb-remoting"]
cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
gecko_debug = ["gkrust-shared/gecko_debug"]
simd-accel = ["gkrust-shared/simd-accel"]
no-static-ideograph-encoder-tables = ["gkrust-shared/no-static-ideograph-encoder-tables"]
-# parallel-utf8 = ["gkrust-shared/parallel-utf8"]
[dependencies]
gkrust-shared = { path = "shared" }
[lib]
path = "lib.rs"
crate-type = ["staticlib"]
test = false
--- a/toolkit/library/rust/shared/Cargo.toml
+++ b/toolkit/library/rust/shared/Cargo.toml
@@ -11,32 +11,31 @@ mp4parse_capi = { path = "../../../../me
nsstring = { path = "../../../../xpcom/rust/nsstring" }
nserror = { path = "../../../../xpcom/rust/nserror" }
rust_url_capi = { path = "../../../../netwerk/base/rust-url-capi" }
webrender_bindings = { path = "../../../../gfx/webrender_bindings", optional = true }
cubeb-pulse = { path = "../../../../media/libcubeb/cubeb-pulse-rs", optional = true, features=["pulse-dlopen"] }
cubeb-core = { path = "../../../../media/cubeb-rs/cubeb-core", optional = true }
cubeb = { path = "../../../../media/cubeb-rs/cubeb-api", optional = true }
cubeb-backend = { path = "../../../../media/cubeb-rs/cubeb-backend", optional = true }
-encoding_c = "0.7.1"
+encoding_c = "0.8.0"
encoding_glue = { path = "../../../../intl/encoding_glue" }
audioipc-client = { path = "../../../../media/audioipc/client", optional = true }
audioipc-server = { path = "../../../../media/audioipc/server", optional = true }
[features]
default = []
bindgen = ["geckoservo/bindgen"]
servo = ["geckoservo"]
quantum_render = ["webrender_bindings"]
cubeb-remoting = ["cubeb-core", "cubeb", "cubeb-backend", "audioipc-client", "audioipc-server"]
cubeb_pulse_rust = ["cubeb-pulse"]
gecko_debug = ["geckoservo/gecko_debug"]
simd-accel = ["encoding_c/simd-accel", "encoding_glue/simd-accel"]
no-static-ideograph-encoder-tables = ["encoding_c/no-static-ideograph-encoder-tables", "encoding_glue/no-static-ideograph-encoder-tables"]
-parallel-utf8 = ["encoding_c/parallel-utf8", "encoding_glue/parallel-utf8"]
[lib]
path = "lib.rs"
test = false
doctest = false
bench = false
doc = false
plugin = false