--- a/third_party/rust/encoding_rs/.cargo-checksum.json
+++ b/third_party/rust/encoding_rs/.cargo-checksum.json
@@ -1,1 +1,1 @@
-{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",".travis.yml":"dc509cc3b8f44fbdf1d806f533c3f005afaf0fd77cd266b38cb69bab3e4ea136","CONTRIBUTING.md":"e4ffa92c979c7e6ca7b676842a708ea05b84181327fcde43dfcd8038b678a057","COPYRIGHT":"20d4fff11cca11529df3f02096fbe8ffe350219cdb07cdedea34e6a762866da5","Cargo.toml":"3865d683e7d7fc4a0e05f2eaf9f808ea6b550eb24fe4f99377749bc1185195bb","Ideas.md":"c1be4cc91621f52f38ea7febda7a4bb68086189cacc834c7edac4ba1a9da02fe","LICENSE-APACHE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","LICENSE-MIT":"74aa8b6d04c36bb640ee81187a3f24a2fa94e36d4c1d4f2ca164c3784ae87a83","README.md":"7d93a6ff036022bd986fc608ea5cda33cd3e865cd85c7ad55494336304d9163e","generate-encoding-data.py":"8a0a5162098d355e4df63532819769fd6626a66a0aa93f2762e315d6147aa0a5","rustfmt.toml":"c01c06dfbdfcf30730535aab911d69068febb921e2faef9571ceeb6a5c2a3eab","src/ascii.rs":"a74affce835efdf930256ee7153fc29e08794df369b895df262b1920bff29b5a","src/big5.rs":"614d479aabc63007f778d1f776a37b885e13d20b7c6c7a2818a729bde342f8a6","src/data.rs":"412c842c698c3ce1cec4a27ab19ca275372ac28940ac49cdf3e0dad71a2c2812","src/euc_jp.rs":"feda0ade5e1c3e4abd7637c59373b977662007990fd164ea7db1acc502ba3534","src/euc_kr.rs":"23e08359ccbe7602f3a90fce78dc76fd4065c236820ac0d11c9d9325045da0e6","src/gb18030.rs":"aa9de27a41715dfb02a3b9161d86e3775f635f625f70d3abaadcd583ee7022c0","src/handles.rs":"8b0691ab21d638bd20078e33247f13afbc8012ff4b843a2fd03e3314353e8520","src/iso_2022_jp.rs":"285e7cea6df41d182a345a0f394a2348b1c313f0d55ed48c349824f2a6aff526","src/lib.rs":"b2898ab31ccc5626474bdcb18e9447bc53ba1c225d8fb4deecadf59e0d6d09d8","src/macros.rs":"9ab30e7194f61f268cd7d899cabb06ff9ca7717663926fd583b20334f49ac8d3","src/replacement.rs":"782f03f04d110e9a0656262bf4296aa0ab8199e196cb63239c30d9649996caa4","src/shift_jis.rs":"84df4ff58b60e0827d6c0c7049f2cf19033f2b9e25a9186bcfb0bbb05e87b380","src/simd_funcs.rs":"a2def7fc35fa8d3eeac1d7fbcd005d50b018a336a2fa7cfd42bfdcf04e564008","src/single_byte.rs":"0342a921427ed160f5cbe4532490aff5db00886a36b70273f54d8f6a9dcf6974","src/test_data/big5_in.txt":"4c5a8691f8dc717311889c63894026d2fb62725a86c4208ca274a9cc8d42a503","src/test_data/big5_in_ref.txt":"99d399e17750cf9c7cf30bb253dbfe35b81c4fcbdead93cfa48b1429213473c7","src/test_data/big5_out.txt":"6193ca97c297aa20e09396038d18e938bb7ea331c26f0f2454097296723a0b13","src/test_data/big5_out_ref.txt":"36567691f557df144f6cc520015a87038dfa156f296fcf103b56ae9a718be1fc","src/test_data/euc_kr_in.txt":"c86a7224f3215fa0d04e685622a752fdc72763e8ae076230c7fd62de57ec4074","src/test_data/euc_kr_in_ref.txt":"1f419f4ca47d708b54c73c461545a022ae2e20498fdbf8005a483d752a204883","src/test_data/euc_kr_out.txt":"e7f32e026f70be1e1b58e0047baf7d3d2c520269c4f9b9992e158b4decb0a1a3","src/test_data/euc_kr_out_ref.txt":"c9907857980b20b8e9e3b584482ed6567a2be6185d72237b6322f0404944924e","src/test_data/gb18030_in.txt":"ab7231b2d3e9afacdbd7d7f3b9e5361a7ff9f7e1cfdb4f3bd905b9362b309e53","src/test_data/gb18030_in_ref.txt":"dc5069421adca2043c55f5012b55a76fdff651d22e6e699fd0978f8d5706815c","src/test_data/gb18030_out.txt":"f0208d527f5ca63de7d9a0323be8d5cf12d8a104b2943d92c2701f0c3364dac1","src/test_data/gb18030_out_ref.txt":"6819fe47627e4ea01027003fc514b9f21a1322e732d7f1fb92cc6c5455bc6c07","src/test_data/iso_2022_jp_in.txt":"cd24bbdcb1834e25db54646fbf4c41560a13dc7540f6be3dba4f5d97d44513af","src/test_data/iso_2022_jp_in_ref.txt":"3dc4e6a5e06471942d086b16c9440945e78415f6f3f47e43717e4bc2eac2cdf5","src/test_data/iso_2022_jp_out.txt":"9b6f015329dda6c3f9ee5ce6dbd6fa9c89acc21283e886836c78b8d833480c21","src/test_data/iso_2022_jp_out_ref.txt":"78cb260093a20116ad9a42f43b05d1848c5ab100b6b9a850749809e943884b35","src/test_data/jis0208_in.txt":"6df3030553ffb0a6615bb33dc8ea9dca6d9623a9028e2ffec754ce3c3da824cc","src/test_data/jis0208_in_ref.txt":"3dc4e6a5e06471942d086b16c9440945e78415f6f3f47e43717e4bc2eac2cdf5","src/test_data/jis0208_out.txt":"4ec24477e1675ce750733bdc3c5add1cd27b6bd4ce1f09289564646e9654e857","src/test_data/jis0208_out_ref.txt":"c3e1cef5032b2b1d93a406f31ff940c4e2dfe8859b8b17ca2761fee7a75a0e48","src/test_data/jis0212_in.txt":"c011f0dd72bd7c8cd922df9374ef8d2769a77190514c77f6c62b415852eeb9fe","src/test_data/jis0212_in_ref.txt":"7d9458b3d2f73e7092a7f505c08ce1d233dde18aa679fbcf9889256239cc9e06","src/test_data/shift_jis_in.txt":"02e389ccef0dd2122e63f503899402cb7f797912c2444cc80ab93131116c5524","src/test_data/shift_jis_in_ref.txt":"512f985950ca902e643c88682dba9708b7c38d3c5ec2925168ab00ac94ab19f9","src/test_data/shift_jis_out.txt":"5fbc44da7bf639bf6cfe0fa1fd3eba7102b88f81919c9ea991302712f69426fb","src/test_data/shift_jis_out_ref.txt":"466322c6fed8286c64582731755290c2296508efdd258826e6279686649b481f","src/test_labels_names.rs":"c962c7aeac3d9ef2aca70c9e21983b231d4cf998cb06879374b0401e5149d1da","src/testing.rs":"60f85c6fb63fd4ab62e90dfa005920e79e0e1885795dc13a7a3c1980507925b1","src/utf_16.rs":"1d2c40857c946f6eecf724efc60a196865b4d84a59b08b42fbe4576fa8308fd0","src/utf_8.rs":"f9b9ea26212598607b925d231d424abdc9d16e9f6b66345ea155406953894b0e","src/utf_8_core.rs":"bbc010dbdfed0f5e7c48a1ab0772eaf2e27711b789bb82f71a678f2240158a65","src/variant.rs":"93dfec2dcfc9fd9711bb55d48177f4a0e9479c7fbd055f08db3853338569da83","src/x_user_defined.rs":"420fae797ea94e7a51eb005b97621ab32d68a8532c565afc60ecce6bdd84b6bd"},"package":"6f0a39f0e2f497d3c2e6a5529a0ec4fc640084fa401493c640421673471f8b72"}
\ No newline at end of file
+{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",".travis.yml":"dc509cc3b8f44fbdf1d806f533c3f005afaf0fd77cd266b38cb69bab3e4ea136","CONTRIBUTING.md":"e4ffa92c979c7e6ca7b676842a708ea05b84181327fcde43dfcd8038b678a057","COPYRIGHT":"20d4fff11cca11529df3f02096fbe8ffe350219cdb07cdedea34e6a762866da5","Cargo.toml":"2bed851f8857df3daf0cef25b3588a0841241624ab326e81cce188a598395352","Ideas.md":"7fbeddb0f8ba7b233673ee705997adc2fddb1636a17fe662532b35ef2810a51d","LICENSE-APACHE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","LICENSE-MIT":"74aa8b6d04c36bb640ee81187a3f24a2fa94e36d4c1d4f2ca164c3784ae87a83","README.md":"cf09a31640f5d556661e2fbe1d07f76046eff94daf6ebb895d14499653b59bde","generate-encoding-data.py":"8a0a5162098d355e4df63532819769fd6626a66a0aa93f2762e315d6147aa0a5","rustfmt.toml":"c01c06dfbdfcf30730535aab911d69068febb921e2faef9571ceeb6a5c2a3eab","src/ascii.rs":"1e9f9a02130933fdba6b7606b47c2308afd6d16df064779245226060211af0ce","src/big5.rs":"780ae537353f899a5772a9e7d062441041276e1eb1506a013e4280c5cda6bb93","src/data.rs":"412c842c698c3ce1cec4a27ab19ca275372ac28940ac49cdf3e0dad71a2c2812","src/euc_jp.rs":"feda0ade5e1c3e4abd7637c59373b977662007990fd164ea7db1acc502ba3534","src/euc_kr.rs":"2699c52055882e34ba4e12d072b8161c635840f3620075ca3f10986aec0e8d3b","src/gb18030.rs":"aa9de27a41715dfb02a3b9161d86e3775f635f625f70d3abaadcd583ee7022c0","src/handles.rs":"c07a3738e43e8aae11108a30d34067c31ddc5d22074b85ef393f00abcc1f4e01","src/iso_2022_jp.rs":"1f780c3ff72f1a867d6c5782135cd01427eca6d74f0dd6cb23c1406b5163af1a","src/lib.rs":"250cabe96d561b38eef9e26141707904b66b612007098287dd2b245240c5a5be","src/macros.rs":"9ab30e7194f61f268cd7d899cabb06ff9ca7717663926fd583b20334f49ac8d3","src/replacement.rs":"782f03f04d110e9a0656262bf4296aa0ab8199e196cb63239c30d9649996caa4","src/shift_jis.rs":"84df4ff58b60e0827d6c0c7049f2cf19033f2b9e25a9186bcfb0bbb05e87b380","src/simd_funcs.rs":"6c5beb75d30c1b3a2e6e9dd86209f9748313ee75f5b43a9d7f5176be310ffabb","src/single_byte.rs":"b3fadb4fa1e66b00efc12b8850b3076580a8cd73c9ed810a19421fd3ade9bbf1","src/test_data/big5_in.txt":"4c5a8691f8dc717311889c63894026d2fb62725a86c4208ca274a9cc8d42a503","src/test_data/big5_in_ref.txt":"99d399e17750cf9c7cf30bb253dbfe35b81c4fcbdead93cfa48b1429213473c7","src/test_data/big5_out.txt":"6193ca97c297aa20e09396038d18e938bb7ea331c26f0f2454097296723a0b13","src/test_data/big5_out_ref.txt":"36567691f557df144f6cc520015a87038dfa156f296fcf103b56ae9a718be1fc","src/test_data/euc_kr_in.txt":"c86a7224f3215fa0d04e685622a752fdc72763e8ae076230c7fd62de57ec4074","src/test_data/euc_kr_in_ref.txt":"1f419f4ca47d708b54c73c461545a022ae2e20498fdbf8005a483d752a204883","src/test_data/euc_kr_out.txt":"e7f32e026f70be1e1b58e0047baf7d3d2c520269c4f9b9992e158b4decb0a1a3","src/test_data/euc_kr_out_ref.txt":"c9907857980b20b8e9e3b584482ed6567a2be6185d72237b6322f0404944924e","src/test_data/gb18030_in.txt":"ab7231b2d3e9afacdbd7d7f3b9e5361a7ff9f7e1cfdb4f3bd905b9362b309e53","src/test_data/gb18030_in_ref.txt":"dc5069421adca2043c55f5012b55a76fdff651d22e6e699fd0978f8d5706815c","src/test_data/gb18030_out.txt":"f0208d527f5ca63de7d9a0323be8d5cf12d8a104b2943d92c2701f0c3364dac1","src/test_data/gb18030_out_ref.txt":"6819fe47627e4ea01027003fc514b9f21a1322e732d7f1fb92cc6c5455bc6c07","src/test_data/iso_2022_jp_in.txt":"cd24bbdcb1834e25db54646fbf4c41560a13dc7540f6be3dba4f5d97d44513af","src/test_data/iso_2022_jp_in_ref.txt":"3dc4e6a5e06471942d086b16c9440945e78415f6f3f47e43717e4bc2eac2cdf5","src/test_data/iso_2022_jp_out.txt":"9b6f015329dda6c3f9ee5ce6dbd6fa9c89acc21283e886836c78b8d833480c21","src/test_data/iso_2022_jp_out_ref.txt":"78cb260093a20116ad9a42f43b05d1848c5ab100b6b9a850749809e943884b35","src/test_data/jis0208_in.txt":"6df3030553ffb0a6615bb33dc8ea9dca6d9623a9028e2ffec754ce3c3da824cc","src/test_data/jis0208_in_ref.txt":"3dc4e6a5e06471942d086b16c9440945e78415f6f3f47e43717e4bc2eac2cdf5","src/test_data/jis0208_out.txt":"4ec24477e1675ce750733bdc3c5add1cd27b6bd4ce1f09289564646e9654e857","src/test_data/jis0208_out_ref.txt":"c3e1cef5032b2b1d93a406f31ff940c4e2dfe8859b8b17ca2761fee7a75a0e48","src/test_data/jis0212_in.txt":"c011f0dd72bd7c8cd922df9374ef8d2769a77190514c77f6c62b415852eeb9fe","src/test_data/jis0212_in_ref.txt":"7d9458b3d2f73e7092a7f505c08ce1d233dde18aa679fbcf9889256239cc9e06","src/test_data/shift_jis_in.txt":"02e389ccef0dd2122e63f503899402cb7f797912c2444cc80ab93131116c5524","src/test_data/shift_jis_in_ref.txt":"512f985950ca902e643c88682dba9708b7c38d3c5ec2925168ab00ac94ab19f9","src/test_data/shift_jis_out.txt":"5fbc44da7bf639bf6cfe0fa1fd3eba7102b88f81919c9ea991302712f69426fb","src/test_data/shift_jis_out_ref.txt":"466322c6fed8286c64582731755290c2296508efdd258826e6279686649b481f","src/test_labels_names.rs":"c962c7aeac3d9ef2aca70c9e21983b231d4cf998cb06879374b0401e5149d1da","src/testing.rs":"60f85c6fb63fd4ab62e90dfa005920e79e0e1885795dc13a7a3c1980507925b1","src/utf_16.rs":"1d2c40857c946f6eecf724efc60a196865b4d84a59b08b42fbe4576fa8308fd0","src/utf_8.rs":"34218c7f4faa81883492fdfeb303b7e77710121b06e8342ac62ccb3d6eb16a37","src/utf_8_core.rs":"bbc010dbdfed0f5e7c48a1ab0772eaf2e27711b789bb82f71a678f2240158a65","src/variant.rs":"93dfec2dcfc9fd9711bb55d48177f4a0e9479c7fbd055f08db3853338569da83","src/x_user_defined.rs":"84d054eec249dd676452585f8eb13dc851095021ed6e1f8c79e952c6d81751df"},"package":"f5215aabf22b83153be3ee44dfe3f940214541b2ce13d419c55e7a115c8c51a9"}
\ No newline at end of file
--- a/third_party/rust/encoding_rs/Cargo.toml
+++ b/third_party/rust/encoding_rs/Cargo.toml
@@ -7,44 +7,44 @@
#
# If you believe there's an error in this file please file an
# issue against the rust-lang/cargo repository. If you're
# editing this file be aware that the upstream Cargo.toml
# will likely look very different (and much more reasonable)
[package]
name = "encoding_rs"
-version = "0.7.0"
+version = "0.7.1"
authors = ["Henri Sivonen <hsivonen@hsivonen.fi>"]
description = "A Gecko-oriented implementation of the Encoding Standard"
homepage = "https://docs.rs/encoding_rs/"
documentation = "https://docs.rs/encoding_rs/"
readme = "README.md"
keywords = ["encoding", "web", "unicode", "charset"]
categories = ["text-processing", "encoding", "web-programming", "email"]
license = "MIT/Apache-2.0"
repository = "https://github.com/hsivonen/encoding_rs"
[profile.release]
lto = true
-[dependencies.cfg-if]
-version = "0.1.0"
+[dependencies.serde]
+version = "1.0"
+optional = true
[dependencies.simd]
version = "0.2.0"
optional = true
-[dependencies.serde]
+[dependencies.cfg-if]
+version = "0.1.0"
+[dev-dependencies.serde_derive]
version = "1.0"
-optional = true
-[dev-dependencies.serde_derive]
+
+[dev-dependencies.serde_json]
version = "1.0"
[dev-dependencies.bincode]
version = "0.8"
-[dev-dependencies.serde_json]
-version = "1.0"
-
[features]
+simd-accel = ["simd"]
no-static-ideograph-encoder-tables = []
-simd-accel = ["simd"]
[badges.travis-ci]
repository = "hsivonen/encoding_rs"
--- a/third_party/rust/encoding_rs/Ideas.md
+++ b/third_party/rust/encoding_rs/Ideas.md
@@ -2,24 +2,16 @@ This document contains notes about vario
are not being actively pursued.
## Next byte is non-ASCII after ASCII optimization
The current plan for a SIMD-accelerated inner loop for handling ASCII bytes
makes no use of the bit of information that if the buffers didn't end but the
ASCII loop exited, the next byte will not be an ASCII byte.
-## The structure of handles.rs and bound checks
-
-handles.rs is designed to make it possible to avoid bound checks when writing
-to the slices. While it would be possible to omit the bound checks manually,
-it probably makes more sense to carry out an investigation to make sure that
-the compiler performs the omission. If not, it makes more sense to file a bug
-on the compiler than to omit the checks manually.
-
## Handling ASCII with table lookups when decoding single-byte to UTF-16
Both uconv and ICU outperform encoding_rs when decoding single-byte to UTF-16.
unconv doesn't even do anything fancy to manually unroll the loop (see below).
Both handle even the ASCII range using table lookup. That is, there's no branch
for checking if we're in the lower or upper half of the encoding.
However, adding SIMD acceleration for the ASCII half will likely be a bigger
@@ -70,8 +62,17 @@ These two are ordered by radical and the
they should be mostly Unicode-ordered, although at least Level 2 Hanzi isn't
fully Unicode-ordered. Is "mostly" good enough for encode accelelation?
## Create a `divmod_94()` function
Experiment with a function that computes `(i / 94, i % 94)` more efficiently
than generic code.
+## Align writes on Aarch64
+
+On [Cortex-A57](https://stackoverflow.com/questions/45714535/performance-of-unaligned-simd-load-store-on-aarch64/45938112#45938112
+), it might be a good idea to move the destination into 16-byte alignment.
+
+## Unalign UTF-8 validation on Aarch64
+
+Currently, Aarch64 runs the generic ALU UTF-8 validation code that aligns
+reads. That's probably unnecessary on Aarch64. (SIMD was slower than ALU!)
--- a/third_party/rust/encoding_rs/README.md
+++ b/third_party/rust/encoding_rs/README.md
@@ -58,27 +58,33 @@ online.
An FFI layer for encoding_rs is available as a
[separate crate](https://github.com/hsivonen/encoding_c). The crate comes
with a [demo C++ wrapper](https://github.com/hsivonen/encoding_c/blob/master/include/encoding_rs_cpp.h)
using the C++ standard library and [GSL](https://github.com/Microsoft/GSL/) types.
For the Gecko context, there's a
[C++ wrapper using the MFBT/XPCOM types](https://searchfox.org/mozilla-central/source/intl/Encoding.h#100).
+## Sample programs
+
+* [Rust](https://github.com/hsivonen/recode_rs)
+* [C](https://github.com/hsivonen/recode_c)
+* [C++](https://github.com/hsivonen/recode_cpp)
+
## Optional features
There are currently three optional cargo features:
### `simd-accel`
-Enables SSE2 acceleration on x86, x86_64 and Aarch64. Requires nightly Rust.
-_Enabling this cargo feature is recommended when building for x86, x86_64 or
-Aarch64 on nightly Rust._ The intention is for the functionality enabled by
-this feature to become the normal on-by-default behavior once explicit SIMD
-becames available on all Rust release channels.
+Enables SSE2 acceleration on x86 and x86_64 and NEON acceleration on Aarch64.
+Requires nightly Rust. _Enabling this cargo feature is recommended when
+building for x86, x86_64 or Aarch64 on nightly Rust._ The intention is for the
+functionality enabled by this feature to become the normal on-by-default
+behavior once explicit SIMD becames available on all Rust release channels.
Enabling this feature breaks the build unless the target is x86 with SSE2
(Rust's default 32-bit x86 target, `i686`, has SSE2, but Linux distros may
use an x86 target without SSE2, i.e. `i586` in `rustup` terms), x86_64 or
Aarch64.
### `serde`
@@ -175,23 +181,28 @@ used in Firefox.
end of the label/name to the start.
- [x] Make labels with non-ASCII bytes fail fast.
- [ ] Parallelize UTF-8 validation using [Rayon](https://github.com/nikomatsakis/rayon).
- [x] Provide an XPCOM/MFBT-flavored C++ API.
- [ ] Investigate accelerating single-byte encode with a single fast-tracked
range per encoding.
- [x] Replace uconv with encoding_rs in Gecko.
- [x] Implement the rust-encoding API in terms of encoding_rs.
-- [ ] Add SIMD acceleration for Aarch64.
+- [x] Add SIMD acceleration for Aarch64.
- [ ] Investigate the use of NEON on 32-bit ARM.
- [ ] Investigate Björn Höhrmann's lookup table acceleration for UTF-8 as
adapted to Rust in rust-encoding.
## Release Notes
+### 0.7.1
+
+* When encoding from invalid UTF-16, correctly handle U+DC00 followed by
+ another low surrogate.
+
### 0.7.0
* [Make `replacement` a label of the replacement
encoding.](https://github.com/whatwg/encoding/issues/70) (Spec change.)
* Remove `Encoding::for_name()`. (`Encoding::for_label(foo).unwrap()` is
now close enough after the above label change.)
* Remove the `parallel-utf8` cargo feature.
* Add optional Serde support for `&'static Encoding`.
--- a/third_party/rust/encoding_rs/src/ascii.rs
+++ b/third_party/rust/encoding_rs/src/ascii.rs
@@ -77,18 +77,18 @@ macro_rules! ascii_alu {
// } else {
// basic_latin_to_ascii
// let dst_until_alignment = (ALIGNMENT - ((dst as usize) & ALIGNMENT_MASK)) & ALIGNMENT_MASK;
// if (src.offset(dst_until_alignment as isize) as usize) & ALIGNMENT_MASK != 0 {
// break;
// }
// dst_until_alignment
// }
- };
- if until_alignment + STRIDE_SIZE <= len {
+ };
+ if until_alignment + STRIDE_SIZE <= len {
// Moving pointers to alignment seems to be a pessimization on
// x86_64 for operations that have UTF-16 as the internal
// Unicode representation. However, since it seems to be a win
// on ARM (tested ARMv7 code running on ARMv8 [rpi3]), except
// mixed results when encoding from UTF-16 and since x86 and
// x86_64 should be using SSE2 in due course, keeping the move
// to alignment here. It would be good to test on more ARM CPUs
// and on real MIPS and POWER hardware.
@@ -350,38 +350,37 @@ macro_rules! ascii_to_basic_latin_simd_s
$load:ident,
$store:ident) => (
#[inline(always)]
pub unsafe fn $name(src: *const u8, dst: *mut u16) -> bool {
let simd = $load(src);
if !is_ascii(simd) {
return false;
}
- let (first, second) = unpack(simd);
+ let (first, second) = simd_unpack(simd);
$store(dst, first);
$store(dst.offset(8), second);
true
});
}
#[allow(unused_macros)]
macro_rules! basic_latin_to_ascii_simd_stride {
($name:ident,
$load:ident,
$store:ident) => (
#[inline(always)]
pub unsafe fn $name(src: *const u16, dst: *mut u8) -> bool {
let first = $load(src);
let second = $load(src.offset(8));
- match pack_basic_latin(first, second) {
- Some(packed) => {
- $store(dst, packed);
- true
- },
- None => false,
+ if is_basic_latin(first | second) {
+ $store(dst, simd_pack(first, second));
+ true
+ } else {
+ false
}
});
}
cfg_if! {
if #[cfg(all(feature = "simd-accel", target_endian = "little", target_arch = "aarch64"))] {
// SIMD with the same instructions for aligned and unaligned loads and stores
--- a/third_party/rust/encoding_rs/src/big5.rs
+++ b/third_party/rust/encoding_rs/src/big5.rs
@@ -385,9 +385,23 @@ mod tests {
fn test_big5_encode_all() {
let input = include_str!("test_data/big5_out.txt");
let expectation = include_bytes!("test_data/big5_out_ref.txt");
let (cow, encoding, had_errors) = BIG5.encode(input);
assert!(!had_errors, "Should not have had errors.");
assert_eq!(encoding, BIG5);
assert_eq!(&cow[..], &expectation[..]);
}
+
+ #[test]
+ fn test_big5_encode_from_two_low_surrogates() {
+ let expectation = b"��";
+ let mut output = [0u8; 40];
+ let mut encoder = BIG5.new_encoder();
+ let (result, read, written, had_errors) =
+ encoder.encode_from_utf16(&[0xDC00u16, 0xDEDEu16], &mut output[..], true);
+ assert_eq!(result, CoderResult::InputEmpty);
+ assert_eq!(read, 2);
+ assert_eq!(written, expectation.len());
+ assert!(had_errors);
+ assert_eq!(&output[..written], expectation);
+ }
}
--- a/third_party/rust/encoding_rs/src/euc_kr.rs
+++ b/third_party/rust/encoding_rs/src/euc_kr.rs
@@ -374,9 +374,23 @@ mod tests {
fn test_euc_kr_encode_all() {
let input = include_str!("test_data/euc_kr_out.txt");
let expectation = include_bytes!("test_data/euc_kr_out_ref.txt");
let (cow, encoding, had_errors) = EUC_KR.encode(input);
assert!(!had_errors, "Should not have had errors.");
assert_eq!(encoding, EUC_KR);
assert_eq!(&cow[..], &expectation[..]);
}
+
+ #[test]
+ fn test_euc_kr_encode_from_two_low_surrogates() {
+ let expectation = b"��";
+ let mut output = [0u8; 40];
+ let mut encoder = EUC_KR.new_encoder();
+ let (result, read, written, had_errors) =
+ encoder.encode_from_utf16(&[0xDC00u16, 0xDEDEu16], &mut output[..], true);
+ assert_eq!(result, CoderResult::InputEmpty);
+ assert_eq!(read, 2);
+ assert_eq!(written, expectation.len());
+ assert!(had_errors);
+ assert_eq!(&output[..written], expectation);
+ }
}
--- a/third_party/rust/encoding_rs/src/handles.rs
+++ b/third_party/rust/encoding_rs/src/handles.rs
@@ -323,17 +323,17 @@ impl<'a> Utf16Destination<'a> {
}
#[inline(always)]
pub fn copy_ascii_from_check_space_bmp<'b>
(&'b mut self,
source: &mut ByteSource)
-> CopyAsciiResult<(DecoderResult, usize, usize), (u8, Utf16BmpHandle<'b, 'a>)> {
let non_ascii_ret = {
let src_remaining = &source.slice[source.pos..];
- let mut dst_remaining = &mut self.slice[self.pos..];
+ let dst_remaining = &mut self.slice[self.pos..];
let (pending, length) = if dst_remaining.len() < src_remaining.len() {
(DecoderResult::OutputFull, dst_remaining.len())
} else {
(DecoderResult::InputEmpty, src_remaining.len())
};
match unsafe {
ascii_to_basic_latin(
src_remaining.as_ptr(),
@@ -359,17 +359,17 @@ impl<'a> Utf16Destination<'a> {
#[inline(always)]
pub fn copy_ascii_from_check_space_astral<'b>
(&'b mut self,
source: &mut ByteSource)
-> CopyAsciiResult<(DecoderResult, usize, usize), (u8, Utf16AstralHandle<'b, 'a>)> {
let non_ascii_ret = {
let dst_len = self.slice.len();
let src_remaining = &source.slice[source.pos..];
- let mut dst_remaining = &mut self.slice[self.pos..];
+ let dst_remaining = &mut self.slice[self.pos..];
let (pending, length) = if dst_remaining.len() < src_remaining.len() {
(DecoderResult::OutputFull, dst_remaining.len())
} else {
(DecoderResult::InputEmpty, src_remaining.len())
};
match unsafe {
ascii_to_basic_latin(
src_remaining.as_ptr(),
@@ -396,17 +396,17 @@ impl<'a> Utf16Destination<'a> {
}
}
};
CopyAsciiResult::GoOn((non_ascii_ret, Utf16AstralHandle::new(self)))
}
#[inline(always)]
pub fn copy_utf8_up_to_invalid_from(&mut self, source: &mut ByteSource) {
let src_remaining = &source.slice[source.pos..];
- let mut dst_remaining = &mut self.slice[self.pos..];
+ let dst_remaining = &mut self.slice[self.pos..];
let (read, written) = convert_utf8_to_utf16_up_to_invalid(src_remaining, dst_remaining);
source.pos += read;
self.pos += written;
}
}
// UTF-8 destination
@@ -618,17 +618,17 @@ impl<'a> Utf8Destination<'a> {
#[inline(always)]
pub fn copy_ascii_from_check_space_bmp<'b>
(&'b mut self,
source: &mut ByteSource)
-> CopyAsciiResult<(DecoderResult, usize, usize), (u8, Utf8BmpHandle<'b, 'a>)> {
let non_ascii_ret = {
let dst_len = self.slice.len();
let src_remaining = &source.slice[source.pos..];
- let mut dst_remaining = &mut self.slice[self.pos..];
+ let dst_remaining = &mut self.slice[self.pos..];
let (pending, length) = if dst_remaining.len() < src_remaining.len() {
(DecoderResult::OutputFull, dst_remaining.len())
} else {
(DecoderResult::InputEmpty, src_remaining.len())
};
match unsafe {
ascii_to_ascii(src_remaining.as_ptr(), dst_remaining.as_mut_ptr(), length)
} {
@@ -656,17 +656,17 @@ impl<'a> Utf8Destination<'a> {
#[inline(always)]
pub fn copy_ascii_from_check_space_astral<'b>
(&'b mut self,
source: &mut ByteSource)
-> CopyAsciiResult<(DecoderResult, usize, usize), (u8, Utf8AstralHandle<'b, 'a>)> {
let non_ascii_ret = {
let dst_len = self.slice.len();
let src_remaining = &source.slice[source.pos..];
- let mut dst_remaining = &mut self.slice[self.pos..];
+ let dst_remaining = &mut self.slice[self.pos..];
let (pending, length) = if dst_remaining.len() < src_remaining.len() {
(DecoderResult::OutputFull, dst_remaining.len())
} else {
(DecoderResult::InputEmpty, src_remaining.len())
};
match unsafe {
ascii_to_ascii(src_remaining.as_ptr(), dst_remaining.as_mut_ptr(), length)
} {
@@ -689,17 +689,17 @@ impl<'a> Utf8Destination<'a> {
}
}
};
CopyAsciiResult::GoOn((non_ascii_ret, Utf8AstralHandle::new(self)))
}
#[inline(always)]
pub fn copy_utf8_up_to_invalid_from(&mut self, source: &mut ByteSource) {
let src_remaining = &source.slice[source.pos..];
- let mut dst_remaining = &mut self.slice[self.pos..];
+ let dst_remaining = &mut self.slice[self.pos..];
let min_len = ::std::cmp::min(src_remaining.len(), dst_remaining.len());
// Validate first, then memcpy to let memcpy do its thing even for
// non-ASCII. (And potentially do something better than SSE2 for ASCII.)
let valid_len = utf8_valid_up_to(&src_remaining[..min_len]);
unsafe {
::std::ptr::copy_nonoverlapping(
src_remaining.as_ptr(),
dst_remaining.as_mut_ptr(),
@@ -741,17 +741,17 @@ impl<'a> Utf16Source<'a> {
fn read(&mut self) -> char {
self.old_pos = self.pos;
let unit = self.slice[self.pos] as u32;
self.pos += 1;
let unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
if unit_minus_surrogate_start > (0xDFFF - 0xD800) {
return unsafe { ::std::mem::transmute(unit) };
}
- if unit_minus_surrogate_start <= (0xDFFF - 0xDBFF) {
+ if unit_minus_surrogate_start <= (0xDBFF - 0xD800) {
// high surrogate
if self.pos < self.slice.len() {
let second = self.slice[self.pos] as u32;
let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00);
if second_minus_low_surrogate_start <= (0xDFFF - 0xDC00) {
// The next code unit is a low surrogate. Advance position.
self.pos += 1;
return unsafe {
@@ -778,17 +778,17 @@ impl<'a> Utf16Source<'a> {
self.pos += 1;
if unit < 0x80 {
return Unicode::Ascii(unit as u8);
}
let unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
if unit_minus_surrogate_start > (0xDFFF - 0xD800) {
return Unicode::NonAscii(NonAscii::BmpExclAscii(unit));
}
- if unit_minus_surrogate_start <= (0xDFFF - 0xDBFF) {
+ if unit_minus_surrogate_start <= (0xDBFF - 0xD800) {
// high surrogate
if self.pos < self.slice.len() {
let second = self.slice[self.pos] as u32;
let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00);
if second_minus_low_surrogate_start <= (0xDFFF - 0xDC00) {
// The next code unit is a low surrogate. Advance position.
self.pos += 1;
return Unicode::NonAscii(
@@ -823,17 +823,17 @@ impl<'a> Utf16Source<'a> {
#[inline(always)]
pub fn copy_ascii_to_check_space_two<'b>
(&mut self,
dest: &'b mut ByteDestination<'a>)
-> CopyAsciiResult<(EncoderResult, usize, usize), (NonAscii, ByteTwoHandle<'b, 'a>)> {
let non_ascii_ret = {
let dst_len = dest.slice.len();
let src_remaining = &self.slice[self.pos..];
- let mut dst_remaining = &mut dest.slice[dest.pos..];
+ let dst_remaining = &mut dest.slice[dest.pos..];
let (pending, length) = if dst_remaining.len() < src_remaining.len() {
(EncoderResult::OutputFull, dst_remaining.len())
} else {
(EncoderResult::InputEmpty, src_remaining.len())
};
match unsafe {
basic_latin_to_ascii(
src_remaining.as_ptr(),
@@ -850,17 +850,17 @@ impl<'a> Utf16Source<'a> {
self.pos += consumed;
dest.pos += consumed;
if dest.pos + 1 < dst_len {
self.pos += 1; // commit to reading `non_ascii`
let unit = non_ascii;
let unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
if unit_minus_surrogate_start > (0xDFFF - 0xD800) {
NonAscii::BmpExclAscii(unit)
- } else if unit_minus_surrogate_start <= (0xDFFF - 0xDBFF) {
+ } else if unit_minus_surrogate_start <= (0xDBFF - 0xD800) {
// high surrogate
if self.pos < self.slice.len() {
let second = self.slice[self.pos] as u32;
let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00);
if second_minus_low_surrogate_start <= (0xDFFF - 0xDC00) {
// The next code unit is a low surrogate. Advance position.
self.pos += 1;
NonAscii::Astral(
@@ -897,17 +897,17 @@ impl<'a> Utf16Source<'a> {
#[inline(always)]
pub fn copy_ascii_to_check_space_four<'b>
(&mut self,
dest: &'b mut ByteDestination<'a>)
-> CopyAsciiResult<(EncoderResult, usize, usize), (NonAscii, ByteFourHandle<'b, 'a>)> {
let non_ascii_ret = {
let dst_len = dest.slice.len();
let src_remaining = &self.slice[self.pos..];
- let mut dst_remaining = &mut dest.slice[dest.pos..];
+ let dst_remaining = &mut dest.slice[dest.pos..];
let (pending, length) = if dst_remaining.len() < src_remaining.len() {
(EncoderResult::OutputFull, dst_remaining.len())
} else {
(EncoderResult::InputEmpty, src_remaining.len())
};
match unsafe {
basic_latin_to_ascii(
src_remaining.as_ptr(),
@@ -924,17 +924,17 @@ impl<'a> Utf16Source<'a> {
self.pos += consumed;
dest.pos += consumed;
if dest.pos + 3 < dst_len {
self.pos += 1; // commit to reading `non_ascii`
let unit = non_ascii;
let unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
if unit_minus_surrogate_start > (0xDFFF - 0xD800) {
NonAscii::BmpExclAscii(unit)
- } else if unit_minus_surrogate_start <= (0xDFFF - 0xDBFF) {
+ } else if unit_minus_surrogate_start <= (0xDBFF - 0xD800) {
// high surrogate
if self.pos == self.slice.len() {
// Unpaired surrogate at the end of the buffer.
NonAscii::BmpExclAscii(0xFFFDu16)
} else {
let second = self.slice[self.pos] as u32;
let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00);
if second_minus_low_surrogate_start <= (0xDFFF - 0xDC00) {
@@ -1118,17 +1118,17 @@ impl<'a> Utf8Source<'a> {
}
#[inline(always)]
pub fn copy_ascii_to_check_space_one<'b>
(&mut self,
dest: &'b mut ByteDestination<'a>)
-> CopyAsciiResult<(EncoderResult, usize, usize), (NonAscii, ByteOneHandle<'b, 'a>)> {
let non_ascii_ret = {
let src_remaining = &self.slice[self.pos..];
- let mut dst_remaining = &mut dest.slice[dest.pos..];
+ let dst_remaining = &mut dest.slice[dest.pos..];
let (pending, length) = if dst_remaining.len() < src_remaining.len() {
(EncoderResult::OutputFull, dst_remaining.len())
} else {
(EncoderResult::InputEmpty, src_remaining.len())
};
match unsafe {
ascii_to_ascii(src_remaining.as_ptr(), dst_remaining.as_mut_ptr(), length)
} {
@@ -1170,17 +1170,17 @@ impl<'a> Utf8Source<'a> {
#[inline(always)]
pub fn copy_ascii_to_check_space_two<'b>
(&mut self,
dest: &'b mut ByteDestination<'a>)
-> CopyAsciiResult<(EncoderResult, usize, usize), (NonAscii, ByteTwoHandle<'b, 'a>)> {
let non_ascii_ret = {
let dst_len = dest.slice.len();
let src_remaining = &self.slice[self.pos..];
- let mut dst_remaining = &mut dest.slice[dest.pos..];
+ let dst_remaining = &mut dest.slice[dest.pos..];
let (pending, length) = if dst_remaining.len() < src_remaining.len() {
(EncoderResult::OutputFull, dst_remaining.len())
} else {
(EncoderResult::InputEmpty, src_remaining.len())
};
match unsafe {
ascii_to_ascii(src_remaining.as_ptr(), dst_remaining.as_mut_ptr(), length)
} {
@@ -1226,17 +1226,17 @@ impl<'a> Utf8Source<'a> {
#[inline(always)]
pub fn copy_ascii_to_check_space_four<'b>
(&mut self,
dest: &'b mut ByteDestination<'a>)
-> CopyAsciiResult<(EncoderResult, usize, usize), (NonAscii, ByteFourHandle<'b, 'a>)> {
let non_ascii_ret = {
let dst_len = dest.slice.len();
let src_remaining = &self.slice[self.pos..];
- let mut dst_remaining = &mut dest.slice[dest.pos..];
+ let dst_remaining = &mut dest.slice[dest.pos..];
let (pending, length) = if dst_remaining.len() < src_remaining.len() {
(EncoderResult::OutputFull, dst_remaining.len())
} else {
(EncoderResult::InputEmpty, src_remaining.len())
};
match unsafe {
ascii_to_ascii(src_remaining.as_ptr(), dst_remaining.as_mut_ptr(), length)
} {
--- a/third_party/rust/encoding_rs/src/iso_2022_jp.rs
+++ b/third_party/rust/encoding_rs/src/iso_2022_jp.rs
@@ -958,9 +958,23 @@ mod tests {
assert_eq!(read, 1);
assert_eq!(written, 2);
assert!(had_errors);
assert_eq!(output[0], 0xFFFD);
assert_eq!(output[1], 0x0041);
}
}
+ #[test]
+ fn test_iso_2022_jp_encode_from_two_low_surrogates() {
+ let expectation = b"��";
+ let mut output = [0u8; 40];
+ let mut encoder = ISO_2022_JP.new_encoder();
+ let (result, read, written, had_errors) =
+ encoder.encode_from_utf16(&[0xDC00u16, 0xDEDEu16], &mut output[..], true);
+ assert_eq!(result, CoderResult::InputEmpty);
+ assert_eq!(read, 2);
+ assert_eq!(written, expectation.len());
+ assert!(had_errors);
+ assert_eq!(&output[..written], expectation);
+ }
+
}
--- a/third_party/rust/encoding_rs/src/lib.rs
+++ b/third_party/rust/encoding_rs/src/lib.rs
@@ -3,17 +3,17 @@
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#![cfg_attr(feature = "cargo-clippy", allow(doc_markdown, inline_always, new_ret_no_self))]
-#![doc(html_root_url = "https://docs.rs/encoding_rs/0.7.0")]
+#![doc(html_root_url = "https://docs.rs/encoding_rs/0.7.1")]
//! encoding_rs is a Gecko-oriented Free Software / Open Source implementation
//! of the [Encoding Standard](https://encoding.spec.whatwg.org/) in Rust.
//! Gecko-oriented means that converting to and from UTF-16 is supported in
//! addition to converting to and from UTF-8, that the performance and
//! streamability goals are browser-oriented, and that FFI-friendliness is a
//! goal.
//!
@@ -2450,17 +2450,17 @@ impl Encoding {
valid_up_to,
decoder.max_utf8_buffer_length(bytes.len() - valid_up_to),
);
let mut string = String::with_capacity(
checked_min(rounded_without_replacement, with_replacement)
.unwrap()
);
unsafe {
- let mut vec = string.as_mut_vec();
+ let vec = string.as_mut_vec();
vec.set_len(valid_up_to);
std::ptr::copy_nonoverlapping(bytes.as_ptr(), vec.as_mut_ptr(), valid_up_to);
}
(decoder, string, valid_up_to)
} else {
let decoder = self.new_decoder_without_bom_handling();
let rounded_without_replacement =
checked_next_power_of_two(
@@ -2551,17 +2551,17 @@ impl Encoding {
valid_up_to,
decoder.max_utf8_buffer_length_without_replacement(
bytes.len() - valid_up_to,
),
)
.unwrap()
);
unsafe {
- let mut vec = string.as_mut_vec();
+ let vec = string.as_mut_vec();
vec.set_len(valid_up_to);
std::ptr::copy_nonoverlapping(bytes.as_ptr(), vec.as_mut_ptr(), valid_up_to);
}
(decoder, string, &bytes[valid_up_to..])
} else {
let decoder = self.new_decoder_without_bom_handling();
let string = String::with_capacity(
decoder
--- a/third_party/rust/encoding_rs/src/simd_funcs.rs
+++ b/third_party/rust/encoding_rs/src/simd_funcs.rs
@@ -63,203 +63,196 @@ extern "platform-intrinsic" {
}
cfg_if! {
if #[cfg(target_feature = "sse2")] {
use simd::i16x8;
use simd::i8x16;
extern "platform-intrinsic" {
- fn x86_mm_packus_epi16(x: i16x8, y: i16x8) -> u8x16;
fn x86_mm_movemask_epi8(x: i8x16) -> i32;
}
- #[inline(always)]
- pub fn is_ascii(s: u8x16) -> bool {
- unsafe {
- let signed: i8x16 = ::std::mem::transmute_copy(&s);
- x86_mm_movemask_epi8(signed) == 0
- }
- }
-
// Expose low-level mask instead of higher-level conclusion,
// because the non-ASCII case would perform less well otherwise.
#[inline(always)]
pub fn mask_ascii(s: u8x16) -> i32 {
unsafe {
let signed: i8x16 = ::std::mem::transmute_copy(&s);
x86_mm_movemask_epi8(signed)
}
}
+ } else {
+
+ }
+}
+
+cfg_if! {
+ if #[cfg(target_feature = "sse2")] {
#[inline(always)]
- pub unsafe fn pack_basic_latin(a: u16x8, b: u16x8) -> Option<u8x16> {
- // If the 16-bit lane is out of range positive, the 8-bit lane becomes 0xFF
- // when packing, which would allow us to pack first and then check for
- // ASCII, but if the 16-bit lane is negative, the 8-bit lane becomes 0x00.
- // Sigh. Hence, check first.
- let highest_ascii = u16x8::splat(0x7F);
- let combined = a | b;
- if combined.gt(highest_ascii).any() {
- None
- } else {
- let first: i16x8 = ::std::mem::transmute_copy(&a);
- let second: i16x8 = ::std::mem::transmute_copy(&b);
- Some(x86_mm_packus_epi16(first, second))
+ pub fn is_ascii(s: u8x16) -> bool {
+ unsafe {
+ let signed: i8x16 = ::std::mem::transmute_copy(&s);
+ x86_mm_movemask_epi8(signed) == 0
}
}
} else if #[cfg(target_arch = "aarch64")]{
-
extern "platform-intrinsic" {
fn aarch64_vmaxvq_u8(x: u8x16) -> u8;
- fn aarch64_vmaxvq_u16(x: u16x8) -> u16;
}
#[inline(always)]
pub fn is_ascii(s: u8x16) -> bool {
unsafe {
aarch64_vmaxvq_u8(s) < 0x80
}
}
-
- #[inline(always)]
- pub unsafe fn pack_basic_latin(a: u16x8, b: u16x8) -> Option<u8x16> {
- let combined = a | b;
- if aarch64_vmaxvq_u16(combined) < 0x80 {
- let first: u8x16 = ::std::mem::transmute_copy(&a);
- let second: u8x16 = ::std::mem::transmute_copy(&b);
- let lower: u8x16 = simd_shuffle16(
- first,
- second,
- [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30],
- );
- Some(lower)
- } else {
- None
- }
- }
-
-
} else {
-
#[inline(always)]
pub fn is_ascii(s: u8x16) -> bool {
let highest_ascii = u8x16::splat(0x7F);
!s.gt(highest_ascii).any()
}
+ }
+}
+
+cfg_if! {
+ if #[cfg(target_arch = "aarch64")]{
+ extern "platform-intrinsic" {
+ fn aarch64_vmaxvq_u16(x: u16x8) -> u16;
+ }
#[inline(always)]
- pub unsafe fn pack_basic_latin(a: u16x8, b: u16x8) -> Option<u8x16> {
- let highest_ascii = u16x8::splat(0x7F);
- let combined = a | b;
- if combined.gt(highest_ascii).any() {
- None
- } else {
- let first: u8x16 = ::std::mem::transmute_copy(&a);
- let second: u8x16 = ::std::mem::transmute_copy(&b);
- let lower: u8x16 = simd_shuffle16(
- first,
- second,
- [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30],
- );
- Some(lower)
+ pub fn is_basic_latin(s: u16x8) -> bool {
+ unsafe {
+ aarch64_vmaxvq_u16(s) < 0x80
}
}
-
+ } else {
+ #[inline(always)]
+ pub fn is_basic_latin(s: u16x8) -> bool {
+ let highest_ascii = u16x8::splat(0x7F);
+ !s.gt(highest_ascii).any()
+ }
}
}
#[inline(always)]
-pub fn unpack(s: u8x16) -> (u16x8, u16x8) {
+pub fn simd_unpack(s: u8x16) -> (u16x8, u16x8) {
unsafe {
let first: u8x16 = simd_shuffle16(
s,
u8x16::splat(0),
[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23],
);
let second: u8x16 = simd_shuffle16(
s,
u8x16::splat(0),
[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31],
);
(::std::mem::transmute_copy(&first), ::std::mem::transmute_copy(&second))
}
}
+cfg_if! {
+ if #[cfg(target_feature = "sse2")] {
+ extern "platform-intrinsic" {
+ fn x86_mm_packus_epi16(x: i16x8, y: i16x8) -> u8x16;
+ }
+
+ #[inline(always)]
+ pub fn simd_pack(a: u16x8, b: u16x8) -> u8x16 {
+ unsafe {
+ let first: i16x8 = ::std::mem::transmute_copy(&a);
+ let second: i16x8 = ::std::mem::transmute_copy(&b);
+ x86_mm_packus_epi16(first, second)
+ }
+ }
+ } else {
+ #[inline(always)]
+ pub fn simd_pack(a: u16x8, b: u16x8) -> u8x16 {
+ unsafe {
+ let first: u8x16 = ::std::mem::transmute_copy(&a);
+ let second: u8x16 = ::std::mem::transmute_copy(&b);
+ simd_shuffle16(
+ first,
+ second,
+ [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30],
+ )
+ }
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_unpack() {
let ascii: [u8; 16] = [0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
0x72, 0x73, 0x74, 0x75, 0x76];
let basic_latin: [u16; 16] = [0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70,
0x71, 0x72, 0x73, 0x74, 0x75, 0x76];
let simd = unsafe { load16_unaligned(ascii.as_ptr()) };
let mut vec = Vec::with_capacity(16);
vec.resize(16, 0u16);
- let (first, second) = unpack(simd);
+ let (first, second) = simd_unpack(simd);
let ptr = vec.as_mut_ptr();
unsafe {
store8_unaligned(ptr, first);
store8_unaligned(ptr.offset(8), second);
}
assert_eq!(&vec[..], &basic_latin[..]);
}
#[test]
- fn test_pack_basic_latin_success() {
+ fn test_is_basic_latin_success() {
let ascii: [u8; 16] = [0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
0x72, 0x73, 0x74, 0x75, 0x76];
let basic_latin: [u16; 16] = [0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70,
0x71, 0x72, 0x73, 0x74, 0x75, 0x76];
let first = unsafe { load8_unaligned(basic_latin.as_ptr()) };
let second = unsafe { load8_unaligned(basic_latin.as_ptr().offset(8)) };
let mut vec = Vec::with_capacity(16);
vec.resize(16, 0u8);
let ptr = vec.as_mut_ptr();
+ assert!(is_basic_latin(first | second));
unsafe {
- let packed = pack_basic_latin(first, second).unwrap();
- store16_unaligned(ptr, packed);
+ store16_unaligned(ptr, simd_pack(first, second));
}
assert_eq!(&vec[..], &ascii[..]);
}
#[test]
- fn test_pack_basic_latin_c0() {
+ fn test_is_basic_latin_c0() {
let input: [u16; 16] = [0x61, 0x62, 0x63, 0x81, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
0x72, 0x73, 0x74, 0x75, 0x76];
let first = unsafe { load8_unaligned(input.as_ptr()) };
let second = unsafe { load8_unaligned(input.as_ptr().offset(8)) };
- unsafe {
- assert!(pack_basic_latin(first, second).is_none());
- }
+ assert!(!is_basic_latin(first | second));
}
#[test]
- fn test_pack_basic_latin_0fff() {
+ fn test_is_basic_latin_0fff() {
let input: [u16; 16] = [0x61, 0x62, 0x63, 0x0FFF, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70,
0x71, 0x72, 0x73, 0x74, 0x75, 0x76];
let first = unsafe { load8_unaligned(input.as_ptr()) };
let second = unsafe { load8_unaligned(input.as_ptr().offset(8)) };
- unsafe {
- assert!(pack_basic_latin(first, second).is_none());
- }
+ assert!(!is_basic_latin(first | second));
}
#[test]
- fn test_pack_basic_latin_ffff() {
+ fn test_is_basic_latin_ffff() {
let input: [u16; 16] = [0x61, 0x62, 0x63, 0xFFFF, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70,
0x71, 0x72, 0x73, 0x74, 0x75, 0x76];
let first = unsafe { load8_unaligned(input.as_ptr()) };
let second = unsafe { load8_unaligned(input.as_ptr().offset(8)) };
- unsafe {
- assert!(pack_basic_latin(first, second).is_none());
- }
+ assert!(!is_basic_latin(first | second));
}
#[test]
fn test_is_ascii_success() {
let ascii: [u8; 16] = [0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71,
0x72, 0x73, 0x74, 0x75, 0x76];
let simd = unsafe { load16_unaligned(ascii.as_ptr()) };
assert!(is_ascii(simd));
--- a/third_party/rust/encoding_rs/src/single_byte.rs
+++ b/third_party/rust/encoding_rs/src/single_byte.rs
@@ -551,16 +551,30 @@ mod tests {
break;
}
}
}
encode_from_utf16(encoding, data, &with_zeros[..]);
}
+ #[test]
+ fn test_single_byte_from_two_low_surrogates() {
+ let expectation = b"��";
+ let mut output = [0u8; 40];
+ let mut encoder = WINDOWS_1253.new_encoder();
+ let (result, read, written, had_errors) =
+ encoder.encode_from_utf16(&[0xDC00u16, 0xDEDEu16], &mut output[..], true);
+ assert_eq!(result, CoderResult::InputEmpty);
+ assert_eq!(read, 2);
+ assert_eq!(written, expectation.len());
+ assert!(had_errors);
+ assert_eq!(&output[..written], expectation);
+ }
+
// These tests are so self-referential that they are pretty useless.
// BEGIN GENERATED CODE. PLEASE DO NOT EDIT.
// Instead, please regenerate using generate-encoding-data.py
#[test]
fn test_single_byte_decode() {
decode_single_byte(IBM866, IBM866_DATA);
--- a/third_party/rust/encoding_rs/src/utf_8.rs
+++ b/third_party/rust/encoding_rs/src/utf_8.rs
@@ -640,17 +640,17 @@ impl Utf8Encoder {
dst[written] = (unit >> 12) as u8 | 0xE0u8;
written += 1;
dst[written] = ((unit & 0xFC0) >> 6) as u8 | 0x80u8;
written += 1;
dst[written] = (unit & 0x3F) as u8 | 0x80u8;
written += 1;
break;
}
- if unit_minus_surrogate_start <= (0xDFFF - 0xDBFF) {
+ if unit_minus_surrogate_start <= (0xDBFF - 0xD800) {
// high surrogate
if read == src.len() {
// Unpaired surrogate at the end of the buffer.
dst[written] = 0xEFu8;
written += 1;
dst[written] = 0xBFu8;
written += 1;
dst[written] = 0xBDu8;
@@ -938,16 +938,17 @@ mod tests {
encode_utf8_from_utf16(&[0xD800], "\u{FFFD}".as_bytes());
encode_utf8_from_utf16(&[0xD800, 0x0062], "\u{FFFD}\u{0062}".as_bytes());
encode_utf8_from_utf16(&[0xDFFF], "\u{FFFD}".as_bytes());
encode_utf8_from_utf16(&[0xDFFF, 0x0062], "\u{FFFD}\u{0062}".as_bytes());
encode_utf8_from_utf16(&[0xE000], "\u{E000}".as_bytes());
encode_utf8_from_utf16(&[0xFFFF], "\u{FFFF}".as_bytes());
encode_utf8_from_utf16(&[0xD800, 0xDC00], "\u{10000}".as_bytes());
encode_utf8_from_utf16(&[0xDBFF, 0xDFFF], "\u{10FFFF}".as_bytes());
+ encode_utf8_from_utf16(&[0xDC00, 0xDEDE], "\u{FFFD}\u{FFFD}".as_bytes());
}
#[test]
fn test_utf8_max_length_from_utf16() {
let mut encoder = UTF_8.new_encoder();
let mut output = [0u8; 13];
let input = &[0x2C9Fu16, 0x2CA9u16, 0x2CA3u16, 0x2C9Fu16];
let needed = encoder
--- a/third_party/rust/encoding_rs/src/x_user_defined.rs
+++ b/third_party/rust/encoding_rs/src/x_user_defined.rs
@@ -134,9 +134,22 @@ mod tests {
// ASCII
encode_x_user_defined("\u{0061}\u{0062}", b"\x61\x62");
encode_x_user_defined("\u{F780}\u{F7FF}", b"\x80\xFF");
encode_x_user_defined("\u{F77F}\u{F800}", b"");
}
+ #[test]
+ fn test_x_user_defined_from_two_low_surrogates() {
+ let expectation = b"��";
+ let mut output = [0u8; 40];
+ let mut encoder = X_USER_DEFINED.new_encoder();
+ let (result, read, written, had_errors) =
+ encoder.encode_from_utf16(&[0xDC00u16, 0xDEDEu16], &mut output[..], true);
+ assert_eq!(result, CoderResult::InputEmpty);
+ assert_eq!(read, 2);
+ assert_eq!(written, expectation.len());
+ assert!(had_errors);
+ assert_eq!(&output[..written], expectation);
+ }
}
--- a/toolkit/library/gtest/rust/Cargo.lock
+++ b/toolkit/library/gtest/rust/Cargo.lock
@@ -434,31 +434,31 @@ name = "either"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "encoding_c"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
- "encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_rs 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "encoding_glue"
version = "0.1.0"
dependencies = [
- "encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_rs 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"nserror 0.1.0",
"nsstring 0.1.0",
]
[[package]]
name = "encoding_rs"
-version = "0.7.0"
+version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"simd 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "env_logger"
@@ -1703,17 +1703,17 @@ dependencies = [
"checksum darling_core 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1486a8b00b45062c997f767738178b43219133dd0c8c826cb811e60563810821"
"checksum darling_macro 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8a86ec160aa0c3dd492dd4a14ec8104ad8f1a9400a820624db857998cc1f80f9"
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab"
"checksum dtoa-short 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "068d4026697c1a18f0b0bb8cfcad1b0c151b90d8edb9bf4c235ad68128920d1d"
"checksum dwrote 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "36e3b27cd0b8a68e00f07e8d8e1e4f4d8a6b8b873290a734f63bd56d792d23e1"
"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a"
"checksum encoding_c 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "93ec52324ca72f423237a413ca0e1c60654c8b3d0934fcd5fd888508dfcc4ba7"
-"checksum encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6f0a39f0e2f497d3c2e6a5529a0ec4fc640084fa401493c640421673471f8b72"
+"checksum encoding_rs 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f5215aabf22b83153be3ee44dfe3f940214541b2ce13d419c55e7a115c8c51a9"
"checksum env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b"
"checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8"
"checksum euclid 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "50c9e4c3b53de731815135191f0b77969bea953211b8bbd3cc3083a7b10e190e"
"checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344"
"checksum freetype 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "398b8a11884898184d55aca9806f002b3cf68f0e860e0cbb4586f834ee39b0e7"
"checksum fs2 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ab76cfd2aaa59b7bf6688ad9ba15bbae64bff97f04ea02144cfd3443e5c2866"
"checksum futures 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "55f0008e13fc853f79ea8fc86e931486860d4c4c156cdffb59fa5f7fa833660a"
"checksum fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
--- a/toolkit/library/rust/Cargo.lock
+++ b/toolkit/library/rust/Cargo.lock
@@ -433,31 +433,31 @@ name = "either"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "encoding_c"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
- "encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_rs 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "encoding_glue"
version = "0.1.0"
dependencies = [
- "encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_rs 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"nserror 0.1.0",
"nsstring 0.1.0",
]
[[package]]
name = "encoding_rs"
-version = "0.7.0"
+version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"simd 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "env_logger"
@@ -1715,17 +1715,17 @@ dependencies = [
"checksum darling_core 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1486a8b00b45062c997f767738178b43219133dd0c8c826cb811e60563810821"
"checksum darling_macro 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8a86ec160aa0c3dd492dd4a14ec8104ad8f1a9400a820624db857998cc1f80f9"
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab"
"checksum dtoa-short 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "068d4026697c1a18f0b0bb8cfcad1b0c151b90d8edb9bf4c235ad68128920d1d"
"checksum dwrote 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "36e3b27cd0b8a68e00f07e8d8e1e4f4d8a6b8b873290a734f63bd56d792d23e1"
"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a"
"checksum encoding_c 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "93ec52324ca72f423237a413ca0e1c60654c8b3d0934fcd5fd888508dfcc4ba7"
-"checksum encoding_rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6f0a39f0e2f497d3c2e6a5529a0ec4fc640084fa401493c640421673471f8b72"
+"checksum encoding_rs 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f5215aabf22b83153be3ee44dfe3f940214541b2ce13d419c55e7a115c8c51a9"
"checksum env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b"
"checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8"
"checksum euclid 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "50c9e4c3b53de731815135191f0b77969bea953211b8bbd3cc3083a7b10e190e"
"checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344"
"checksum freetype 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "398b8a11884898184d55aca9806f002b3cf68f0e860e0cbb4586f834ee39b0e7"
"checksum fs2 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ab76cfd2aaa59b7bf6688ad9ba15bbae64bff97f04ea02144cfd3443e5c2866"
"checksum futures 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "55f0008e13fc853f79ea8fc86e931486860d4c4c156cdffb59fa5f7fa833660a"
"checksum fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
