Bug 1294156 - NaNs with sign-bit aren't sorted correctly
MozReview-Commit-ID: BUUdSLjErHB
--- a/js/src/builtin/Sorting.js
+++ b/js/src/builtin/Sorting.js
@@ -120,25 +120,31 @@ function RadixSort(array, len, buffer, n
assert(buffer !== null, "Attached data buffer should be reified when array length is >= 128.");
view = new Int32Array(buffer);
// Flip sign bit for positive numbers; flip all bits for negative
// numbers
for (let i = 0; i < len; i++) {
if (view[i] & signMask) {
- view[i] ^= 0xFFFFFFFF;
+ // If sign-bit is set, but the entry was originally a NaN, then use
+ // a regular NaN and flip the sign bit
+ if ((view[i] & 0x7F800000) === 0x7F800000 && (view[i] & 0x007FFFFF) !== 0) {
+ view[i] = 0x7FC00000 ^ signMask;
+ } else {
+ view[i] ^= 0xFFFFFFFF;
+ }
} else {
- view[i] ^= signMask
+ view[i] ^= signMask;
}
}
} else if (signed) {
// Flip sign bit
for (let i = 0; i < len; i++) {
- view[i] ^= signMask
+ view[i] ^= signMask;
}
}
// Sort
for (let col = 0; col < nbytes; col++) {
SortByColumn(view, len, aux, col);
}
@@ -148,17 +154,17 @@ function RadixSort(array, len, buffer, n
if (view[i] & signMask) {
view[i] ^= signMask;
} else {
view[i] ^= 0xFFFFFFFF;
}
}
} else if (signed) {
for (let i = 0; i < len; i++) {
- view[i] ^= signMask
+ view[i] ^= signMask;
}
}
return array;
}
// For sorting small arrays.
function InsertionSort(array, from, to, comparefn) {