Bug 1294156 - NaNs with sign-bit aren't sorted correctly draft
authorSumit Tiwari <sumi29@gmail.com>
Wed, 31 Aug 2016 00:18:20 -0400
changeset 407851 57cd105cf829ab0891edd14a75c575c55a2a8a68
parent 406350 0ee53958c641842607f72dfa2f393d87f23f772f
child 411859 64e84d2c15f1f5bfefcd25fe0a0a2a1c3884e131
child 412335 72ef97c8c1fe7395ad777599cdb6cccb6ef79168
child 413300 8fde43f7b511a80f436d1297462317435f51911d
push id28063
push userbmo:sumi29@gmail.com
push dateWed, 31 Aug 2016 04:19:24 +0000
bugs1294156
milestone51.0a1
Bug 1294156 - NaNs with sign-bit aren't sorted correctly MozReview-Commit-ID: BUUdSLjErHB
js/src/builtin/Sorting.js
--- a/js/src/builtin/Sorting.js
+++ b/js/src/builtin/Sorting.js
@@ -120,25 +120,31 @@ function RadixSort(array, len, buffer, n
         assert(buffer !== null, "Attached data buffer should be reified when array length is >= 128.");
 
         view = new Int32Array(buffer);
 
         // Flip sign bit for positive numbers; flip all bits for negative
         // numbers
         for (let i = 0; i < len; i++) {
             if (view[i] & signMask) {
-                view[i] ^= 0xFFFFFFFF;
+                // If sign-bit is set, but the entry was originally a NaN, then use
+                // a regular NaN and flip the sign bit
+                if ((view[i] & 0x7F800000) === 0x7F800000 && (view[i] & 0x007FFFFF) !== 0) {
+                    view[i] = 0x7FC00000 ^ signMask;
+                } else {
+                    view[i] ^= 0xFFFFFFFF;
+                }
             } else {
-                view[i] ^= signMask
+                view[i] ^= signMask;
             }
         }
     } else if (signed) {
         // Flip sign bit
         for (let i = 0; i < len; i++) {
-            view[i] ^= signMask
+            view[i] ^= signMask;
         }
     }
 
     // Sort
     for (let col = 0; col < nbytes; col++) {
         SortByColumn(view, len, aux, col);
     }
 
@@ -148,17 +154,17 @@ function RadixSort(array, len, buffer, n
             if (view[i] & signMask) {
                 view[i] ^= signMask;
             } else {
                 view[i] ^= 0xFFFFFFFF;
             }
         }
     } else if (signed) {
         for (let i = 0; i < len; i++) {
-            view[i] ^= signMask
+            view[i] ^= signMask;
         }
     }
     return array;
 }
 
 
 // For sorting small arrays.
 function InsertionSort(array, from, to, comparefn) {