Bug 1290173 - Introduce word tracking in Narrate. r?mikedeboer draft
authorEitan Isaacson <eitan@monotonous.org>
Mon, 25 Jul 2016 15:13:27 -0700
changeset 419601 75725bdf6e56477c224090928e5cfcd811088b67
parent 416191 1ce69a3f66df767b514c67a0bf4667d39d508bba
child 419602 9db94003aa478040c66ef694869ae14b5a1f0682
push id30967
push userbmo:eitan@monotonous.org
push dateFri, 30 Sep 2016 16:57:36 +0000
reviewersmikedeboer
bugs1290173
milestone51.0a1
Bug 1290173 - Introduce word tracking in Narrate. r?mikedeboer MozReview-Commit-ID: BvcCaGmurh3
toolkit/components/narrate/Narrator.jsm
toolkit/themes/shared/aboutReaderControls.css
toolkit/themes/shared/narrate.css
--- a/toolkit/components/narrate/Narrator.jsm
+++ b/toolkit/components/narrate/Narrator.jsm
@@ -13,16 +13,21 @@ XPCOMUtils.defineLazyModuleGetter(this, 
 XPCOMUtils.defineLazyModuleGetter(this, "Services",
   "resource://gre/modules/Services.jsm");
 
 this.EXPORTED_SYMBOLS = [ "Narrator" ];
 
 // Maximum time into paragraph when pressing "skip previous" will go
 // to previous paragraph and not the start of current one.
 const PREV_THRESHOLD = 2000;
+// All text-related style rules that we should copy over to the highlight node.
+const kTextStylesRules = ["font-family", "font-kerning", "font-size",
+  "font-size-adjust", "font-stretch", "font-variant", "font-weight",
+  "line-height", "letter-spacing", "text-orientation",
+  "text-transform", "word-spacing"];
 
 function Narrator(win) {
   this._winRef = Cu.getWeakReference(win);
   this._inTest = Services.prefs.getBoolPref("narrate.test");
   this._speechOptions = {};
   this._startTime = 0;
   this._stopped = false;
 }
@@ -155,16 +160,18 @@ Narrator.prototype = {
     if (this._speechOptions.voice) {
       utterance.voice = this._speechOptions.voice;
     } else {
       utterance.lang = this._speechOptions.lang;
     }
 
     this._startTime = Date.now();
 
+    let highlighter = new Highlighter(paragraph);
+
     return new Promise((resolve, reject) => {
       utterance.addEventListener("start", () => {
         paragraph.classList.add("narrating");
         let bb = paragraph.getBoundingClientRect();
         if (bb.top < 0 || bb.bottom > this._win.innerHeight) {
           paragraph.scrollIntoView({ behavior: "smooth", block: "start"});
         }
 
@@ -178,16 +185,17 @@ Narrator.prototype = {
       });
 
       utterance.addEventListener("end", () => {
         if (!this._win) {
           // page got unloaded, don't do anything.
           return;
         }
 
+        highlighter.remove();
         paragraph.classList.remove("narrating");
         this._startTime = 0;
         if (this._inTest) {
           this._sendTestEvent("paragraphend", {});
         }
 
         if (this._stopped) {
           // User pressed stopped.
@@ -197,16 +205,33 @@ Narrator.prototype = {
           this._speakInner().then(resolve, reject);
         }
       });
 
       utterance.addEventListener("error", () => {
         reject("speech synthesis failed");
       });
 
+      utterance.addEventListener("boundary", e => {
+        if (e.name != "word") {
+          // We are only interested in word boundaries for now.
+          return;
+        }
+
+        // Match non-whitespace. This isn't perfect, but the most universal
+        // solution for now.
+        let reWordBoundary = /\S+/g;
+        // Match the first word from the boundary event offset.
+        reWordBoundary.lastIndex = e.charIndex;
+        let firstIndex = reWordBoundary.exec(paragraph.textContent);
+        if (firstIndex) {
+          highlighter.highlight(firstIndex.index, reWordBoundary.lastIndex);
+        }
+      });
+
       this._win.speechSynthesis.speak(utterance);
     });
   },
 
   start: function(speechOptions) {
     this._speechOptions = {
       rate: speechOptions.rate,
       voice: this._getVoice(speechOptions.voice)
@@ -261,8 +286,147 @@ Narrator.prototype = {
     for (let i = 0; i < count; i++) {
       if (!tw.previousNode()) {
         tw.currentNode = tw.root;
       }
     }
     this._win.speechSynthesis.cancel();
   }
 };
+
+/**
+ * The Highlighter class is used to highlight a range of text in a container.
+ *
+ * @param {nsIDOMElement} container a text container
+ */
+function Highlighter(container) {
+  this.container = container;
+}
+
+Highlighter.prototype = {
+  /**
+   * Highlight the range within offsets relative to the container.
+   *
+   * @param {Number} startOffset the start offset
+   * @param {Number} endOffset the end offset
+   */
+  highlight: function(startOffset, endOffset) {
+    let containerRect = this.container.getBoundingClientRect();
+    let range = this._getRange(startOffset, endOffset);
+    let rangeRects = range.getClientRects();
+    let win = this.container.ownerDocument.defaultView;
+    let computedStyle = win.getComputedStyle(range.endContainer.parentNode);
+    let nodes = this._getFreshHighlightNodes(rangeRects.length);
+
+    let textStyle = {};
+    for (let textStyleRule of kTextStylesRules) {
+      textStyle[textStyleRule] = computedStyle[textStyleRule];
+    }
+
+    for (let i = 0; i < rangeRects.length; i++) {
+      let r = rangeRects[i];
+      let node = nodes[i];
+
+      let style = Object.assign({
+        "top": `${r.top - containerRect.top + r.height / 2}px`,
+        "left": `${r.left - containerRect.left + r.width / 2}px`,
+        "width": `${r.width}px`,
+        "height": `${r.height}px`
+      }, textStyle);
+
+      // Enables us to vary the CSS transition on a line change.
+      node.classList.toggle("newline", style.top != node.dataset.top);
+      node.dataset.top = style.top;
+
+      // Enables CSS animations.
+      node.classList.remove("animate");
+      win.requestAnimationFrame(() => {
+        node.classList.add("animate");
+      });
+
+      // Enables alternative word display with a CSS pseudo-element.
+      node.dataset.word = range.toString();
+
+      // Apply style
+      node.style = Object.entries(style).map(
+        s => `${s[0]}: ${s[1]};`).join(" ");
+    }
+  },
+
+  /**
+   * Releases reference to container and removes all highlight nodes.
+   */
+  remove: function() {
+    for (let node of this._nodes) {
+      node.remove();
+    }
+
+    this.container = null;
+  },
+
+  /**
+   * Returns specified amount of highlight nodes. Creates new ones if necessary
+   * and purges any additional nodes that are not needed.
+   *
+   * @param {Number} count number of nodes needed
+   */
+  _getFreshHighlightNodes: function(count) {
+    let doc = this.container.ownerDocument;
+    let nodes = Array.from(this._nodes);
+
+    // Remove nodes we don't need anymore (nodes.length - count > 0).
+    for (let toRemove = 0; toRemove < nodes.length - count; toRemove++) {
+      nodes.shift().remove();
+    }
+
+    // Add additional nodes if we need them (count - nodes.length > 0).
+    for (let toAdd = 0; toAdd < count - nodes.length; toAdd++) {
+      let node = doc.createElement("div");
+      node.className = "narrate-word-highlight";
+      this.container.appendChild(node);
+      nodes.push(node);
+    }
+
+    return nodes;
+  },
+
+  /**
+   * Create and return a range object with the start and end offsets relative
+   * to the container node.
+   *
+   * @param {Number} startOffset the start offset
+   * @param {Number} endOffset the end offset
+   */
+  _getRange: function(startOffset, endOffset) {
+    let doc = this.container.ownerDocument;
+    let i = 0;
+    let treeWalker = doc.createTreeWalker(
+      this.container, doc.defaultView.NodeFilter.SHOW_TEXT);
+    let node = treeWalker.nextNode();
+
+    function _findNodeAndOffset(offset) {
+      do {
+        let length = node.data.length;
+        if (offset >= i && offset <= i + length) {
+          return [node, offset - i];
+        }
+        i += length;
+      } while ((node = treeWalker.nextNode()));
+
+      // Offset is out of bounds, return last offset of last node.
+      node = treeWalker.lastChild();
+      return [node, node.data.length];
+    }
+
+    let range = doc.createRange();
+    range.setStart(..._findNodeAndOffset(startOffset));
+    range.setEnd(..._findNodeAndOffset(endOffset));
+
+    return range;
+  },
+
+  /*
+   * Get all existing highlight nodes for container.
+   */
+  get _nodes() {
+    return this.container.querySelectorAll(".narrate-word-highlight")
+  }
+};
--- a/toolkit/themes/shared/aboutReaderControls.css
+++ b/toolkit/themes/shared/aboutReaderControls.css
@@ -77,16 +77,17 @@
   top: 0;
   left: 0;
   margin: 0;
   padding: 0;
   list-style: none;
   background-color: #fbfbfb;
   -moz-user-select: none;
   border-right: 1px solid #b5b5b5;
+  z-index: 1;
 }
 
 .button {
   display: block;
   background-size: 24px 24px;
   background-repeat: no-repeat;
   color: #333;
   background-color: #fbfbfb;
--- a/toolkit/themes/shared/narrate.css
+++ b/toolkit/themes/shared/narrate.css
@@ -1,11 +1,46 @@
+.narrating {
+  position: relative;
+  z-index: 1;
+}
+
 body.light .narrating {
   background-color: #ffc;
 }
 
 body.sepia .narrating {
   background-color: #e0d7c5;
 }
 
 body.dark .narrating {
   background-color: #242424;
 }
+
+.narrate-word-highlight {
+  position: absolute;
+  display: none;
+  transform: translate(-50%, calc(-50% - 2px));
+  z-index: -1;
+  border-bottom-style: solid;
+  border-bottom-width: 7px;
+  transition: left 0.1s ease;
+}
+
+.narrating > .narrate-word-highlight {
+  display: inline-block;
+}
+
+.narrate-word-highlight.newline {
+  transition: none;
+}
+
+body.light .narrate-word-highlight {
+  border-bottom-color: #ffe087;
+}
+
+body.sepia .narrate-word-highlight {
+  border-bottom-color: #bdb5a5;
+}
+
+body.dark .narrate-word-highlight {
+  border-bottom-color: #6f6f6f;
+}