hgmo: stream JSON responses (bug 1303904); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Thu, 08 Jun 2017 13:27:20 -0700
changeset 11233 d9d1f05f3062383ba367e28a91ae5f692f223068
parent 11232 e29d0b981e9134ec6a02d60719e22d353606ec2a
child 11234 442f057545cd56c529841eccc103f55cae50acf1
push id1708
push usergszorc@mozilla.com
push dateMon, 19 Jun 2017 18:36:36 +0000
reviewersglob
bugs1303904
hgmo: stream JSON responses (bug 1303904); r?glob Previously, we called json.dumps() for some web commands. This buffers the JSON response in memory then sends it. That buffering can take a long time if the returned document is large. Because of differences in how the JSON encoder is implemented, this improves throughput of sending JSON from ~315KB/s to >7MB/s on my i7-6700K. Test output for the mozbuildinfo command has changed because we no longer send trailing whitespace (which is what the default JSONEncoder does in Python 2.7). Because of clownshoes that will be fixed in a subsequent commit, this drops the time to first byte on json-automationrelevance/5dddbefdf759f09b1411f33fa0920835b919fc81 against the mozilla-aurora repo from ~119s to ~80s. MozReview-Commit-ID: 2V4Y3iWfZRe
hgext/hgmo/__init__.py
hgext/hgmo/tests/test-mozbuildinfo-webcommand.t
--- a/hgext/hgmo/__init__.py
+++ b/hgext/hgmo/__init__.py
@@ -122,16 +122,26 @@ command = cmdutil.command(cmdtable)
 
 
 @templatefilters.templatefilter('mozlink')
 def mozlink(text):
     """Any text. Hyperlink to Bugzilla and other detected things."""
     return commitparser.add_hyperlinks(text)
 
 
+def stream_json(data):
+    """Convert a data structure to a generator of chunks representing JSON."""
+    # We use latin1 as the encoding because all data should be treated as
+    # byte strings. ensure_ascii will escape non-ascii values using \uxxxx.
+    # Also, use stable output and indentation to make testing easier.
+    encoder = json.JSONEncoder(indent=2, sort_keys=True, encoding='latin1',
+                               separators=(',', ': '))
+    return encoder.iterencode(data)
+
+
 def addmetadata(repo, ctx, d, onlycheap=False):
     """Add changeset metadata for hgweb templates."""
     description = encoding.fromlocal(ctx.description())
 
     d['bugs'] = []
     for bug in commitparser.parse_bugs(description):
         d['bugs'].append({
             'no': str(bug),
@@ -304,17 +314,17 @@ def mozbuildinfowebcommand(web, req, tmp
         return json.dumps({'error': 'unable to obtain moz.build info'},
                           indent=2)
     elif stderr.strip():
         repo.ui.log('moz.build evaluation output: %s\n' % stderr.strip())
 
     # Round trip to ensure we have valid JSON.
     try:
         d = json.loads(stdout)
-        return json.dumps(d, indent=2, sort_keys=True)
+        return stream_json(d)
     except Exception:
         return json.dumps({'error': 'invalid JSON returned; report this error'},
                           indent=2)
 
 
 def infowebcommand(web, req, tmpl):
     """Get information about the specified changeset(s).
 
@@ -461,20 +471,17 @@ def automationrelevancewebcommand(web, r
         visible = None
 
     data = {
         'changesets': csets,
         'visible': visible,
     }
 
     req.respond(HTTP_OK, 'application/json')
-    # We use latin1 as the encoding here because all data should be treated as
-    # byte strings. ensure_ascii will escape non-ascii values using \uxxxx.
-    return json.dumps(data, indent=2, sort_keys=True, encoding='latin1',
-                      separators=(',', ': '))
+    return stream_json(data)
 
 
 def revset_reviewer(repo, subset, x):
     """``reviewer(REVIEWER)``
 
     Changesets reviewed by a specific person.
     """
     l = revset.getargs(x, 1, 1, 'reviewer requires one argument')
--- a/hgext/hgmo/tests/test-mozbuildinfo-webcommand.t
+++ b/hgext/hgmo/tests/test-mozbuildinfo-webcommand.t
@@ -76,31 +76,31 @@ mozbuildinfo is available via web comman
   200
   content-type: application/json
   
   {
     "aggregate": {
       "bug_component_counts": [
         [
           [
-            "Product1", 
+            "Product1",
             "Component 1"
-          ], 
+          ],
           1
         ]
-      ], 
+      ],
       "recommended_bug_component": [
-        "Product1", 
+        "Product1",
         "Component 1"
       ]
-    }, 
+    },
     "files": {
       "moz.build": {
         "bug_component": [
-          "Product1", 
+          "Product1",
           "Component 1"
         ]
       }
     }
   }
 
 we can request info for specific files
 
@@ -108,37 +108,37 @@ we can request info for specific files
   200
   content-type: application/json
   
   {
     "aggregate": {
       "bug_component_counts": [
         [
           [
-            "Product1", 
+            "Product1",
             "Component 1"
-          ], 
+          ],
           2
         ]
-      ], 
+      ],
       "recommended_bug_component": [
-        "Product1", 
+        "Product1",
         "Component 1"
       ]
-    }, 
+    },
     "files": {
       "file1": {
         "bug_component": [
-          "Product1", 
+          "Product1",
           "Component 1"
         ]
-      }, 
+      },
       "file2": {
         "bug_component": [
-          "Product1", 
+          "Product1",
           "Component 1"
         ]
       }
     }
   }
 
 Errors displayed properly