hgmo: stream JSON responses (
bug 1303904); r?glob
Previously, we called json.dumps() for some web commands. This buffers
the JSON response in memory then sends it. That buffering can take a
long time if the returned document is large.
Because of differences in how the JSON encoder is implemented, this
improves throughput of sending JSON from ~315KB/s to >7MB/s on my
i7-6700K.
Test output for the mozbuildinfo command has changed because we no
longer send trailing whitespace (which is what the default JSONEncoder
does in Python 2.7).
Because of clownshoes that will be fixed in a subsequent commit, this
drops the time to first byte on
json-automationrelevance/5dddbefdf759f09b1411f33fa0920835b919fc81
against the mozilla-aurora repo from ~119s to ~80s.
MozReview-Commit-ID: 2V4Y3iWfZRe
--- a/hgext/hgmo/__init__.py
+++ b/hgext/hgmo/__init__.py
@@ -122,16 +122,26 @@ command = cmdutil.command(cmdtable)
@templatefilters.templatefilter('mozlink')
def mozlink(text):
"""Any text. Hyperlink to Bugzilla and other detected things."""
return commitparser.add_hyperlinks(text)
+def stream_json(data):
+ """Convert a data structure to a generator of chunks representing JSON."""
+ # We use latin1 as the encoding because all data should be treated as
+ # byte strings. ensure_ascii will escape non-ascii values using \uxxxx.
+ # Also, use stable output and indentation to make testing easier.
+ encoder = json.JSONEncoder(indent=2, sort_keys=True, encoding='latin1',
+ separators=(',', ': '))
+ return encoder.iterencode(data)
+
+
def addmetadata(repo, ctx, d, onlycheap=False):
"""Add changeset metadata for hgweb templates."""
description = encoding.fromlocal(ctx.description())
d['bugs'] = []
for bug in commitparser.parse_bugs(description):
d['bugs'].append({
'no': str(bug),
@@ -304,17 +314,17 @@ def mozbuildinfowebcommand(web, req, tmp
return json.dumps({'error': 'unable to obtain moz.build info'},
indent=2)
elif stderr.strip():
repo.ui.log('moz.build evaluation output: %s\n' % stderr.strip())
# Round trip to ensure we have valid JSON.
try:
d = json.loads(stdout)
- return json.dumps(d, indent=2, sort_keys=True)
+ return stream_json(d)
except Exception:
return json.dumps({'error': 'invalid JSON returned; report this error'},
indent=2)
def infowebcommand(web, req, tmpl):
"""Get information about the specified changeset(s).
@@ -461,20 +471,17 @@ def automationrelevancewebcommand(web, r
visible = None
data = {
'changesets': csets,
'visible': visible,
}
req.respond(HTTP_OK, 'application/json')
- # We use latin1 as the encoding here because all data should be treated as
- # byte strings. ensure_ascii will escape non-ascii values using \uxxxx.
- return json.dumps(data, indent=2, sort_keys=True, encoding='latin1',
- separators=(',', ': '))
+ return stream_json(data)
def revset_reviewer(repo, subset, x):
"""``reviewer(REVIEWER)``
Changesets reviewed by a specific person.
"""
l = revset.getargs(x, 1, 1, 'reviewer requires one argument')
--- a/hgext/hgmo/tests/test-mozbuildinfo-webcommand.t
+++ b/hgext/hgmo/tests/test-mozbuildinfo-webcommand.t
@@ -76,31 +76,31 @@ mozbuildinfo is available via web comman
200
content-type: application/json
{
"aggregate": {
"bug_component_counts": [
[
[
- "Product1",
+ "Product1",
"Component 1"
- ],
+ ],
1
]
- ],
+ ],
"recommended_bug_component": [
- "Product1",
+ "Product1",
"Component 1"
]
- },
+ },
"files": {
"moz.build": {
"bug_component": [
- "Product1",
+ "Product1",
"Component 1"
]
}
}
}
we can request info for specific files
@@ -108,37 +108,37 @@ we can request info for specific files
200
content-type: application/json
{
"aggregate": {
"bug_component_counts": [
[
[
- "Product1",
+ "Product1",
"Component 1"
- ],
+ ],
2
]
- ],
+ ],
"recommended_bug_component": [
- "Product1",
+ "Product1",
"Component 1"
]
- },
+ },
"files": {
"file1": {
"bug_component": [
- "Product1",
+ "Product1",
"Component 1"
]
- },
+ },
"file2": {
"bug_component": [
- "Product1",
+ "Product1",
"Component 1"
]
}
}
}
Errors displayed properly