hgmo: don't send list of push nodes with automationrelevant response (bug 1303904); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Thu, 08 Jun 2017 13:33:33 -0700
changeset 11238 10b4978a9f5f3e58beafc66b159620962abf89cc
parent 11237 bb140762d07e7164199f302892a6ca058f4da014
push id1708
push usergszorc@mozilla.com
push dateMon, 19 Jun 2017 18:36:36 +0000
reviewersglob
bugs1303904
hgmo: don't send list of push nodes with automationrelevant response (bug 1303904); r?glob This data is redundant with other data in the JSON. Furthermore, for large pushes (such as uplifts, which contains thousands of changesets), the redundancy is massive. I'm pretty sure nobody looks at this data. The only consumer of this API I'm aware of (TaskCluster decision task) only cares about the "node," "desc," and "files" elements. For json-automationrelevance/5dddbefdf759f09b1411f33fa0920835b919fc81 against the mozilla-aurora repo, this change reduces the raw HTTP response size from 2348 MB to 10.8 MB and the server-side CPU required from ~311s to ~6s (as measured on my i7-6700K). Yes, you read that correctly. MozReview-Commit-ID: 4Q1VfAllOis
hgext/hgmo/__init__.py
hgext/hgmo/tests/test-automationrelevant-obsolete.t
hgext/hgmo/tests/test-automationrelevant.t
--- a/hgext/hgmo/__init__.py
+++ b/hgext/hgmo/__init__.py
@@ -450,16 +450,25 @@ def automationrelevancewebcommand(web, r
     # So we pre load and cache data for pushlog entries we care about.
     cl = urepo.changelog
     nodes = [cl.node(rev) for rev in revs]
 
     with repo.unfiltered().pushlog.cache_data_for_nodes(nodes):
         for rev in revs:
             ctx = urepo[rev]
             entry = webutil.changelistentry(web, ctx, tmpl)
+
+            # The pushnodes list is redundant with data from other changesets.
+            # The amount of redundant data for pushes containing N>100
+            # changesets can add up to megabytes in size.
+            try:
+                del entry['pushnodes']
+            except KeyError:
+                pass
+
             # Some items in changelistentry are generators, which json.dumps()
             # can't handle. So we expand them.
             for k, v in entry.items():
                 # "files" is a generator that attempts to call a template.
                 # Don't even bother and just repopulate it.
                 if k == 'files':
                     entry['files'] = sorted(ctx.files())
                 elif k == 'allparents':
--- a/hgext/hgmo/tests/test-automationrelevant-obsolete.t
+++ b/hgext/hgmo/tests/test-automationrelevant-obsolete.t
@@ -113,20 +113,16 @@ Unless --hidden is used
           "d406a5ad38f255efb8657338e56a2bd6b8149cca"
         ],
         "pushdate": [
           \d+, (re)
           0
         ],
         "pushhead": "3208166ea10954e86c390c32fe6f7166f06161b2",
         "pushid": 3,
-        "pushnodes": [
-          "11743f8081842eb047711e85120177ed46be343e",
-          "3208166ea10954e86c390c32fe6f7166f06161b2"
-        ],
         "pushuser": "testuser",
         "rev": 3,
         "reviewers": []
       },
       {
         "author": "test",
         "backsoutnodes": [],
         "bugs": [],
@@ -146,20 +142,16 @@ Unless --hidden is used
           "11743f8081842eb047711e85120177ed46be343e"
         ],
         "pushdate": [
           \d+, (re)
           0
         ],
         "pushhead": "3208166ea10954e86c390c32fe6f7166f06161b2",
         "pushid": 3,
-        "pushnodes": [
-          "11743f8081842eb047711e85120177ed46be343e",
-          "3208166ea10954e86c390c32fe6f7166f06161b2"
-        ],
         "pushuser": "testuser",
         "rev": 4,
         "reviewers": []
       }
     ],
     "visible": false
   }
 
--- a/hgext/hgmo/tests/test-automationrelevant.t
+++ b/hgext/hgmo/tests/test-automationrelevant.t
@@ -150,21 +150,16 @@ web command for exposing automation rele
           "cb5c79007e91b09a4ba7ebe9210311491d09e96e"
         ],
         "pushdate": [
           \d+, (re)
           0
         ],
         "pushhead": "5d04c4fd236c19e241d1587e120b39840344eee8",
         "pushid": 3,
-        "pushnodes": [
-          "13855aae8fb3291c663ff46a8510c0e3fa673a4c",
-          "66a66c6c6ae312ec88240754300468a6cea8f71d",
-          "5d04c4fd236c19e241d1587e120b39840344eee8"
-        ],
         "pushuser": "testuser",
         "rev": 3,
         "reviewers": []
       },
       {
         "author": "test",
         "backsoutnodes": [],
         "bugs": [],
@@ -184,21 +179,16 @@ web command for exposing automation rele
           "13855aae8fb3291c663ff46a8510c0e3fa673a4c"
         ],
         "pushdate": [
           \d+, (re)
           0
         ],
         "pushhead": "5d04c4fd236c19e241d1587e120b39840344eee8",
         "pushid": 3,
-        "pushnodes": [
-          "13855aae8fb3291c663ff46a8510c0e3fa673a4c",
-          "66a66c6c6ae312ec88240754300468a6cea8f71d",
-          "5d04c4fd236c19e241d1587e120b39840344eee8"
-        ],
         "pushuser": "testuser",
         "rev": 4,
         "reviewers": []
       },
       {
         "author": "test",
         "backsoutnodes": [],
         "bugs": [],
@@ -218,21 +208,16 @@ web command for exposing automation rele
           "66a66c6c6ae312ec88240754300468a6cea8f71d"
         ],
         "pushdate": [
           \d+, (re)
           0
         ],
         "pushhead": "5d04c4fd236c19e241d1587e120b39840344eee8",
         "pushid": 3,
-        "pushnodes": [
-          "13855aae8fb3291c663ff46a8510c0e3fa673a4c",
-          "66a66c6c6ae312ec88240754300468a6cea8f71d",
-          "5d04c4fd236c19e241d1587e120b39840344eee8"
-        ],
         "pushuser": "testuser",
         "rev": 5,
         "reviewers": []
       }
     ],
     "visible": true
   }