Bug 1346778 - Stop using multiprocessing to parallelise log downloads, r=emorley draft
authorJames Graham <james@hoppipolla.co.uk>
Thu, 09 Mar 2017 18:39:40 +0000
changeset 497560 1065219eebfa6b8e9acad0129b675ad60b22019e
parent 497559 96210e4459aa3e1fc56cdd0562a01b6cff6bf03e
child 497561 cb1da81a5c3e03963aa9555cef3f2fab989ba4c1
push id48938
push userbmo:james@hoppipolla.co.uk
push dateMon, 13 Mar 2017 14:19:39 +0000
reviewersemorley
bugs1346778
milestone55.0a1
Bug 1346778 - Stop using multiprocessing to parallelise log downloads, r=emorley This was causing data corruption MozReview-Commit-ID: 2aEyPQ7pug8
testing/web-platform/update/fetchlogs.py
--- a/testing/web-platform/update/fetchlogs.py
+++ b/testing/web-platform/update/fetchlogs.py
@@ -101,22 +101,18 @@ def get_structured_logs(branch, commit, 
         job_type_name = result["job_type_name"]
         if (job_type_name.startswith("W3C Web Platform") or
             job_type_name.startswith("test-") and "-web-platform-tests-" in job_type_name):
             url = get_blobber_url(branch, result)
             if url:
                 prefix = result["platform"] # platform
                 tasks.append((url, prefix, None))
 
-    pool = Pool(8)
-    for item in pool.imap_unordered(download_logs, tasks, 1):
-        pass
-
-def download_logs(args):
-    download(*args)
+    for task in tasks:
+        download(*task)
 
 def main():
     parser = create_parser()
     args = parser.parse_args()
 
     get_structured_logs(args.branch, args.commit)
 
 if __name__ == "__main__":