Bug 1346778 - Stop using multiprocessing to parallelise log downloads, r=emorley
This was causing data corruption
MozReview-Commit-ID: 2aEyPQ7pug8
--- a/testing/web-platform/update/fetchlogs.py
+++ b/testing/web-platform/update/fetchlogs.py
@@ -101,22 +101,18 @@ def get_structured_logs(branch, commit,
job_type_name = result["job_type_name"]
if (job_type_name.startswith("W3C Web Platform") or
job_type_name.startswith("test-") and "-web-platform-tests-" in job_type_name):
url = get_blobber_url(branch, result)
if url:
prefix = result["platform"] # platform
tasks.append((url, prefix, None))
- pool = Pool(8)
- for item in pool.imap_unordered(download_logs, tasks, 1):
- pass
-
-def download_logs(args):
- download(*args)
+ for task in tasks:
+ download(*task)
def main():
parser = create_parser()
args = parser.parse_args()
get_structured_logs(args.branch, args.commit)
if __name__ == "__main__":