Bug 1289225: increase createTask concurrency; r=gps, r?jonasfj draft
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 25 Jul 2016 20:52:43 +0000
changeset 392575 fc33d4717dd0687c981e7f515b9eb9ce0c5424f0
parent 392573 836321463a2642ed518990efb5cbef4f9cf5498d
child 526359 008cb8eaa0b39a32b11f044d9515665b069f4625
push id24057
push userdmitchell@mozilla.com
push dateMon, 25 Jul 2016 21:09:18 +0000
reviewersgps, jonasfj
bugs1289225
milestone50.0a1
Bug 1289225: increase createTask concurrency; r=gps, r?jonasfj MozReview-Commit-ID: DuIzZUbTsaY
taskcluster/taskgraph/create.py
--- a/taskcluster/taskgraph/create.py
+++ b/taskcluster/taskgraph/create.py
@@ -14,27 +14,30 @@ import logging
 from slugid import nice as slugid
 from taskgraph.util.time import (
     current_json_time,
     json_time_from_now
 )
 
 logger = logging.getLogger(__name__)
 
+# the maximum number of parallel createTask calls to make
+CONCURRENCY = 50
+
 
 def create_tasks(taskgraph, label_to_taskid):
     # TODO: use the taskGroupId of the decision task
     task_group_id = slugid()
     taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()}
 
     session = requests.Session()
 
     decision_task_id = os.environ.get('TASK_ID')
 
-    with futures.ThreadPoolExecutor(requests.adapters.DEFAULT_POOLSIZE) as e:
+    with futures.ThreadPoolExecutor(CONCURRENCY) as e:
         fs = {}
 
         # We can't submit a task until its dependencies have been submitted.
         # So our strategy is to walk the graph and submit tasks once all
         # their dependencies have been submitted.
         #
         # Using visit_postorder() here isn't the most efficient: we'll
         # block waiting for dependencies of task N to submit even though