Bug 1289225: increase createTask concurrency; r=gps, r?jonasfj
MozReview-Commit-ID: DuIzZUbTsaY
--- a/taskcluster/taskgraph/create.py
+++ b/taskcluster/taskgraph/create.py
@@ -14,27 +14,30 @@ import logging
from slugid import nice as slugid
from taskgraph.util.time import (
current_json_time,
json_time_from_now
)
logger = logging.getLogger(__name__)
+# the maximum number of parallel createTask calls to make
+CONCURRENCY = 50
+
def create_tasks(taskgraph, label_to_taskid):
# TODO: use the taskGroupId of the decision task
task_group_id = slugid()
taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()}
session = requests.Session()
decision_task_id = os.environ.get('TASK_ID')
- with futures.ThreadPoolExecutor(requests.adapters.DEFAULT_POOLSIZE) as e:
+ with futures.ThreadPoolExecutor(CONCURRENCY) as e:
fs = {}
# We can't submit a task until its dependencies have been submitted.
# So our strategy is to walk the graph and submit tasks once all
# their dependencies have been submitted.
#
# Using visit_postorder() here isn't the most efficient: we'll
# block waiting for dependencies of task N to submit even though