--- a/testing/talos/requirements.txt
+++ b/testing/talos/requirements.txt
@@ -3,8 +3,9 @@ mozcrash>=0.15
 mozfile>=1.2
 mozhttpd>=0.7
 mozinfo>=0.8
 mozprocess>=0.22
 mozversion>=1.3
 mozprofile>=0.25
 psutil>=3.1.1
 simplejson>=2.1.1
+requests>=2.9.1

--- a/testing/talos/talos/cmdline.py
+++ b/testing/talos/talos/cmdline.py
@@ -174,16 +174,17 @@ def create_parser(mach_interface=False):
             help='If given, enable Stylo via Environment variables and '
                  'upload results with Stylo options.')
     add_arg('--disable-stylo', action="store_true",
             dest='disable_stylo',
             help='If given, disable Stylo via Environment variables.')
     add_arg('--stylo-threads', type=int,
             dest='stylothreads',
             help='If given, run Stylo with a certain number of threads')
-
+    add_arg('--profile', type=str, default=None,
+            help="Downloads a profile from TaskCluster and uses it")
     add_logging_group(parser)
     return parser
 
 
 def parse_args(argv=None):
     parser = create_parser()
     return parser.parse_args(argv)

--- a/testing/talos/talos/ffsetup.py
+++ b/testing/talos/talos/ffsetup.py
@@ -14,16 +14,17 @@ import mozfile
 import mozinfo
 import mozrunner
 from mozlog import get_proxy_logger
 from mozprocess import ProcessHandlerMixin
 from mozprofile.profile import Profile
 from talos import utils
 from talos.gecko_profile import GeckoProfile
 from talos.utils import TalosError
+from talos import heavy
 
 LOG = get_proxy_logger()
 
 
 class FFSetup(object):
     """
     Initialize the browser environment before running a test.
 
@@ -87,16 +88,25 @@ class FFSetup(object):
             if type(value) is str:
                 value = utils.interpolate(value, webserver=webserver)
                 preferences[name] = value
 
         extensions = self.browser_config['extensions'][:]
         if self.test_config.get('extensions'):
             extensions.append(self.test_config['extensions'])
 
+        if self.browser_config['develop'] or \
+           'try' in str.lower(self.browser_config['branch_name']):
+            extensions = [os.path.dirname(i) for i in extensions]
+
+        # downloading a profile instead of using the empty one
+        if self.test_config['profile'] is not None:
+            path = heavy.download_profile(self.test_config['profile'])
+            self.test_config['profile_path'] = path
+
         profile = Profile.clone(
             os.path.normpath(self.test_config['profile_path']),
             self.profile_dir,
             restore=False)
 
         profile.set_preferences(preferences)
 
         # installing addons

new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/heavy.py
@@ -0,0 +1,142 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Downloads Heavy profiles from TaskCluster.
+"""
+from __future__ import absolute_import
+import os
+import tarfile
+import functools
+import datetime
+from email.utils import parsedate
+
+import requests
+from requests.adapters import HTTPAdapter
+from mozlog import get_proxy_logger
+
+
+LOG = get_proxy_logger()
+TC_LINK = ("https://index.taskcluster.net/v1/task/garbage.heavyprofile/"
+           "artifacts/public/today-%s.tgz")
+
+
+class ProgressBar(object):
+    def __init__(self, size, template="\r%d%%"):
+        self.size = size
+        self.current = 0
+        self.tens = 0
+        self.template = template
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+    def incr(self):
+        if self.current == self.size:
+            return
+        percent = float(self.current) / float(self.size) * 100
+        tens, __ = divmod(percent, 10)
+        if tens > self.tens:
+            LOG.info(self.template % percent)
+            self.tens = tens
+
+        self.current += 1
+
+
+def follow_redirects(url, max=3):
+    location = url
+    current = 0
+    page = requests.head(url)
+    while page.status_code == 303 and current < max:
+        current += 1
+        location = page.headers['Location']
+        page = requests.head(location)
+    if page.status_code == 303 and current == max:
+        raise ValueError("Max redirects Reached")
+    last_modified = page.headers['Last-Modified']
+    last_modified = datetime.datetime(*parsedate(last_modified)[:6])
+    return location, last_modified
+
+
+def _recursive_mtime(path):
+    max = os.path.getmtime(path)
+    for root, dirs, files in os.walk(path):
+        for element in dirs + files:
+            age = os.path.getmtime(os.path.join(root, element))
+            if age > max:
+                max = age
+    return max
+
+
+def profile_age(profile_dir, last_modified=None):
+    if last_modified is None:
+        last_modified = datetime.datetime.now()
+
+    profile_ts = _recursive_mtime(profile_dir)
+    profile_ts = datetime.datetime.fromtimestamp(profile_ts)
+    return (last_modified - profile_ts).days
+
+
+def download_profile(name, profiles_dir=None):
+    if profiles_dir is None:
+        profiles_dir = os.path.join(os.path.expanduser('~'), '.mozilla',
+                                    'profiles')
+    profiles_dir = os.path.abspath(profiles_dir)
+    if not os.path.exists(profiles_dir):
+        os.makedirs(profiles_dir)
+
+    target = os.path.join(profiles_dir, name)
+    url = TC_LINK % name
+    cache_dir = os.path.join(profiles_dir, '.cache')
+    if not os.path.exists(cache_dir):
+        os.makedirs(cache_dir)
+
+    archive_file = os.path.join(cache_dir, 'today-%s.tgz' % name)
+
+    url, last_modified = follow_redirects(url)
+    if os.path.exists(target):
+        age = profile_age(target, last_modified)
+        if age < 7:
+            # profile is not older than a week, we're good
+            LOG.info("Local copy of %r is fresh enough" % name)
+            LOG.info("%d days old" % age)
+            return target
+
+    LOG.info("Downloading from %r" % url)
+    session = requests.Session()
+    session.mount('https://', HTTPAdapter(max_retries=5))
+    req = session.get(url, stream=True, timeout=20)
+    req.raise_for_status()
+
+    total_length = int(req.headers.get('content-length'))
+
+    # XXX implement Range to resume download on disconnects
+    template = 'Download progress %d%%'
+    with open(archive_file, 'wb') as f:
+        iter = req.iter_content(chunk_size=1024)
+        size = total_length / 1024 + 1
+        with ProgressBar(size=size, template=template) as bar:
+            for chunk in iter:
+                if chunk:
+                    f.write(chunk)
+                bar.incr()
+
+    LOG.info("Extracting profile in %r" % target)
+    template = 'Extraction progress %d%%'
+
+    with tarfile.open(archive_file, "r:gz") as tar:
+        LOG.info("Checking the tarball content...")
+        size = len(list(tar))
+        with ProgressBar(size=size, template=template) as bar:
+            def _extract(self, *args, **kw):
+                bar.incr()
+                return self.old(*args, **kw)
+            tar.old = tar.extract
+            tar.extract = functools.partial(_extract, tar)
+            tar.extractall(target)
+    LOG.info("Profile downloaded.")
+    return target

--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -105,23 +105,30 @@ def run_tests(config, browser_config):
                 os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'],
                                                '/\\t:\\')))
         if not test.get('url'):
             # build 'url' for tptest
             test['url'] = buildCommandLine(test)
         test['url'] = utils.interpolate(test['url'])
         test['setup'] = utils.interpolate(test['setup'])
         test['cleanup'] = utils.interpolate(test['cleanup'])
+        test['profile'] = config.get('profile')
 
     # pass --no-remote to firefox launch, if --develop is specified
     # we do that to allow locally the user to have another running firefox
     # instance
     if browser_config['develop']:
         browser_config['extra_args'] = '--no-remote'
 
+    # with addon signing for production talos, we want to develop without it
+    if browser_config['develop'] or 'try' in str.lower(browser_config['branch_name']):
+        browser_config['preferences']['xpinstall.signatures.required'] = False
+
+    browser_config['preferences']['extensions.allow-non-mpc-extensions'] = True
+
     # if using firstNonBlankPaint, must turn on pref for it
     if test.get('fnbpaint', False):
         LOG.info("Using firstNonBlankPaint, so turning on pref for it")
         browser_config['preferences']['dom.performance.time_to_non_blank_paint.enabled'] = True
 
     # set defaults
     testdate = config.get('testdate', '')
 

new file mode 100644
index 0000000000000000000000000000000000000000..0ca4fe62d3e134a5985bea4ab97d06e5099eb1e3
GIT binary patch
literal 338
zc$|~(=3qGd_CzED^Xc>6sS8XEfWX)=km2I9v}FlW90^+*8z(ND*|c!uLdgkiOw6pz
z&dSWm!fdR}vv%`vcy)UVbvO7iPFi`h;>y(YCCi>YnCSj>3e(hBPh*AqXAf?8__ou7
z?b&wm+MRsb99wLsSgyJgZ2z;kYoWujL!3d?uWIHTd0({U-Ku%_wf4VgtopEZvt`gd
zJ+)O&&tLmv^EKzw_vCl4`PZEj3fgX+^yvNZZ?)HFf1h1+t;%)(-W|p^HO0R^+wX6_
zKlgg>^*>p^mY4rC{&{}dSM~INDQ;y=hbxb9ncct4BgHJrrYt!nMP%uvT`USy{yWN4
z{4-~KUY{)d`Qjgc{(zu){>9cFUG>Sq1wZq%tfc-gP7vv73N+wyJE$SSN+jn91A|V^
K&S(Y=1_l6+wwhW1

new file mode 100644
--- /dev/null
+++ b/testing/talos/tests/test_heavy.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+
+"""
+test talos' heavy module:
+
+http://hg.mozilla.org/build/talos/file/tip/talos/heavy.py
+"""
+from __future__ import absolute_import
+import unittest
+import tempfile
+import shutil
+import datetime
+import contextlib
+import os
+import time
+
+import talos.heavy
+
+
+archive = os.path.join(os.path.dirname(__file__), 'profile.tgz')
+archive_size = os.stat(archive).st_size
+
+
+@contextlib.contextmanager
+def mock_requests(**kw):
+    class Session:
+        def mount(self, *args, **kw):
+            pass
+
+    kw['Session'] = Session
+    old = {}
+    for meth, func in kw.items():
+        curr = getattr(talos.heavy.requests, meth)
+        old[meth] = curr
+        setattr(talos.heavy.requests, meth, func)
+        setattr(Session, meth, func)
+    try:
+        yield
+    finally:
+        for meth, func in old.items():
+            setattr(talos.heavy.requests, meth, func)
+
+
+class _Response(object):
+    def __init__(self, code, headers=None, file=None):
+        if headers is None:
+            headers = {}
+        self.headers = headers
+        self.status_code = code
+        self.file = file
+
+    def raise_for_status(self):
+        pass
+
+    def iter_content(self, chunk_size):
+        with open(self.file, 'rb') as f:
+            yield f.read(chunk_size)
+
+
+class Logger:
+    def __init__(self):
+        self.data = []
+
+    def info(self, msg):
+        self.data.append(msg)
+
+
+class TestFilter(unittest.TestCase):
+
+    def setUp(self):
+        self.temp = tempfile.mkdtemp()
+        self.logs = talos.heavy.LOG.logger = Logger()
+
+    def tearDown(self):
+        shutil.rmtree(self.temp)
+
+    def test_profile_age(self):
+        """test profile_age function"""
+        days = talos.heavy.profile_age(self.temp)
+        self.assertEqual(days, 0)
+
+        _8_days = datetime.datetime.now() + datetime.timedelta(days=8)
+        days = talos.heavy.profile_age(self.temp, _8_days)
+        self.assertEqual(days, 8)
+
+    def test_directory_age(self):
+        """make sure it detects changes in files in subdirs"""
+        with open(os.path.join(self.temp, 'file'), 'w') as f:
+            f.write('xxx')
+
+        current_age = talos.heavy._recursive_mtime(self.temp)
+        time.sleep(1.1)
+
+        with open(os.path.join(self.temp, 'file'), 'w') as f:
+            f.write('----')
+
+        self.assertTrue(current_age < talos.heavy._recursive_mtime(self.temp))
+
+    def test_follow_redirect(self):
+        """test follow_redirect function"""
+        _8_days = datetime.datetime.now() + datetime.timedelta(days=8)
+        _8_days = _8_days.strftime('%a, %d %b %Y %H:%M:%S UTC')
+
+        resps = [_Response(303, {'Location': 'blah'}),
+                 _Response(303, {'Location': 'bli'}),
+                 _Response(200, {'Last-Modified': _8_days})]
+
+        class Counter:
+            c = 0
+
+        def _head(url, curr=Counter()):
+            curr.c += 1
+            return resps[curr.c]
+
+        with mock_requests(head=_head):
+            loc, lm = talos.heavy.follow_redirects('https://example.com')
+            days = talos.heavy.profile_age(self.temp, lm)
+            self.assertEqual(days, 8)
+
+    def _test_download(self, age):
+
+        def _days(num):
+            d = datetime.datetime.now() + datetime.timedelta(days=num)
+            return d.strftime('%a, %d %b %Y %H:%M:%S UTC')
+
+        resps = [_Response(303, {'Location': 'blah'}),
+                 _Response(303, {'Location': 'bli'}),
+                 _Response(200, {'Last-Modified': _days(age)})]
+
+        class Counter:
+            c = 0
+
+        def _head(url, curr=Counter()):
+            curr.c += 1
+            return resps[curr.c]
+
+        def _get(url, *args, **kw):
+            return _Response(200, {'Last-Modified': _days(age),
+                                   'content-length': str(archive_size)},
+                             file=archive)
+
+        with mock_requests(head=_head, get=_get):
+            target = talos.heavy.download_profile('simple',
+                                                  profiles_dir=self.temp)
+            profile = os.path.join(self.temp, 'simple')
+            self.assertTrue(os.path.exists(profile))
+            return target
+
+    def test_download_profile(self):
+        """test downloading heavy profile"""
+        # a 12 days old profile gets updated
+        self._test_download(12)
+
+        # a 8 days two
+        self._test_download(8)
+
+        # a 2 days sticks
+        self._test_download(2)
+        self.assertTrue("fresh enough" in self.logs.data[-2])
+
+
+if __name__ == '__main__':
+    unittest.main()