Bug 1472171 - Upload raw structured logs as artifacts from Python unit tests; r?ahal draft
authorDave Hunt <dhunt@mozilla.com>
Fri, 29 Jun 2018 14:37:17 +0100
changeset 812477 87f4ccc002eb8a8d2c5dc3c5d2f6e47b86d13134
parent 812437 bf149090f6b5db3fe86618b5f1047b70b1bff8b5
push id114567
push userbmo:dave.hunt@gmail.com
push dateFri, 29 Jun 2018 14:03:19 +0000
reviewersahal
bugs1472171
milestone63.0a1
Bug 1472171 - Upload raw structured logs as artifacts from Python unit tests; r?ahal MozReview-Commit-ID: 860MlyItg1B
config/mozunit/mozunit/mozunit.py
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/run_task.py
--- a/config/mozunit/mozunit/mozunit.py
+++ b/config/mozunit/mozunit/mozunit.py
@@ -227,17 +227,22 @@ def main(*args, **kwargs):
 
     if runwith == 'unittest':
         unittest.main(testRunner=MozTestRunner(), *args, **kwargs)
     else:
         args = list(args)
         if os.environ.get('MACH_STDOUT_ISATTY') and not any(a.startswith('--color') for a in args):
             args.append('--color=yes')
 
-        module = __import__('__main__')
+        path = __import__('__main__').__file__
+
+        if os.environ.get('MOZ_AUTOMATION') and os.environ.get('MOZ_UPLOAD_DIR'):
+            name = 'log_{}_raw.log'.format(os.path.basename(os.path.splitext(path)[0]))
+            args.extend(['--log-raw', os.path.join(os.environ['MOZ_UPLOAD_DIR'], name)])
+
         args.extend([
             '-c', os.path.join(here, 'pytest.ini'),
             '-vv',
             '-p', 'mozlog.pytest_mozlog.plugin',
             '-p', 'no:cacheprovider',
-            module.__file__,
+            path,
         ])
         sys.exit(pytest.main(args))
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -39,16 +39,17 @@ def docker_worker_add_workspace_cache(co
 
 
 def add_artifacts(config, job, taskdesc, path):
     taskdesc['worker'].setdefault('artifacts', []).append({
         'name': get_artifact_prefix(taskdesc),
         'path': path,
         'type': 'directory',
     })
+    taskdesc['worker'].setdefault('env', {})['MOZ_UPLOAD_DIR'] = path
 
 
 def docker_worker_add_artifacts(config, job, taskdesc):
     """ Adds an artifact directory to the task """
     add_artifacts(config, job, taskdesc, path='{workdir}/artifacts/'.format(**job['run']))
 
 
 def generic_worker_add_artifacts(config, job, taskdesc):
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -2,20 +2,25 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Support for running jobs that are invoked via the `run-task` script.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+from voluptuous import Required, Any
+
 from taskgraph.transforms.job import run_job_using
 from taskgraph.util.schema import Schema
-from taskgraph.transforms.job.common import support_use_artifacts, support_vcs_checkout
-from voluptuous import Required, Any
+from taskgraph.transforms.job.common import (
+    docker_worker_add_artifacts,
+    support_use_artifacts,
+    support_vcs_checkout,
+)
 
 run_task_schema = Schema({
     Required('using'): 'run-task',
 
     # if true, add a cache at ~worker/.cache, which is where things like pip
     # tend to hide their caches.  This cache is never added for level-1 jobs.
     Required('cache-dotcache'): bool,
 
@@ -89,16 +94,18 @@ def docker_worker_run_task(config, job, 
     if run.get('cache-dotcache'):
         worker['caches'].append({
             'type': 'persistent',
             'name': 'level-{level}-{project}-dotcache'.format(**config.params),
             'mount-point': '{workdir}/.cache'.format(**run),
             'skip-untrusted': True,
         })
 
+    docker_worker_add_artifacts(config, job, taskdesc)
+
     run_command = run['command']
     if isinstance(run_command, basestring):
         run_command = ['bash', '-cx', run_command]
     command = ['{workdir}/bin/run-task'.format(**run)]
     add_checkout_to_command(run, command)
     if run['comm-checkout']:
         command.append('--comm-checkout={workdir}/checkouts/gecko/comm'.format(**run))
     command.append('--fetch-hgfingerprint')