Bug 1462783 - Artifact raptor test results in treeherder; r?ahal
MozReview-Commit-ID: FYXU2nVhn3n
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -5,16 +5,18 @@
from __future__ import absolute_import, print_function, unicode_literals
import copy
import json
import os
import re
import sys
+from shutil import copyfile
+
import mozharness
from mozharness.base.config import parse_config_file
from mozharness.base.errors import PythonErrorList
from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO, WARNING
from mozharness.base.python import Python3Virtualenv
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
from mozharness.mozilla.tooltool import TooltoolMixin
@@ -248,25 +250,32 @@ class Raptor(TestingMixin, MercurialScri
schema_path = os.path.join(external_tools_path,
'performance-artifact-schema.json')
self.info("Validating PERFHERDER_DATA against %s" % schema_path)
try:
with open(schema_path) as f:
schema = json.load(f)
data = json.loads(parser.found_perf_data[0])
jsonschema.validate(data, schema)
- except:
+ except Exception as e:
self.exception("Error while validating PERFHERDER_DATA")
+ self.info(e)
def _artifact_perf_data(self, dest):
- src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'local.json')
+ src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor.json')
+ if not os.path.isdir(os.path.dirname(dest)):
+ # create upload dir if it doesn't already exist
+ self.info("creating dir: %s" % os.path.dirname(dest))
+ os.makedirs(os.path.dirname(dest))
+ self.info('copying raptor results from %s to %s' % (src, dest))
try:
- shutil.copyfile(src, dest)
- except:
+ copyfile(src, dest)
+ except Exception as e:
self.critical("Error copying results %s to upload dir %s" % (src, dest))
+ self.info(e)
def run_tests(self, args=None, **kw):
"""run raptor tests"""
# get raptor options
options = self.raptor_options(args=args, **kw)
# python version check
@@ -341,17 +350,19 @@ class Raptor(TestingMixin, MercurialScri
if self.return_code == 4:
log_level = WARNING
elif '--no-upload-results' not in options:
if not self.gecko_profile:
self._validate_treeherder_data(parser)
if not self.run_local:
# copy results to upload dir so they are included as an artifact
+ self.info("copying raptor results to upload dir:")
dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'perfherder-data.json')
+ self.info(str(dest))
self._artifact_perf_data(dest)
class RaptorOutputParser(OutputParser):
minidump_regex = re.compile(r'''raptorError: "error executing: '(\S+) (\S+) (\S+)'"''')
RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})')
def __init__(self, **kwargs):
--- a/testing/raptor/raptor/output.py
+++ b/testing/raptor/raptor/output.py
@@ -6,19 +6,21 @@
# some parts of this originally taken from /testing/talos/talos/output.py
"""output raptor test results"""
from __future__ import absolute_import
import filter
import json
+import os
+
from mozlog import get_proxy_logger
-LOG = get_proxy_logger(component="output")
+LOG = get_proxy_logger(component="raptor-output")
class Output(object):
"""class for raptor output"""
def __init__(self, results):
"""
- results : list of RaptorTestResult instances
@@ -78,29 +80,38 @@ class Output(object):
self.summarized_results = test_results
def output(self):
"""output to file and perfherder data json """
if self.summarized_results == {}:
LOG.error("error: no summarized raptor results found!")
return False
- results_path = "raptor.json"
+ if os.environ['MOZ_UPLOAD_DIR']:
+ # i.e. testing/mozharness/build/raptor.json locally; in production it will
+ # be at /tasks/task_*/build/ (where it will be picked up by mozharness later
+ # and made into a tc artifact accessible in treeherder as perfherder-data.json)
+ results_path = os.path.join(os.path.dirname(os.environ['MOZ_UPLOAD_DIR']),
+ 'raptor.json')
+ else:
+ results_path = os.path.join(os.getcwd(), 'raptor.json')
with open(results_path, 'w') as f:
for result in self.summarized_results:
f.write("%s\n" % result)
# the output that treeherder expects to find
extra_opts = self.summarized_results['suites'][0].get('extraOptions', [])
if 'geckoProfile' not in extra_opts:
LOG.info("PERFHERDER_DATA: %s" % json.dumps(self.summarized_results))
json.dump(self.summarized_results, open(results_path, 'w'), indent=2,
sort_keys=True)
+
+ LOG.info("results can also be found locally at: %s" % results_path)
return True
@classmethod
def v8_Metric(cls, val_list):
results = [i for i, j in val_list]
score = 100 * filter.geometric_mean(results)
return score