Bug 1462783 - Artifact raptor test results in treeherder; r?ahal draft
authorRob Wood <rwood@mozilla.com>
Thu, 24 May 2018 10:50:39 -0400
changeset 800608 bc77410aee7efb332a115a624c879c303dc7a42d
parent 800535 1d53c55e96dfbcd965b8a722539ea049bb45fffe
push id111425
push userrwood@mozilla.com
push dateMon, 28 May 2018 17:35:11 +0000
reviewersahal
bugs1462783
milestone62.0a1
Bug 1462783 - Artifact raptor test results in treeherder; r?ahal MozReview-Commit-ID: FYXU2nVhn3n
testing/mozharness/mozharness/mozilla/testing/raptor.py
testing/raptor/raptor/output.py
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -5,16 +5,18 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 import json
 import os
 import re
 import sys
 
+from shutil import copyfile
+
 import mozharness
 
 from mozharness.base.config import parse_config_file
 from mozharness.base.errors import PythonErrorList
 from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO, WARNING
 from mozharness.base.python import Python3Virtualenv
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.mozilla.tooltool import TooltoolMixin
@@ -248,25 +250,32 @@ class Raptor(TestingMixin, MercurialScri
         schema_path = os.path.join(external_tools_path,
                                    'performance-artifact-schema.json')
         self.info("Validating PERFHERDER_DATA against %s" % schema_path)
         try:
             with open(schema_path) as f:
                 schema = json.load(f)
             data = json.loads(parser.found_perf_data[0])
             jsonschema.validate(data, schema)
-        except:
+        except Exception as e:
             self.exception("Error while validating PERFHERDER_DATA")
+            self.info(e)
 
     def _artifact_perf_data(self, dest):
-        src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'local.json')
+        src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor.json')
+        if not os.path.isdir(os.path.dirname(dest)):
+            # create upload dir if it doesn't already exist
+            self.info("creating dir: %s" % os.path.dirname(dest))
+            os.makedirs(os.path.dirname(dest))
+        self.info('copying raptor results from %s to %s' % (src, dest))
         try:
-            shutil.copyfile(src, dest)
-        except:
+            copyfile(src, dest)
+        except Exception as e:
             self.critical("Error copying results %s to upload dir %s" % (src, dest))
+            self.info(e)
 
     def run_tests(self, args=None, **kw):
         """run raptor tests"""
 
         # get raptor options
         options = self.raptor_options(args=args, **kw)
 
         # python version check
@@ -341,17 +350,19 @@ class Raptor(TestingMixin, MercurialScri
             if self.return_code == 4:
                 log_level = WARNING
 
         elif '--no-upload-results' not in options:
             if not self.gecko_profile:
                 self._validate_treeherder_data(parser)
                 if not self.run_local:
                     # copy results to upload dir so they are included as an artifact
+                    self.info("copying raptor results to upload dir:")
                     dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'perfherder-data.json')
+                    self.info(str(dest))
                     self._artifact_perf_data(dest)
 
 
 class RaptorOutputParser(OutputParser):
     minidump_regex = re.compile(r'''raptorError: "error executing: '(\S+) (\S+) (\S+)'"''')
     RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})')
 
     def __init__(self, **kwargs):
--- a/testing/raptor/raptor/output.py
+++ b/testing/raptor/raptor/output.py
@@ -6,19 +6,21 @@
 # some parts of this originally taken from /testing/talos/talos/output.py
 
 """output raptor test results"""
 from __future__ import absolute_import
 
 import filter
 
 import json
+import os
+
 from mozlog import get_proxy_logger
 
-LOG = get_proxy_logger(component="output")
+LOG = get_proxy_logger(component="raptor-output")
 
 
 class Output(object):
     """class for raptor output"""
 
     def __init__(self, results):
         """
         - results : list of RaptorTestResult instances
@@ -78,29 +80,38 @@ class Output(object):
         self.summarized_results = test_results
 
     def output(self):
         """output to file and perfherder data json """
         if self.summarized_results == {}:
             LOG.error("error: no summarized raptor results found!")
             return False
 
-        results_path = "raptor.json"
+        if os.environ['MOZ_UPLOAD_DIR']:
+            # i.e. testing/mozharness/build/raptor.json locally; in production it will
+            # be at /tasks/task_*/build/ (where it will be picked up by mozharness later
+            # and made into a tc artifact accessible in treeherder as perfherder-data.json)
+            results_path = os.path.join(os.path.dirname(os.environ['MOZ_UPLOAD_DIR']),
+                                        'raptor.json')
+        else:
+            results_path = os.path.join(os.getcwd(), 'raptor.json')
 
         with open(results_path, 'w') as f:
             for result in self.summarized_results:
                 f.write("%s\n" % result)
 
         # the output that treeherder expects to find
         extra_opts = self.summarized_results['suites'][0].get('extraOptions', [])
         if 'geckoProfile' not in extra_opts:
             LOG.info("PERFHERDER_DATA: %s" % json.dumps(self.summarized_results))
 
         json.dump(self.summarized_results, open(results_path, 'w'), indent=2,
                   sort_keys=True)
+
+        LOG.info("results can also be found locally at: %s" % results_path)
         return True
 
     @classmethod
     def v8_Metric(cls, val_list):
         results = [i for i, j in val_list]
         score = 100 * filter.geometric_mean(results)
         return score