Bug 1239808 - Rename test_packages.json to include package basename as prefix. r=chmanchester, r=jlund draft
authorHenrik Skupin <mail@hskupin.info>
Thu, 25 Feb 2016 21:37:47 +0100
changeset 334669 6b9c69f287e491efee4411fe94c7282d5a5cbf11
parent 334171 ba56be94d03d8079358115e3def6a08e09d6f435
child 334670 573a218cae828930c1631bb26765c303c4dc05af
push id11607
push userbmo:hskupin@gmail.com
push dateThu, 25 Feb 2016 20:51:19 +0000
reviewerschmanchester, jlund
bugs1239808
milestone47.0a1
Bug 1239808 - Rename test_packages.json to include package basename as prefix. r=chmanchester, r=jlund MozReview-Commit-ID: 6JiHIbSjhoR
build/gen_test_packages_manifest.py
testing/mozharness/mozharness/mozilla/testing/testbase.py
testing/testsuite-targets.mk
toolkit/mozapps/installer/package-name.mk
--- a/build/gen_test_packages_manifest.py
+++ b/build/gen_test_packages_manifest.py
@@ -40,53 +40,46 @@ OPTIONAL_PACKAGES = [
 def parse_args():
     parser = ArgumentParser(description='Generate a test_packages.json file to tell automation which harnesses require which test packages.')
     parser.add_argument("--common", required=True,
                         action="store", dest="tests_common",
                         help="Name of the \"common\" archive, a package to be used by all harnesses.")
     parser.add_argument("--jsshell", required=True,
                         action="store", dest="jsshell",
                         help="Name of the jsshell zip.")
-    parser.add_argument("--use-short-names", action="store_true",
-                        help="Use short names for packages (target.$name.tests.zip "
-                             "instead of $(PACKAGE_BASENAME).$name.tests.zip)")
     for harness in PACKAGE_SPECIFIED_HARNESSES:
         parser.add_argument("--%s" % harness, required=True,
                             action="store", dest=harness,
                             help="Name of the %s zip." % harness)
     for harness in OPTIONAL_PACKAGES:
         parser.add_argument("--%s" % harness, required=False,
                             action="store", dest=harness,
                             help="Name of the %s zip." % harness)
     parser.add_argument("--dest-file", required=True,
                         action="store", dest="destfile",
                         help="Path to the output file to be written.")
     return parser.parse_args()
 
+
 def generate_package_data(args):
     # Generate a dictionary mapping test harness names (exactly as they're known to
     # mozharness and testsuite-targets.mk, ideally) to the set of archive names that
     # harness depends on to run.
     # mozharness will use this file to determine what test zips to download,
     # which will be an optimization once parts of the main zip are split to harness
     # specific zips.
     tests_common = args.tests_common
-    if args.use_short_names:
-        tests_common = 'target.common.tests.zip'
-
     jsshell = args.jsshell
 
     harness_requirements = dict([(k, [tests_common]) for k in ALL_HARNESSES])
     harness_requirements['jittest'].append(jsshell)
     for harness in PACKAGE_SPECIFIED_HARNESSES + OPTIONAL_PACKAGES:
         pkg_name = getattr(args, harness, None)
         if pkg_name is None:
             continue
-        if args.use_short_names:
-            pkg_name = 'target.%s.tests.zip' % harness
         harness_requirements[harness].append(pkg_name)
     return harness_requirements
 
 if __name__ == '__main__':
     args = parse_args()
     packages_data = generate_package_data(args)
     with open(args.destfile, 'w') as of:
         json.dump(packages_data, of, indent=4)
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -159,26 +159,44 @@ class TestingMixin(VirtualenvMixin, Buil
             self.fatal("Can't figure out build directory urls without an installer_url "
                        "or test_packages_url!")
 
         last_slash = reference_url.rfind('/')
         base_url = reference_url[:last_slash]
 
         return '%s/%s' % (base_url, file_name)
 
+    def query_prefixed_build_dir_url(self, suffix):
+        """Resolve a file name prefixed with platform and build details to a potential url
+        in the build upload directory where that file can be found.
+        """
+        if self.test_packages_url:
+            reference_suffixes = ['.test_packages.json']
+            reference_url = self.test_packages_url
+        elif self.installer_url:
+            reference_suffixes = INSTALLER_SUFFIXES
+            reference_url = self.installer_url
+        else:
+            self.fatal("Can't figure out build directory urls without an installer_url "
+                       "or test_packages_url!")
+
+        url = None
+        for reference_suffix in reference_suffixes:
+            if reference_url.endswith(reference_suffix):
+                url = reference_url[:-len(reference_suffix)] + suffix
+                break
+
+        return url
+
     def query_symbols_url(self):
         if self.symbols_url:
             return self.symbols_url
 
         elif self.installer_url:
-            symbols_url = None
-            for suffix in INSTALLER_SUFFIXES:
-                if self.installer_url.endswith(suffix):
-                    symbols_url = self.installer_url[:-len(suffix)] + '.crashreporter-symbols.zip'
-                    break
+            symbols_url = self.query_prefixed_build_dir_url('.crashreporter-symbols.zip')
 
             # Check if the URL exists. If not, use none to allow mozcrash to auto-check for symbols
             try:
                 if symbols_url:
                     self._urlopen(symbols_url)
                     self.symbols_url = symbols_url
             except urllib2.URLError:
                 self.warning("Can't figure out symbols_url from installer_url: %s!" %
@@ -531,17 +549,17 @@ 2. running via buildbot and running the 
 
             self._download_test_zip(target_unzip_dirs)
         else:
             if not self.test_packages_url:
                 # The caller intends to download harness specific packages, but doesn't know
                 # where the packages manifest is located. This is the case when the
                 # test package manifest isn't set as a buildbot property, which is true
                 # for some self-serve jobs and platforms using parse_make_upload.
-                self.test_packages_url = self.query_build_dir_url('test_packages.json')
+                self.test_packages_url = self.query_prefixed_build_dir_url('.test_packages.json')
 
             suite_categories = suite_categories or ['common']
             self._download_test_packages(suite_categories, target_unzip_dirs)
 
         self._download_installer()
         if self.config.get('download_symbols'):
             self._download_and_extract_symbols()
 
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -261,17 +261,16 @@ stage-all: \
   stage-mochitest \
   stage-xpcshell \
   stage-jstests \
   stage-jetpack \
   stage-marionette \
   stage-cppunittests \
   stage-luciddream \
   test-packages-manifest \
-  test-packages-manifest-tc \
   $(NULL)
 ifdef MOZ_WEBRTC
 stage-all: stage-steeplechase
 endif
 
 TEST_PKGS := \
   common \
   cppunittest \
@@ -284,26 +283,16 @@ TEST_PKGS := \
 
 ifdef BUILD_GTEST
 stage-all: stage-gtest
 TEST_PKGS += gtest
 endif
 
 PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.zip'
 
-test-packages-manifest-tc:
-	@rm -f $(MOZ_TEST_PACKAGES_FILE_TC)
-	$(NSINSTALL) -D $(dir $(MOZ_TEST_PACKAGES_FILE_TC))
-	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
-      --jsshell $(JSSHELL_NAME) \
-      --dest-file $(MOZ_TEST_PACKAGES_FILE_TC) \
-      --use-short-names \
-      $(call PKG_ARG,common) \
-      $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
-
 test-packages-manifest:
 	@rm -f $(MOZ_TEST_PACKAGES_FILE)
 	$(NSINSTALL) -D $(dir $(MOZ_TEST_PACKAGES_FILE))
 	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
       --jsshell $(JSSHELL_NAME) \
       --dest-file $(MOZ_TEST_PACKAGES_FILE) \
       $(call PKG_ARG,common) \
       $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
@@ -473,11 +462,9 @@ stage-extensions: make-stage-dir
   stage-jstests \
   stage-android \
   stage-jetpack \
   stage-marionette \
   stage-steeplechase \
   stage-instrumentation-tests \
   stage-luciddream \
   test-packages-manifest \
-  test-packages-manifest-tc \
   $(NULL)
-
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -151,18 +151,17 @@ ifndef INCLUDED_RCS_MK
   USE_RCS_MK := 1
   include $(MOZILLA_DIR)/config/makefiles/makeutils.mk
 endif
 
 MOZ_SOURCESTAMP_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).txt
 MOZ_BUILDINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).json
 MOZ_BUILDID_INFO_TXT_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME)_info.txt
 MOZ_MOZINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).mozinfo.json
-MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/test_packages.json
-MOZ_TEST_PACKAGES_FILE_TC = $(DIST)/$(PKG_PATH)/test_packages_tc.json
+MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/$(PKG_BASENAME).test_packages.json
 
 # JavaScript Shell
 ifdef MOZ_SIMPLE_PACKAGE_NAME
 JSSHELL_NAME := $(MOZ_SIMPLE_PACKAGE_NAME).jsshell.zip
 else
 JSSHELL_NAME = jsshell-$(MOZ_PKG_PLATFORM).zip
 endif
 PKG_JSSHELL = $(DIST)/$(JSSHELL_NAME)