Bug 1284475 - migrate ToolTool blueprint to new codebase of relengapi draft
authorRok Garbas <rok@garbas.si>
Tue, 08 Aug 2017 10:55:52 +0200
changeset 651415 8c47b99825c8447757370f6fa5e8ac50f94f3afd
parent 651395 c86b7150523c10e1d1dbc0be2d8ed96f205be35f
child 727710 1ba3084dacc318e5cdaef994ceb2c3579d15e560
push id75724
push userrgarbas@mozilla.com
push dateWed, 23 Aug 2017 18:04:24 +0000
bugs1284475
milestone57.0a1
Bug 1284475 - migrate ToolTool blueprint to new codebase of relengapi UPGRADE_NSS_RELEASE a=kaie MozReview-Commit-ID: 7CB1VYb8OJP
js/src/devtools/automation/winbuildenv.sh
python/mozboot/bin/bootstrap-msys2.vbs
python/mozbuild/mozbuild/action/tooltool.py
python/mozbuild/mozbuild/mach_commands.py
security/nss/automation/taskcluster/windows/setup.sh
taskcluster/docker/centos6-build/system-setup.sh
taskcluster/docker/recipes/common.sh
taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh
taskcluster/scripts/builder/build-haz-linux.sh
taskcluster/scripts/builder/sm-tooltool-config.sh
testing/mozharness/configs/developer_config.py
testing/mozharness/configs/single_locale/win32.py
testing/mozharness/configs/single_locale/win32_devedition.py
testing/mozharness/configs/single_locale/win64.py
testing/mozharness/configs/single_locale/win64_devedition.py
testing/mozharness/mozharness/mozilla/testing/codecoverage.py
testing/mozharness/mozharness/mozilla/tooltool.py
testing/mozharness/scripts/fx_desktop_build.py
tools/mach_commands.py
--- a/js/src/devtools/automation/winbuildenv.sh
+++ b/js/src/devtools/automation/winbuildenv.sh
@@ -25,12 +25,12 @@ else
   . $topsrcdir/build/win32/mozconfig.vs-latest
 fi
 
 # PATH also needs to point to mozmake.exe, which can come from either
 # newer mozilla-build or tooltool.
 if ! which mozmake 2>/dev/null; then
     export PATH="$PATH:$SOURCE/.."
     if ! which mozmake 2>/dev/null; then
-  TT_SERVER=${TT_SERVER:-https://api.pub.build.mozilla.org/tooltool/}
+  TT_SERVER=${TT_SERVER:-https://tooltool.mozilla-releng.net/}
   ( cd $SOURCE/..; $SOURCE/mach artifact toolchain -v --tooltool-manifest $SOURCE/browser/config/tooltool-manifests/${platform:-win32}/releng.manifest --tooltool-url $TT_SERVER --retry 4${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
     fi
 fi
--- a/python/mozboot/bin/bootstrap-msys2.vbs
+++ b/python/mozboot/bin/bootstrap-msys2.vbs
@@ -48,17 +48,17 @@ If NOT fso.FolderExists(installPath) The
 End If
 If NOT fso.FolderExists(installPath) Then
     MsgBox("Failed to create folder. Do you have permission to install in this directory?")
     WScript.Quit 1
 End If
 
 On Error Resume Next
 ' Download and move MSYS2 into the right place
-Download "https://api.pub.build.mozilla.org/tooltool/sha512/f93a685c8a10abbd349cbef5306441ba235c4cbfba1cc000299e11b58f258e9953cbe23463515407925eeca94c3f5d8e5f637c95be387e620845efa43cdcb0c0", "msys2.zip"
+Download "https://tooltool.mozilla-releng.net/sha512/f93a685c8a10abbd349cbef5306441ba235c4cbfba1cc000299e11b58f258e9953cbe23463515407925eeca94c3f5d8e5f637c95be387e620845efa43cdcb0c0", "msys2.zip"
 Set FilesInZip = appShell.NameSpace(fso.GetAbsolutePathName("msys2.zip")).Items()
 appShell.NameSpace(msysPath).CopyHere(FilesInZip)
 ' MSYS2 archive doesn't have tmp directory...
 fso.CreateFolder(fso.BuildPath(msysPath, "tmp"))
 fso.DeleteFile("msys2.zip")
 If Err.Number <> 0 Then
     MsgBox("Error downloading and installing MSYS2. Make sure you have internet connection. If you think this is a bug, please file one in Bugzilla https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=Build%20Config")
     WScript.Quit 1
--- a/python/mozbuild/mozbuild/action/tooltool.py
+++ b/python/mozbuild/mozbuild/action/tooltool.py
@@ -1,9 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 
 # tooltool is a lookaside cache implemented in Python
 # Copyright (C) 2011 John H. Ford <john@johnford.info>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation version 2
 #
@@ -994,17 +994,17 @@ def main(argv, _skip_logging=False):
                       help='Use the RelengAPI token found in the given file to '
                            'authenticate to the RelengAPI server.',
                       dest='auth_file')
 
     (options_obj, args) = parser.parse_args(argv[1:])
 
     # default the options list if not provided
     if not options_obj.base_url:
-        options_obj.base_url = ['https://api.pub.build.mozilla.org/tooltool/']
+        options_obj.base_url = ['https://tooltool.mozilla-releng.net/']
 
     # ensure all URLs have a trailing slash
     def add_slash(url):
         return url if url.endswith('/') else (url + '/')
     options_obj.base_url = [add_slash(u) for u in options_obj.base_url]
 
     # expand ~ in --authentication-file
     if options_obj.auth_file:
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1797,17 +1797,17 @@ class PackageFrontend(MachCommandBase):
             logging.getLogger('redo').addHandler(
                 self.log_manager.terminal_handler)
             self.log_manager.terminal_handler.addFilter(
                 self.log_manager.structured_filter)
         if not cache_dir:
             cache_dir = os.path.join(self._mach_context.state_dir, 'toolchains')
 
         tooltool_url = (tooltool_url or
-                        'https://api.pub.build.mozilla.org/tooltool').rstrip('/')
+                        'https://tooltool.mozilla-releng.net').rstrip('/')
 
         cache = ArtifactCache(cache_dir=cache_dir, log=self.log,
                               skip_cache=skip_cache)
 
         if authentication_file:
             with open(authentication_file, 'rb') as f:
                 token = f.read().strip()
 
--- a/security/nss/automation/taskcluster/windows/setup.sh
+++ b/security/nss/automation/taskcluster/windows/setup.sh
@@ -18,9 +18,9 @@ hg_clone() {
         sleep $i
         hg clone -r "$rev" "$repo" "$dir" && return
         rm -rf "$dir"
     done
     exit 1
 }
 
 hg_clone https://hg.mozilla.org/build/tools tools default
-tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://api.pub.build.mozilla.org/tooltool/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache
+tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://tooltool.mozilla-releng.net/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache
--- a/taskcluster/docker/centos6-build/system-setup.sh
+++ b/taskcluster/docker/centos6-build/system-setup.sh
@@ -279,17 +279,17 @@ mkdir $BUILD
 # for the builds below, there's no sense using ccache
 export CCACHE_DISABLE=1
 
 cd $BUILD
 curl https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py > tooltool.py
 
 tooltool_fetch() {
     cat >manifest.tt
-    python $BUILD/tooltool.py fetch
+    python2.7 $BUILD/tooltool.py fetch
     rm manifest.tt
 }
 
 # For a few packges, we want to run the very latest, which is hard to find for
 # stable old CentOS 6.  Custom yum repostiories are cumbersome and can cause
 # unhappy failures when they contain multiple versions of the same package.  So
 # we either build from source or install an RPM from tooltool (the former being
 # the preferred solution for transparency).  Each of these source files was
--- a/taskcluster/docker/recipes/common.sh
+++ b/taskcluster/docker/recipes/common.sh
@@ -1,10 +1,10 @@
 #!/bin/bash
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 tooltool_fetch() {
     cat >manifest.tt
-    python /setup/tooltool.py fetch
+    python2.7 /setup/tooltool.py fetch
     rm manifest.tt
 }
--- a/taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh
+++ b/taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh
@@ -121,17 +121,17 @@ apt-get update
 # This allows ubuntu-desktop to be installed without human interaction
 export DEBIAN_FRONTEND=noninteractive
 apt-get install -y --force-yes ${apt_packages[@]}
 
 dpkg-reconfigure locales
 
 tooltool_fetch() {
     cat >manifest.tt
-    python /setup/tooltool.py fetch
+    python2.7 /setup/tooltool.py fetch
     rm manifest.tt
 }
 
 . /tmp/install-mercurial.sh
 
 # install peep
 tooltool_fetch <<'EOF'
 [
--- a/taskcluster/scripts/builder/build-haz-linux.sh
+++ b/taskcluster/scripts/builder/build-haz-linux.sh
@@ -48,17 +48,17 @@ GECKO_DIR=$( cd "$GECKO_DIR" && pwd )
 # Directory to populate with tooltool-installed tools
 export TOOLTOOL_DIR="$WORKSPACE"
 
 # Directory to hold the (useless) object files generated by the analysis.
 export MOZ_OBJDIR="$WORKSPACE/obj-analyzed"
 mkdir -p "$MOZ_OBJDIR"
 
 if [ -n "$DO_TOOLTOOL" ]; then
-  ( cd $TOOLTOOL_DIR; $GECKO_DIR/mach artifact toolchain -v --tooltool-url https://api.pub.build.mozilla.org/tooltool/ --tooltool-manifest $GECKO_DIR/$TOOLTOOL_MANIFEST --cache-dir $TOOLTOOL_CACHE${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}} )
+  ( cd $TOOLTOOL_DIR; $GECKO_DIR/mach artifact toolchain -v --tooltool-url https://tooltool.mozilla-releng.net/ --tooltool-manifest $GECKO_DIR/$TOOLTOOL_MANIFEST --cache-dir $TOOLTOOL_CACHE${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}} )
 fi
 
 export NO_MERCURIAL_SETUP_CHECK=1
 
 if [[ "$PROJECT" = "browser" ]]; then (
     cd "$WORKSPACE"
     set "$WORKSPACE"
     . setup-ccache.sh
--- a/taskcluster/scripts/builder/sm-tooltool-config.sh
+++ b/taskcluster/scripts/builder/sm-tooltool-config.sh
@@ -1,13 +1,13 @@
 #!/bin/bash
 
 set -xe
 
-: ${TOOLTOOL_SERVER:=https://api.pub.build.mozilla.org/tooltool/}
+: ${TOOLTOOL_SERVER:=https://tooltool.mozilla-releng.net/}
 : ${SPIDERMONKEY_VARIANT:=plain}
 : ${UPLOAD_DIR:=$HOME/artifacts/}
 : ${WORK:=$HOME/workspace}
 : ${SRCDIR:=$WORK/build/src}
 
 mkdir -p $WORK
 cd $WORK
 
--- a/testing/mozharness/configs/developer_config.py
+++ b/testing/mozharness/configs/developer_config.py
@@ -41,10 +41,10 @@ config = {
     "tooltool_cache": os.path.join(LOCAL_WORKDIR, "builds/tooltool_cache"),
     "tooltool_cache_path": os.path.join(LOCAL_WORKDIR, "builds/tooltool_cache"),
     "tooltool_authentication_file": os.path.join(LOCAL_WORKDIR, "relengapi.tok"),
 
     # VCS tools
     "gittool.py": 'http://hg.mozilla.org/build/puppet/raw-file/faaf5abd792e/modules/packages/files/gittool.py',
 
     # Android related
-    "host_utils_url": "https://api.pub.build.mozilla.org/tooltool/sha512/372c89f9dccaf5ee3b9d35fd1cfeb089e1e5db3ff1c04e35aa3adc8800bc61a2ae10e321f37ae7bab20b56e60941f91bb003bcb22035902a73d70872e7bd3282",
+    "host_utils_url": "https://tooltool.mozilla-releng.net/sha512/372c89f9dccaf5ee3b9d35fd1cfeb089e1e5db3ff1c04e35aa3adc8800bc61a2ae10e321f37ae7bab20b56e60941f91bb003bcb22035902a73d70872e7bd3282",
 }
--- a/testing/mozharness/configs/single_locale/win32.py
+++ b/testing/mozharness/configs/single_locale/win32.py
@@ -24,17 +24,17 @@ config = {
     },
     "ssh_key_dir": "~/.ssh",
     "log_name": "single_locale",
     "objdir": "obj-firefox",
     "js_src_dir": "js/src",
     "vcs_share_base": "c:/builds/hg-shared",
 
     # tooltool
-    'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
     'tooltool_script': [sys.executable,
                         'C:/mozilla-build/tooltool.py'],
     'tooltool_manifest_src': 'browser/config/tooltool-manifests/win32/l10n.manifest',
     # balrog credential file:
     'balrog_credentials_file': 'oauth.txt',
 
     # l10n
     "ignore_locales": ["en-US", "ja-JP-mac"],
--- a/testing/mozharness/configs/single_locale/win32_devedition.py
+++ b/testing/mozharness/configs/single_locale/win32_devedition.py
@@ -24,17 +24,17 @@ config = {
     },
     "ssh_key_dir": "~/.ssh",
     "log_name": "single_locale",
     "objdir": "obj-firefox",
     "js_src_dir": "js/src",
     "vcs_share_base": "c:/builds/hg-shared",
 
     # tooltool
-    'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
     'tooltool_script': [sys.executable,
                         'C:/mozilla-build/tooltool.py'],
     'tooltool_manifest_src': 'browser/config/tooltool-manifests/win32/l10n.manifest',
     # balrog credential file:
     'balrog_credentials_file': 'oauth.txt',
 
     # l10n
     "ignore_locales": ["en-US", "ja-JP-mac"],
--- a/testing/mozharness/configs/single_locale/win64.py
+++ b/testing/mozharness/configs/single_locale/win64.py
@@ -24,17 +24,17 @@ config = {
     },
     "ssh_key_dir": "~/.ssh",
     "log_name": "single_locale",
     "objdir": "obj-firefox",
     "js_src_dir": "js/src",
     "vcs_share_base": "c:/builds/hg-shared",
 
     # tooltool
-    'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
     'tooltool_script': [sys.executable,
                         'C:/mozilla-build/tooltool.py'],
     'tooltool_manifest_src': 'browser/config/tooltool-manifests/win64/l10n.manifest',
     # balrog credential file:
     'balrog_credentials_file': 'oauth.txt',
 
     # l10n
     "ignore_locales": ["en-US", "ja-JP-mac"],
--- a/testing/mozharness/configs/single_locale/win64_devedition.py
+++ b/testing/mozharness/configs/single_locale/win64_devedition.py
@@ -24,17 +24,17 @@ config = {
     },
     "ssh_key_dir": "~/.ssh",
     "log_name": "single_locale",
     "objdir": "obj-firefox",
     "js_src_dir": "js/src",
     "vcs_share_base": "c:/builds/hg-shared",
 
     # tooltool
-    'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
     'tooltool_script': [sys.executable,
                         'C:/mozilla-build/tooltool.py'],
     'tooltool_manifest_src': 'browser/config/tooltool-manifests/win64/l10n.manifest',
     # balrog credential file:
     'balrog_credentials_file': 'oauth.txt',
 
     # l10n
     "ignore_locales": ["en-US", "ja-JP-mac"],
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -76,17 +76,17 @@ class CodeCoverageMixin(object):
 
         # Create the grcov directory, get the tooltool manifest, and finally
         # download and unpack the grcov binary.
         self.grcov_dir = tempfile.mkdtemp()
         manifest = os.path.join(dirs.get('abs_test_install_dir', os.path.join(dirs['abs_work_dir'], 'tests')), \
             'config/tooltool-manifests/linux64/ccov.manifest')
 
         tooltool_path = self._fetch_tooltool_py()
-        cmd = [tooltool_path, '--url', 'https://api.pub.build.mozilla.org/tooltool/', 'fetch', \
+        cmd = [tooltool_path, '--url', 'https://tooltool.mozilla-releng.net/', 'fetch', \
             '-m', manifest, '-o', '-c', '/home/worker/tooltool-cache']
         self.run_command(cmd, cwd=self.grcov_dir)
         self.run_command(['tar', '-jxvf', os.path.join(self.grcov_dir, 'grcov-linux-standalone-x86_64.tar.bz2'), \
             '-C', self.grcov_dir], cwd=self.grcov_dir)
 
     @PostScriptAction('run-tests')
     def _package_coverage_data(self, action, success=None):
         if not self.code_coverage_enabled:
--- a/testing/mozharness/mozharness/mozilla/tooltool.py
+++ b/testing/mozharness/mozharness/mozilla/tooltool.py
@@ -10,17 +10,17 @@ TooltoolErrorList = PythonErrorList + [{
     'substr': 'ERROR - ', 'level': ERROR
 }]
 
 
 TOOLTOOL_PY_URL = \
     "https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py"
 
 TOOLTOOL_SERVERS = [
-    'https://api.pub.build.mozilla.org/tooltool/',
+    'https://tooltool.mozilla-releng.net/',
 ]
 
 
 class TooltoolMixin(object):
     """Mixin class for handling tooltool manifests.
     To use a tooltool server other than the Mozilla server, override
     config['tooltool_servers'].  To specify a different authentication
     file than that used in releng automation,override
--- a/testing/mozharness/scripts/fx_desktop_build.py
+++ b/testing/mozharness/scripts/fx_desktop_build.py
@@ -61,17 +61,17 @@ class FxDesktopBuild(BuildScript, TryToo
                 'taskcluster_credentials_file': 'oauth.txt',
                 'periodic_clobber': 168,
                 # hg tool stuff
                 "tools_repo": "https://hg.mozilla.org/build/tools",
                 # Seed all clones with mozilla-unified. This ensures subsequent
                 # jobs have a minimal `hg pull`.
                 "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
                 "repo_base": "https://hg.mozilla.org",
-                'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+                'tooltool_url': 'https://tooltool.mozilla-releng.net/',
                 "graph_selector": "/server/collect.cgi",
                 # only used for make uploadsymbols
                 'old_packages': [
                     "%(objdir)s/dist/firefox-*",
                     "%(objdir)s/dist/fennec*",
                     "%(objdir)s/dist/seamonkey*",
                     "%(objdir)s/dist/thunderbird*",
                     "%(objdir)s/dist/install/sea/*.exe"
--- a/tools/mach_commands.py
+++ b/tools/mach_commands.py
@@ -225,17 +225,17 @@ class FormatProvider(MachCommandBase):
             "Linux": "e6da4f6df074bfb15caefcf7767eb5670c02bb4768ba86ae4ab6b35235b53db012900a4f9e9a950ee140158a19532a71f21b986f511826bebc16f2ef83984e57",  # noqa: E501
             "Darwin": "18000940a11e5ab0c1fe950d4360292216c8e963dd708679c4c5fb8cc845f5919cef3f58a7e092555b8ea6b8d8a809d66153ea6d1e7c226a2c4f2b0b7ad1b2f3",  # noqa: E501
             "python_script": "34b6934a48a263ea3f88d48c2981d61ae6698823cfa689b9b0c8a607c224437ca0b9fdd434d260bd790d52a98455e2c2e2c745490d327ba84b4e22b7bb55b757",  # noqa: E501
         }
 
         target = os.path.join(self._mach_context.state_dir, os.path.basename(root))
 
         if not os.path.exists(target):
-            tooltool_url = "https://api.pub.build.mozilla.org/tooltool/sha512/"
+            tooltool_url = "https://tooltool.mozilla-releng.net/sha512/"
             if self.prompt and raw_input("Download clang-format executables from {0} (yN)? ".format(tooltool_url)).lower() != 'y':  # noqa: E501,F821
                 print("Download aborted.")
                 return None
             self.prompt = False
             plat = platform.system()
             if python_script:
                 # We want to download the python script (clang-format-diff)
                 dl = bin_sha["python_script"]