Bug 1347579 - Add mozharness script (and docker build shell script) to allow running mach repackage. r=aki draft
authorJustin Wood <Callek@gmail.com>
Tue, 28 Mar 2017 10:41:38 -0400
changeset 552779 054219511419b8bf44b1f57a8d834a12c13710e3
parent 552445 d4af7ec6cfcd9b81cd1f433a00b412de61e95b62
child 552780 f067fa3b33157a1afc0906412fb7b4f7e5fe24f3
push id51458
push userCallek@gmail.com
push dateWed, 29 Mar 2017 01:24:52 +0000
reviewersaki
bugs1347579, 1347576, 1346015, 1318505
milestone55.0a1
Bug 1347579 - Add mozharness script (and docker build shell script) to allow running mach repackage. r=aki (For "Integrate and fully support OSX Signing in taskcluster") Written as a mozharness script rather than using bare ./mach command because we need to download the upstream artifact and because we need to download artifacts from tooltool to do the packing back into a .dmg. Future ideal would be to get rid of the mozharness script and use JUST ./mach. This is using the ./mach repackage code being created in Bug 1347576. Taking a signed tarball from a dmg supported with Bug 1346015, and the taskgraph work to schedule this is in Bug 1318505. MozReview-Commit-ID: rv9l285HKC
taskcluster/scripts/builder/repackage.sh
testing/mozharness/configs/repackage/osx_signed.py
testing/mozharness/scripts/repackage.py
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/builder/repackage.sh
@@ -0,0 +1,93 @@
+#! /bin/bash -vex
+
+set -x -e
+
+echo "running as" $(id)
+
+. /home/worker/scripts/xvfb.sh
+
+####
+# Taskcluster friendly wrapper for performing fx desktop builds via mozharness.
+####
+
+# Inputs, with defaults
+
+: MOZHARNESS_SCRIPT             ${MOZHARNESS_SCRIPT}
+: MOZHARNESS_CONFIG             ${MOZHARNESS_CONFIG}
+: MOZHARNESS_ACTIONS            ${MOZHARNESS_ACTIONS}
+: MOZHARNESS_OPTIONS            ${MOZHARNESS_OPTIONS}
+
+: TOOLTOOL_CACHE                ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache}
+
+: WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
+
+set -v
+
+fail() {
+    echo # make sure error message is on a new line
+    echo "[build-linux.sh:error]" "${@}"
+    exit 1
+}
+
+export MOZ_CRASHREPORTER_NO_REPORT=1
+export MOZ_OBJDIR=obj-firefox
+export TINDERBOX_OUTPUT=1
+
+# use "simple" package names so that they can be hard-coded in the task's
+# extras.locations
+export MOZ_SIMPLE_PACKAGE_NAME=target
+
+# test required parameters are supplied
+if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
+if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi
+
+cleanup() {
+    local rv=$?
+    cleanup_xvfb
+    exit $rv
+}
+trap cleanup EXIT INT
+
+# set up mozharness configuration, via command line, env, etc.
+
+debug_flag=""
+if [ 0$DEBUG -ne 0 ]; then
+  debug_flag='--debug'
+fi
+
+# $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the
+# cache.  However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be
+# entirely effective.
+export TOOLTOOL_CACHE
+
+# support multiple, space delimited, config files
+config_cmds=""
+for cfg in $MOZHARNESS_CONFIG; do
+  config_cmds="${config_cmds} --config ${cfg}"
+done
+
+# if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions
+# in the mozharness configuration)
+if [ -n "$MOZHARNESS_ACTIONS" ]; then
+    actions=""
+    for action in $MOZHARNESS_ACTIONS; do
+        actions="$actions --$action"
+    done
+fi
+
+# if MOZHARNESS_OPTIONS is given, append them to mozharness command line run
+# e.g. enable-pgo
+if [ -n "$MOZHARNESS_OPTIONS" ]; then
+    options=""
+    for option in $MOZHARNESS_OPTIONS; do
+        options="$options --$option"
+    done
+fi
+
+cd /home/worker
+
+python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \
+  $actions \
+  $options \
+  --log-level=debug \
+  --work-dir=$WORKSPACE/build \
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/repackage/osx_signed.py
@@ -0,0 +1,19 @@
+import os
+
+config = {
+    "input_filename": "target.tar.gz",
+    "output_filename": "target.dmg",
+    "input_home": "/home/worker/workspace/inputs",
+
+    # ToolTool
+    "tooltool_manifest_src": 'browser/config/tooltool-manifests/macosx64/cross-releng.manifest',
+    "tooltool_url": 'http://relengapi/tooltool/',
+    "tooltool_bootstrap": "setup.sh",
+    'tooltool_script': ["/builds/tooltool.py"],
+    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+
+    # Tools to pack a DMG
+    "hfs_tool": "dmg/hfsplus",
+    "dmg_tool": "dmg/dmg",
+    "mkfshfs_tool": "hfsplus-tools/newfs_hfs",
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/repackage.py
@@ -0,0 +1,122 @@
+import os
+import sys
+
+sys.path.insert(1, os.path.dirname(sys.path[0]))  # noqa - don't warn about imports
+
+from mozharness.base.script import BaseScript
+from mozharness.mozilla.mock import ERROR_MSGS
+
+
+class Repackage(BaseScript):
+
+    config_options = [[
+        ['--signed-input', ],
+        {"action": "store",
+         "dest": "signed_input",
+         "type": "string",
+         "default": os.environ.get('SIGNED_INPUT'),
+         "help": "Specify the signed input (url)"}
+    ], [
+        ['--output-file', ],
+        {"action": "store",
+         "dest": "output_file",
+         "type": "string",
+         "help": "Specify the output filename"}
+    ]]
+
+    def __init__(self, require_config_file=False):
+        script_kwargs = {
+            'all_actions': [
+                "download_input",
+                "setup",
+                "repackage",
+            ],
+        }
+        BaseScript.__init__(
+            self,
+            config_options=self.config_options,
+            require_config_file=require_config_file,
+            **script_kwargs
+        )
+
+        # Assert we have it either passed in or in environment
+        assert self.config.get('signed_input'), \
+            "Must pass --signed-input or be set in the environment as SIGNED_INPUT"
+
+    def download_input(self):
+        config = self.config
+
+        url = config['signed_input']
+        status = self.download_file(url=url,
+                                    file_name=config['input_filename'],
+                                    parent_dir=config['input_home'])
+        if not status:
+            self.fatal("Unable to fetch signed input from %s" % config['signed_input'])
+
+    def setup(self):
+        self._run_tooltool()
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(Repackage, self).query_abs_dirs()
+        for directory in abs_dirs:
+            value = abs_dirs[directory]
+            abs_dirs[directory] = value
+        dirs = {}
+        dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools')
+        dirs['abs_mozilla_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'src')
+        for key in dirs.keys():
+            if key not in abs_dirs:
+                abs_dirs[key] = dirs[key]
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    def repackage(self):
+        config = self.config
+        dirs = self.query_abs_dirs()
+        python = self.query_exe('python2.7')
+        infile = os.path.join(config['input_home'], config['input_filename'])
+        outfile = os.path.join(dirs['abs_upload_dir'], config['output_filename'])
+        env = {
+            'HFS_TOOL': os.path.join(dirs['abs_mozilla_dir'], config['hfs_tool']),
+            'DMG_TOOL': os.path.join(dirs['abs_mozilla_dir'], config['dmg_tool']),
+            'MKFSHFS': os.path.join(dirs['abs_mozilla_dir'], config['mkfshfs_tool'])
+        }
+        command = [python, 'mach', '--log-no-times', 'repackage',
+                   '--input', infile,
+                   '--output', outfile]
+        return self.run_command(
+            command=command,
+            cwd=dirs['abs_mozilla_dir'],
+            partial_env=env,
+            halt_on_failure=True,
+        )
+
+    def _run_tooltool(self):
+        config = self.config
+        dirs = self.query_abs_dirs()
+        if not config.get('tooltool_manifest_src'):
+            return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
+        fetch_script_path = os.path.join(dirs['abs_tools_dir'],
+                                         'scripts/tooltool/tooltool_wrapper.sh')
+        tooltool_manifest_path = os.path.join(dirs['abs_mozilla_dir'],
+                                              config['tooltool_manifest_src'])
+        cmd = [
+            'sh',
+            fetch_script_path,
+            tooltool_manifest_path,
+            config['tooltool_url'],
+            config['tooltool_bootstrap'],
+        ]
+        cmd.extend(config['tooltool_script'])
+        cache = config.get('tooltool_cache')
+        if cache:
+            cmd.extend(['-c', cache])
+        self.info(str(cmd))
+        self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)
+
+
+if __name__ == '__main__':
+    repack = Repackage()
+    repack.run_and_exit()