Bug 1347579 - Add mozharness script (and docker build shell script) to allow running mach repackage. r=aki
(For "Integrate and fully support OSX Signing in taskcluster")
Written as a mozharness script rather than using bare ./mach command because we need to download the upstream artifact
and because we need to download artifacts from tooltool to do the packing back into a .dmg. Future ideal would be to get
rid of the mozharness script and use JUST ./mach.
This is using the ./mach repackage code being created in
Bug 1347576. Taking a signed tarball from a dmg supported with
Bug 1346015, and the taskgraph work to schedule this is in
Bug 1318505.
MozReview-Commit-ID: rv9l285HKC
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/builder/repackage.sh
@@ -0,0 +1,93 @@
+#! /bin/bash -vex
+
+set -x -e
+
+echo "running as" $(id)
+
+. /home/worker/scripts/xvfb.sh
+
+####
+# Taskcluster friendly wrapper for performing fx desktop builds via mozharness.
+####
+
+# Inputs, with defaults
+
+: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT}
+: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG}
+: MOZHARNESS_ACTIONS ${MOZHARNESS_ACTIONS}
+: MOZHARNESS_OPTIONS ${MOZHARNESS_OPTIONS}
+
+: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache}
+
+: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
+
+set -v
+
+fail() {
+ echo # make sure error message is on a new line
+ echo "[build-linux.sh:error]" "${@}"
+ exit 1
+}
+
+export MOZ_CRASHREPORTER_NO_REPORT=1
+export MOZ_OBJDIR=obj-firefox
+export TINDERBOX_OUTPUT=1
+
+# use "simple" package names so that they can be hard-coded in the task's
+# extras.locations
+export MOZ_SIMPLE_PACKAGE_NAME=target
+
+# test required parameters are supplied
+if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
+if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi
+
+cleanup() {
+ local rv=$?
+ cleanup_xvfb
+ exit $rv
+}
+trap cleanup EXIT INT
+
+# set up mozharness configuration, via command line, env, etc.
+
+debug_flag=""
+if [ 0$DEBUG -ne 0 ]; then
+ debug_flag='--debug'
+fi
+
+# $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the
+# cache. However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be
+# entirely effective.
+export TOOLTOOL_CACHE
+
+# support multiple, space delimited, config files
+config_cmds=""
+for cfg in $MOZHARNESS_CONFIG; do
+ config_cmds="${config_cmds} --config ${cfg}"
+done
+
+# if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions
+# in the mozharness configuration)
+if [ -n "$MOZHARNESS_ACTIONS" ]; then
+ actions=""
+ for action in $MOZHARNESS_ACTIONS; do
+ actions="$actions --$action"
+ done
+fi
+
+# if MOZHARNESS_OPTIONS is given, append them to mozharness command line run
+# e.g. enable-pgo
+if [ -n "$MOZHARNESS_OPTIONS" ]; then
+ options=""
+ for option in $MOZHARNESS_OPTIONS; do
+ options="$options --$option"
+ done
+fi
+
+cd /home/worker
+
+python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \
+ $actions \
+ $options \
+ --log-level=debug \
+ --work-dir=$WORKSPACE/build \
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/repackage/osx_signed.py
@@ -0,0 +1,19 @@
+import os
+
+config = {
+ "input_filename": "target.tar.gz",
+ "output_filename": "target.dmg",
+ "input_home": "/home/worker/workspace/inputs",
+
+ # ToolTool
+ "tooltool_manifest_src": 'browser/config/tooltool-manifests/macosx64/cross-releng.manifest',
+ "tooltool_url": 'http://relengapi/tooltool/',
+ "tooltool_bootstrap": "setup.sh",
+ 'tooltool_script': ["/builds/tooltool.py"],
+ 'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+
+ # Tools to pack a DMG
+ "hfs_tool": "dmg/hfsplus",
+ "dmg_tool": "dmg/dmg",
+ "mkfshfs_tool": "hfsplus-tools/newfs_hfs",
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/repackage.py
@@ -0,0 +1,122 @@
+import os
+import sys
+
+sys.path.insert(1, os.path.dirname(sys.path[0])) # noqa - don't warn about imports
+
+from mozharness.base.script import BaseScript
+from mozharness.mozilla.mock import ERROR_MSGS
+
+
+class Repackage(BaseScript):
+
+ config_options = [[
+ ['--signed-input', ],
+ {"action": "store",
+ "dest": "signed_input",
+ "type": "string",
+ "default": os.environ.get('SIGNED_INPUT'),
+ "help": "Specify the signed input (url)"}
+ ], [
+ ['--output-file', ],
+ {"action": "store",
+ "dest": "output_file",
+ "type": "string",
+ "help": "Specify the output filename"}
+ ]]
+
+ def __init__(self, require_config_file=False):
+ script_kwargs = {
+ 'all_actions': [
+ "download_input",
+ "setup",
+ "repackage",
+ ],
+ }
+ BaseScript.__init__(
+ self,
+ config_options=self.config_options,
+ require_config_file=require_config_file,
+ **script_kwargs
+ )
+
+ # Assert we have it either passed in or in environment
+ assert self.config.get('signed_input'), \
+ "Must pass --signed-input or be set in the environment as SIGNED_INPUT"
+
+ def download_input(self):
+ config = self.config
+
+ url = config['signed_input']
+ status = self.download_file(url=url,
+ file_name=config['input_filename'],
+ parent_dir=config['input_home'])
+ if not status:
+ self.fatal("Unable to fetch signed input from %s" % config['signed_input'])
+
+ def setup(self):
+ self._run_tooltool()
+
+ def query_abs_dirs(self):
+ if self.abs_dirs:
+ return self.abs_dirs
+ abs_dirs = super(Repackage, self).query_abs_dirs()
+ for directory in abs_dirs:
+ value = abs_dirs[directory]
+ abs_dirs[directory] = value
+ dirs = {}
+ dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools')
+ dirs['abs_mozilla_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'src')
+ for key in dirs.keys():
+ if key not in abs_dirs:
+ abs_dirs[key] = dirs[key]
+ self.abs_dirs = abs_dirs
+ return self.abs_dirs
+
+ def repackage(self):
+ config = self.config
+ dirs = self.query_abs_dirs()
+ python = self.query_exe('python2.7')
+ infile = os.path.join(config['input_home'], config['input_filename'])
+ outfile = os.path.join(dirs['abs_upload_dir'], config['output_filename'])
+ env = {
+ 'HFS_TOOL': os.path.join(dirs['abs_mozilla_dir'], config['hfs_tool']),
+ 'DMG_TOOL': os.path.join(dirs['abs_mozilla_dir'], config['dmg_tool']),
+ 'MKFSHFS': os.path.join(dirs['abs_mozilla_dir'], config['mkfshfs_tool'])
+ }
+ command = [python, 'mach', '--log-no-times', 'repackage',
+ '--input', infile,
+ '--output', outfile]
+ return self.run_command(
+ command=command,
+ cwd=dirs['abs_mozilla_dir'],
+ partial_env=env,
+ halt_on_failure=True,
+ )
+
+ def _run_tooltool(self):
+ config = self.config
+ dirs = self.query_abs_dirs()
+ if not config.get('tooltool_manifest_src'):
+ return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
+ fetch_script_path = os.path.join(dirs['abs_tools_dir'],
+ 'scripts/tooltool/tooltool_wrapper.sh')
+ tooltool_manifest_path = os.path.join(dirs['abs_mozilla_dir'],
+ config['tooltool_manifest_src'])
+ cmd = [
+ 'sh',
+ fetch_script_path,
+ tooltool_manifest_path,
+ config['tooltool_url'],
+ config['tooltool_bootstrap'],
+ ]
+ cmd.extend(config['tooltool_script'])
+ cache = config.get('tooltool_cache')
+ if cache:
+ cmd.extend(['-c', cache])
+ self.info(str(cmd))
+ self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)
+
+
+if __name__ == '__main__':
+ repack = Repackage()
+ repack.run_and_exit()