--- a/misc.py
+++ b/misc.py
@@ -1996,16 +1996,19 @@ def generateBranchObjects(config, name,
config.get('enable_hsts_update', False) or \
config.get('enable_hpkp_update', False):
periodicFileUpdateBuilder = generatePeriodicFileUpdateBuilder(
config, name, platform, pf['base_name'], pf['slaves'])
branchObjects['builders'].append(periodicFileUpdateBuilder)
# -- end of per-platform loop --
+ # Make sure builders have the right properties
+ addBuilderProperties(branchObjects['builders'])
+
return branchObjects
def _makeGenerateMozharnessTalosBuilderArgs(suite, talos_branch, platform,
factory_kwargs, branch_config, platform_config):
mh_conf = platform_config['mozharness_config']
extra_args = []
@@ -2212,17 +2215,17 @@ def generateTalosBranchObjects(branch, b
pgo_factory = generateMozharnessTalosBuilder(**args)
properties['script_repo_revision'] = branch_config['mozharness_tag']
properties['repo_path'] = branch_config['repo_path']
pgo_builder = {
'name': "%s %s pgo talos %s" % (platform_name, branch, suite),
'slavenames': platform_config[slave_platform]['slaves'],
'builddir': builddir + '-pgo',
- 'slavebuilddir': slavebuilddir + '-pgo',
+ 'slavebuilddir': slavebuilddir,
'factory': pgo_factory,
'category': branch,
'properties': properties,
'env': MozillaEnvironments[platform_config['env_name']],
}
if not merge:
nomergeBuilders.add(pgo_builder['name'])
@@ -2492,16 +2495,19 @@ def generateTalosBranchObjects(branch, b
)
schedulers.append(s)
return schedulers
# Create talos schedulers
branchObjects['schedulers'].extend(makeTalosScheduler(talos_builders, False))
branchObjects['schedulers'].extend(makeTalosScheduler(talos_pgo_builders, True))
+ # Make sure builders have the right properties
+ addBuilderProperties(branchObjects['builders'])
+
return branchObjects
def mirrorAndBundleArgs(config):
args = []
mirrors = None
if config.get('base_mirror_urls'):
mirrors = ["%s/%s" % (url, config['repo_path'])
@@ -2945,16 +2951,17 @@ def mh_l10n_builddir_from_builder_name(b
# replaces / with _
return b_dir.replace('/', '_')
def mh_l10n_scheduler_name(config, platform):
pf = config['platforms'][platform]
return '%s nightly l10n' % (pf['base_name'])
+
def mh_l10n_builder_names(config, platform, branch, is_nightly):
# let's check if we need to create builders for this config/platform
names = []
pf = config['platforms'][platform]
product_name = pf['product_name']
name = '%s %s %s l10n' % (product_name, branch, platform)
name = name.capitalize()
if is_nightly:
@@ -2963,8 +2970,38 @@ def mh_l10n_builder_names(config, platfo
l10n_chunks = repacks['l10n_chunks']
for chunk in range(1, l10n_chunks + 1):
builder_name = "%s-%s" % (name, chunk)
names.append(builder_name)
return names
+def addBuilderProperties(builders):
+ for b in builders:
+ if not isinstance(b['factory'], ScriptFactory):
+ continue
+
+ # TODO: do the same for 'master' property?
+ if 'basedir' in b['properties']:
+ continue
+
+ if 'slavebuilddir' in b:
+ slavebuilddir = b['slavebuilddir']
+ else:
+ slavebuilddir = b['builddir']
+
+ platform = b['properties']['platform']
+
+ if platform.startswith('win') or platform.startswith('xp-'):
+ # On Windows, test slaves use C:\slave\test, but build slaves
+ # use /c/builds/moz2_slave
+ if slavebuilddir == 'test': # TODO: This check is too fragile
+ rootdir = r'C:\slave'
+ basedir = '%s\%s' % (rootdir, slavebuilddir)
+ else:
+ rootdir = '/c/builds/moz2_slave'
+ basedir = '%s/%s' % (rootdir, slavebuilddir)
+ else:
+ rootdir = '/builds/slave'
+ basedir = '%s/%s' % (rootdir, slavebuilddir)
+
+ b['properties']['basedir'] = basedir
--- a/process/factory.py
+++ b/process/factory.py
@@ -4409,48 +4409,31 @@ class ScriptFactory(RequestSortingBuildF
if platform and 'win' in platform:
self.get_basedir_cmd = ['cd']
self.addStep(SetBuildProperty(
property_name='master',
value=lambda b: b.builder.botmaster.parent.buildbotURL
))
- self.addStep(SetProperty(
- name='get_basedir',
- property='basedir',
- command=self.get_basedir_cmd,
- workdir='.',
- haltOnFailure=True,
- ))
self.env['PROPERTIES_FILE'] = WithProperties(
'%(basedir)s/' + properties_file)
self.addStep(JSONPropertiesDownload(
name="download_props",
slavedest=properties_file,
workdir="."
))
if extra_data:
self.addStep(JSONStringDownload(
extra_data,
name="download_extra",
slavedest="data.json",
workdir="."
))
self.env['EXTRA_DATA'] = WithProperties('%(basedir)s/data.json')
- self.addStep(ShellCommand(
- name="clobber_properties",
- command=['rm', '-rf', 'properties'],
- workdir=".",
- ))
- self.addStep(SetBuildProperty(
- property_name='script_repo_url',
- value=scriptRepo,
- ))
- script_repo_url = WithProperties('%(script_repo_url)s')
if relengapi_archiver_repo_path:
if relengapi_archiver_release_tag:
archiver_revision = "--tag %s " % relengapi_archiver_release_tag
script_repo_revision = relengapi_archiver_release_tag
else:
archiver_revision = "--rev %s " % (relengapi_archiver_rev or '%(revision)s',)
script_repo_revision = "%s" % (relengapi_archiver_rev or '%(revision)s',)
@@ -4469,17 +4452,17 @@ class ScriptFactory(RequestSortingBuildF
'https://hg.mozilla.org/build/tools/raw-file/default/buildfarm/utils/archiver_client.py'],
haltOnFailure=True,
workdir=".",
))
archiver_client_path = 'archiver_client.py'
self.addStep(ShellCommand(
name="clobber_scripts",
- command=['rm', '-rf', 'scripts'],
+ command=['rm', '-rf', 'scripts', 'properties'],
workdir=".",
haltOnFailure=True,
log_eval_func=rc_eval_func({0: SUCCESS, None: RETRY}),
))
self.addStep(ShellCommand(
name="download_and_extract_scripts_archive",
command=['bash', '-c',
WithProperties(
@@ -4492,128 +4475,40 @@ class ScriptFactory(RequestSortingBuildF
log_eval_func=rc_eval_func({0: SUCCESS, None: EXCEPTION}),
haltOnFailure=True,
workdir=".",
))
if scriptName.startswith('/'):
script_path = scriptName
else:
script_path = 'scripts/%s' % scriptName
- self.addStep(SetProperty(
+ self.addStep(SetBuildProperty(
name='get_script_repo_revision',
- property='script_repo_revision',
- command=['echo', WithProperties(script_repo_revision)],
- workdir=".",
+ property_name='script_repo_revision',
+ value=lambda b: b.getProperties().render(WithProperties(script_repo_revision)),
haltOnFailure=False,
))
elif self.script_repo_cache:
- # all slaves bar win tests have a copy of hgtool on their path.
- # However, let's use runner's checkout version like we do for
- # script repo
- assert self.tools_repo_cache
- # ScriptFactory adds the props file into its env but we don't
- # want to pass that to the hgtool call because hgtool will assume
- # things like ['sourcestamp']['branch'] should be our branch
- # that script_repo pulls from
- hgtool_path = \
- os.path.join(self.tools_repo_cache,
- 'buildfarm',
- 'utils',
- 'hgtool.py')
- repository_manifest_path = \
- os.path.join(self.tools_repo_cache,
- 'buildfarm',
- 'utils',
- 'repository_manifest.py')
-
- if script_repo_manifest:
- self.addStep(SetProperty(
- name="set_script_repo_url_and_script_repo_revision",
- extract_fn=extractProperties,
- command=['bash', '-c',
- WithProperties(
- 'python %s ' % repository_manifest_path +
- '--default-repo %s ' % scriptRepo +
- '--default-revision %(script_repo_revision:-default)s ' +
- '--default-checkout %s ' % self.script_repo_cache +
- '--checkout %(basedir)s/scripts ' +
- '--manifest-url %s' % script_repo_manifest)],
- log_eval_func=rc_eval_func({0: SUCCESS, None: EXCEPTION}),
- haltOnFailure=True,
- ))
- else:
- self.addStep(SetBuildProperty(
- property_name='script_repo_checkout',
- value=self.script_repo_cache,
- ))
-
- hg_script_repo_env = self.env.copy()
- hg_script_repo_env.pop('PROPERTIES_FILE', None)
-
- hgtool_cmd = [
- 'python', hgtool_path, '--purge',
- '-r', WithProperties('%(script_repo_revision:-default)s'),
- WithProperties('%(script_repo_url)s'),
- WithProperties('%(script_repo_checkout)s'),
- ]
-
- self.addStep(ShellCommand(
- name='update_script_repo_cache',
- command=hgtool_cmd,
- env=hg_script_repo_env,
- haltOnFailure=True,
- flunkOnFailure=True,
- ))
- self.addStep(SetProperty(
- name='get_script_repo_revision',
- property='script_repo_revision',
- command=[hg_bin, 'id', '-i'],
- workdir=WithProperties('%(script_repo_checkout)s'),
- haltOnFailure=False,
- ))
- script_path = WithProperties('%(script_repo_checkout)s/' + scriptName)
+ # This code path is no longer used
+ assert False, 'script_repo_cache is not used any more on its own'
else:
# fall back to legacy clobbering + cloning script repo
if script_repo_manifest:
- # By setting scriptRepoManifest we indicate that we don't
- # want to use scriptRepo but we want to let the manifest associated
- # to set the repo to checkout and which revision/branch to update to
- # If the repo specified in the manifest matches scriptRepo we will
- # use the cached version if available (i.e. script_repo_cache has
- # been set)
- self.addStep(ShellCommand(
- command=['bash', '-c',
- WithProperties('wget -Orepository_manifest.py ' + \
- '--no-check-certificate --tries=10 --waitretry=3 ' + \
- 'http://hg.mozilla.org/build/tools/raw-file/default/buildfarm/utils/repository_manifest.py')],
- haltOnFailure=True,
- ))
- self.addStep(SetProperty(
- name="set_script_repo_url_and_script_repo_revision",
- extract_fn=extractProperties,
- command=['bash', '-c',
- WithProperties(
- 'python repository_manifest.py ' +
- '--default-repo %s ' % scriptRepo +
- '--default-revision %(script_repo_revision)s ' +
- '--manifest-url %s' % script_repo_manifest)],
- log_eval_func=rc_eval_func({0: SUCCESS, None: EXCEPTION}),
- haltOnFailure=True,
- ))
+ assert False, 'legacy script_repo_manifest unsupported now'
self.addStep(ShellCommand(
name="clobber_scripts",
- command=['rm', '-rf', 'scripts'],
+ command=['rm', '-rf', 'scripts', 'properties'],
workdir=".",
haltOnFailure=True,
log_eval_func=rc_eval_func({0: SUCCESS, None: RETRY}),
))
self.addStep(MercurialCloneCommand(
name="clone_scripts",
- command=[hg_bin, 'clone', script_repo_url, 'scripts'],
+ command=[hg_bin, 'clone', scriptRepo, 'scripts'],
workdir=".",
haltOnFailure=True,
retry=False,
log_eval_func=rc_eval_func({0: SUCCESS, None: RETRY}),
))
self.addStep(ShellCommand(
name="update_scripts",
command=[hg_bin, 'update', '-C', '-r',
@@ -4808,21 +4703,15 @@ class SigningScriptFactory(ScriptFactory
))
else:
self.addStep(SetProperty(
name='set_toolsdir',
command=self.get_basedir_cmd,
property='toolsdir',
workdir='scripts',
))
- self.addStep(SetProperty(
- name='set_basedir',
- command=self.get_basedir_cmd,
- property='basedir',
- workdir='.',
- ))
signing_env = self.env.copy()
signing_env['MOZ_SIGN_CMD'] = WithProperties(get_signing_cmd(
self.signingServers, self.env.get('PYTHON26')))
signing_env['MOZ_SIGNING_SERVERS'] = ",".join(
"%s:%s" % (":".join(s[3]), s[0]) for s in self.signingServers)
ScriptFactory.runScript(self, env=signing_env)