--- a/misc.py
+++ b/misc.py
@@ -562,19 +562,17 @@ def generateTestBuilder(config, branch_n
else:
slavenames = slaves
if not category:
category = branch_name
branchProperty = branch_name
properties = {'branch': branchProperty, 'platform': platform,
'slavebuilddir': 'test', 'stage_platform': stagePlatform,
'product': stageProduct, 'repo_path': config['repo_path'],
- 'moz_repo_path': config.get('moz_repo_path', ''),
- 'basedir': None, # TODO
- }
+ 'moz_repo_path': config.get('moz_repo_path', '')}
# suites is a dict!
if mozharness_suite_config is None:
mozharness_suite_config = {}
extra_args = []
if mozharness_suite_config.get('config_files'):
extra_args.extend(['--cfg', ','.join(mozharness_suite_config['config_files'])])
extra_args.extend(mozharness_suite_config.get('extra_args', suites.get('extra_args', [])))
if is_debug is True:
@@ -750,18 +748,16 @@ def generateDesktopMozharnessBuilders(na
mh_build_properties = {
# our buildbot master.cfg requires us to at least have
# these but mozharness doesn't need them
'branch': name,
'platform': platform,
'product': pf['stage_product'],
'repo_path': config['repo_path'],
'script_repo_revision': config["mozharness_tag"],
- 'toolsdir': None, # TODO
- 'basedir': None, # TODO
}
dep_signing_servers = secrets.get(pf.get('dep_signing_servers'))
nightly_signing_servers = secrets.get(pf.get('nightly_signing_servers'))
# grab the l10n schedulers that nightlies will trigger (if any)
triggered_nightly_schedulers = []
if config.get("enable_triggered_nightly_scheduler", True):
if is_l10n_with_mh(config, platform):
@@ -2000,21 +1996,18 @@ def generateBranchObjects(config, name,
config.get('enable_hsts_update', False) or \
config.get('enable_hpkp_update', False):
periodicFileUpdateBuilder = generatePeriodicFileUpdateBuilder(
config, name, platform, pf['base_name'], pf['slaves'])
branchObjects['builders'].append(periodicFileUpdateBuilder)
# -- end of per-platform loop --
- # Check that we have all our stuff
- for builder in branchObjects['builders']:
- if isinstance(builder['factory'], ScriptFactory):
- assert 'toolsdir' in builder['properties'], builder['name']
- assert 'basedir' in builder['properties'], builder['name']
+ # Sanity check builders
+ validateBuilders(branchObjects['builders'])
return branchObjects
def _makeGenerateMozharnessTalosBuilderArgs(suite, talos_branch, platform,
factory_kwargs, branch_config, platform_config):
mh_conf = platform_config['mozharness_config']
@@ -2173,17 +2166,16 @@ def generateTalosBranchObjects(branch, b
slavebuilddir = 'test'
properties = {
'branch': branchProperty,
'platform': slave_platform,
'stage_platform': stage_platform,
'product': stage_product,
'builddir': builddir,
'slavebuilddir': slavebuilddir,
- 'basedir': None, # TODO
}
assert branch_config.get('mozharness_talos', True) and platform_config[slave_platform].get('mozharness_talos', True)
args = _makeGenerateMozharnessTalosBuilderArgs(suite, opt_talos_branch, platform,
factory_kwargs, branch_config, platform_config)
factory = generateMozharnessTalosBuilder(**args)
properties['script_repo_revision'] = branch_config['mozharness_tag']
properties['repo_path'] = branch_config['repo_path']
@@ -2211,30 +2203,29 @@ def generateTalosBranchObjects(branch, b
if create_pgo_builders:
properties = {
'branch': branchProperty,
'platform': slave_platform,
'stage_platform': stage_platform + '-pgo',
'product': stage_product,
'builddir': builddir,
'slavebuilddir': slavebuilddir,
- 'basedir': None, # TODO
}
assert branch_config.get('mozharness_talos') and not platform_config.get('is_mobile')
args = _makeGenerateMozharnessTalosBuilderArgs(suite, talosBranch, platform,
factory_kwargs, branch_config, platform_config)
pgo_factory = generateMozharnessTalosBuilder(**args)
properties['script_repo_revision'] = branch_config['mozharness_tag']
properties['repo_path'] = branch_config['repo_path']
pgo_builder = {
'name': "%s %s pgo talos %s" % (platform_name, branch, suite),
'slavenames': platform_config[slave_platform]['slaves'],
'builddir': builddir + '-pgo',
- 'slavebuilddir': slavebuilddir + '-pgo',
+ 'slavebuilddir': slavebuilddir,
'factory': pgo_factory,
'category': branch,
'properties': properties,
'env': MozillaEnvironments[platform_config['env_name']],
}
if not merge:
nomergeBuilders.add(pgo_builder['name'])
@@ -2504,21 +2495,18 @@ def generateTalosBranchObjects(branch, b
)
schedulers.append(s)
return schedulers
# Create talos schedulers
branchObjects['schedulers'].extend(makeTalosScheduler(talos_builders, False))
branchObjects['schedulers'].extend(makeTalosScheduler(talos_pgo_builders, True))
- # Check that we have all our stuff
- for builder in branchObjects['builders']:
- if isinstance(builder['factory'], ScriptFactory):
- #assert 'toolsdir' in builder['properties'], builder['name']
- assert 'basedir' in builder['properties'], builder['name']
+ # Sanity check builders
+ validateBuilders(branchObjects['builders'])
return branchObjects
def mirrorAndBundleArgs(config):
args = []
mirrors = None
if config.get('base_mirror_urls'):
@@ -2963,16 +2951,17 @@ def mh_l10n_builddir_from_builder_name(b
# replaces / with _
return b_dir.replace('/', '_')
def mh_l10n_scheduler_name(config, platform):
pf = config['platforms'][platform]
return '%s nightly l10n' % (pf['base_name'])
+
def mh_l10n_builder_names(config, platform, branch, is_nightly):
# let's check if we need to create builders for this config/platform
names = []
pf = config['platforms'][platform]
product_name = pf['product_name']
name = '%s %s %s l10n' % (product_name, branch, platform)
name = name.capitalize()
if is_nightly:
@@ -2981,8 +2970,37 @@ def mh_l10n_builder_names(config, platfo
l10n_chunks = repacks['l10n_chunks']
for chunk in range(1, l10n_chunks + 1):
builder_name = "%s-%s" % (name, chunk)
names.append(builder_name)
return names
+def validateBuilders(builders):
+ for b in builders:
+ if isinstance(b['factory'], ScriptFactory):
+ # TODO: do the same for 'master' property?
+ if 'basedir' in b['properties']:
+ continue
+
+ if 'slavebuilddir' in b:
+ slavebuilddir = b['slavebuilddir']
+ else:
+ slavebuilddir = b['builddir']
+
+ platform = b['properties']['platform']
+
+ if platform.startswith('win') or platform.startswith('xp-'):
+ # On Windows, test slaves use C:\slave\test, but build slaves
+ # use /c/builds/moz2_slave
+ if slavebuilddir == 'test': # TODO: This check is too fragile
+ rootdir = r'C:\slave'
+ basedir = '%s\%s' % (rootdir, slavebuilddir)
+ else:
+ rootdir = '/c/builds/moz2_slave'
+ basedir = '%s/%s' % (rootdir, slavebuilddir)
+ else:
+ rootdir = '/builds/slave'
+ basedir = '%s/%s' % (rootdir, slavebuilddir)
+
+ b['properties']['basedir'] = basedir
+ assert 'basedir' in b['properties'], b['name']
--- a/process/factory.py
+++ b/process/factory.py
@@ -4388,29 +4388,33 @@ class ScriptFactory(RequestSortingBuildF
self.log_eval_func = log_eval_func
self.script_maxtime = script_maxtime
self.reboot_command = reboot_command
self.platform = platform
self.use_mock = use_mock
self.mock_target = mock_target
self.mock_packages = mock_packages
self.mock_copyin_files = mock_copyin_files
+ self.get_basedir_cmd = ['bash', '-c', 'pwd']
self.triggered_schedulers = triggered_schedulers
self.env = env.copy()
self.use_credentials_file = use_credentials_file
self.copy_properties = copy_properties or []
self.script_repo_cache = script_repo_cache
self.tools_repo_cache = tools_repo_cache
self.tooltool_manifest_src = tooltool_manifest_src
self.tooltool_url_list = tooltool_url_list or []
self.tooltool_script = tooltool_script or ['/tools/tooltool.py']
self.tooltool_bootstrap = tooltool_bootstrap
assert len(self.tooltool_url_list) <= 1, "multiple urls not currently supported by tooltool"
+ if platform and 'win' in platform:
+ self.get_basedir_cmd = ['cd']
+
self.addStep(SetBuildProperty(
property_name='master',
value=lambda b: b.builder.botmaster.parent.buildbotURL
))
self.env['PROPERTIES_FILE'] = WithProperties(
'%(basedir)s/' + properties_file)
self.addStep(JSONPropertiesDownload(
name="download_props",
@@ -4420,28 +4424,16 @@ class ScriptFactory(RequestSortingBuildF
if extra_data:
self.addStep(JSONStringDownload(
extra_data,
name="download_extra",
slavedest="data.json",
workdir="."
))
self.env['EXTRA_DATA'] = WithProperties('%(basedir)s/data.json')
- self.addStep(ShellCommand(
- name="clobber_properties",
- command=['rm', '-rf', 'properties'],
- workdir=".",
- ))
- # TODO: This should be set at reconfig time
- # TODO: Add checks that we have toolsdir, etc. as static properties
- self.addStep(SetBuildProperty(
- property_name='script_repo_url',
- value=scriptRepo,
- ))
- script_repo_url = WithProperties('%(script_repo_url)s')
if relengapi_archiver_repo_path:
if relengapi_archiver_release_tag:
archiver_revision = "--tag %s " % relengapi_archiver_release_tag
script_repo_revision = relengapi_archiver_release_tag
else:
archiver_revision = "--rev %s " % (relengapi_archiver_rev or '%(revision)s',)
script_repo_revision = "%s" % (relengapi_archiver_rev or '%(revision)s',)
@@ -4460,17 +4452,17 @@ class ScriptFactory(RequestSortingBuildF
'https://hg.mozilla.org/build/tools/raw-file/default/buildfarm/utils/archiver_client.py'],
haltOnFailure=True,
workdir=".",
))
archiver_client_path = 'archiver_client.py'
self.addStep(ShellCommand(
name="clobber_scripts",
- command=['rm', '-rf', 'scripts'],
+ command=['rm', '-rf', 'scripts', 'properties'],
workdir=".",
haltOnFailure=True,
log_eval_func=rc_eval_func({0: SUCCESS, None: RETRY}),
))
self.addStep(ShellCommand(
name="download_and_extract_scripts_archive",
command=['bash', '-c',
WithProperties(
@@ -4486,124 +4478,37 @@ class ScriptFactory(RequestSortingBuildF
))
if scriptName.startswith('/'):
script_path = scriptName
else:
script_path = 'scripts/%s' % scriptName
self.addStep(SetBuildProperty(
name='get_script_repo_revision',
property_name='script_repo_revision',
- value=lambda b: b.getProperty('script_repo_revision'),
+ value=lambda b: b.getProperties().render(WithProperties(script_repo_revision)),
+ haltOnFailure=False,
))
elif self.script_repo_cache:
- # all slaves bar win tests have a copy of hgtool on their path.
- # However, let's use runner's checkout version like we do for
- # script repo
- assert self.tools_repo_cache
- # ScriptFactory adds the props file into its env but we don't
- # want to pass that to the hgtool call because hgtool will assume
- # things like ['sourcestamp']['branch'] should be our branch
- # that script_repo pulls from
- hgtool_path = \
- os.path.join(self.tools_repo_cache,
- 'buildfarm',
- 'utils',
- 'hgtool.py')
- repository_manifest_path = \
- os.path.join(self.tools_repo_cache,
- 'buildfarm',
- 'utils',
- 'repository_manifest.py')
-
- if script_repo_manifest:
- self.addStep(SetProperty(
- name="set_script_repo_url_and_script_repo_revision",
- extract_fn=extractProperties,
- command=['bash', '-c',
- WithProperties(
- 'python %s ' % repository_manifest_path +
- '--default-repo %s ' % scriptRepo +
- '--default-revision %(script_repo_revision:-default)s ' +
- '--default-checkout %s ' % self.script_repo_cache +
- '--checkout %(basedir)s/scripts ' +
- '--manifest-url %s' % script_repo_manifest)],
- log_eval_func=rc_eval_func({0: SUCCESS, None: EXCEPTION}),
- haltOnFailure=True,
- ))
- else:
- # TODO: Set this statically
- self.addStep(SetBuildProperty(
- property_name='script_repo_checkout',
- value=self.script_repo_cache,
- ))
-
- hg_script_repo_env = self.env.copy()
- hg_script_repo_env.pop('PROPERTIES_FILE', None)
-
- hgtool_cmd = [
- 'python', hgtool_path, '--purge',
- '-r', WithProperties('%(script_repo_revision:-default)s'),
- WithProperties('%(script_repo_url)s'),
- WithProperties('%(script_repo_checkout)s'),
- ]
-
- self.addStep(ShellCommand(
- name='update_script_repo_cache',
- command=hgtool_cmd,
- env=hg_script_repo_env,
- haltOnFailure=True,
- flunkOnFailure=True,
- ))
- self.addStep(SetProperty(
- name='get_script_repo_revision',
- property='script_repo_revision',
- command=[hg_bin, 'id', '-i'],
- workdir=WithProperties('%(script_repo_checkout)s'),
- haltOnFailure=False,
- ))
- script_path = WithProperties('%(script_repo_checkout)s/' + scriptName)
+ # This code path is no longer used
+ assert False, 'script_repo_cache is not used any more on its own'
else:
# fall back to legacy clobbering + cloning script repo
if script_repo_manifest:
- # By setting scriptRepoManifest we indicate that we don't
- # want to use scriptRepo but we want to let the manifest associated
- # to set the repo to checkout and which revision/branch to update to
- # If the repo specified in the manifest matches scriptRepo we will
- # use the cached version if available (i.e. script_repo_cache has
- # been set)
- self.addStep(ShellCommand(
- command=['bash', '-c',
- WithProperties('wget -Orepository_manifest.py ' + \
- '--no-check-certificate --tries=10 --waitretry=3 ' + \
- 'http://hg.mozilla.org/build/tools/raw-file/default/buildfarm/utils/repository_manifest.py')],
- haltOnFailure=True,
- ))
- self.addStep(SetProperty(
- name="set_script_repo_url_and_script_repo_revision",
- extract_fn=extractProperties,
- command=['bash', '-c',
- WithProperties(
- 'python repository_manifest.py ' +
- '--default-repo %s ' % scriptRepo +
- '--default-revision %(script_repo_revision)s ' +
- '--manifest-url %s' % script_repo_manifest)],
- log_eval_func=rc_eval_func({0: SUCCESS, None: EXCEPTION}),
- haltOnFailure=True,
- ))
+ assert False, 'legacy script_repo_manifest unsupported now'
self.addStep(ShellCommand(
name="clobber_scripts",
- command=['rm', '-rf', 'scripts'],
+ command=['rm', '-rf', 'scripts', 'properties'],
workdir=".",
haltOnFailure=True,
log_eval_func=rc_eval_func({0: SUCCESS, None: RETRY}),
))
self.addStep(MercurialCloneCommand(
name="clone_scripts",
- command=[hg_bin, 'clone', script_repo_url, 'scripts'],
+ command=[hg_bin, 'clone', scriptRepo, 'scripts'],
workdir=".",
haltOnFailure=True,
retry=False,
log_eval_func=rc_eval_func({0: SUCCESS, None: RETRY}),
))
self.addStep(ShellCommand(
name="update_scripts",
command=[hg_bin, 'update', '-C', '-r',
@@ -4630,16 +4535,17 @@ class ScriptFactory(RequestSortingBuildF
else:
self.cmd = [interpreter, script_path]
else:
self.cmd = [script_path]
if extra_args:
self.cmd.extend(extra_args)
+
if use_credentials_file:
self.addStep(FileDownload(
mastersrc=os.path.join(os.getcwd(), 'BuildSlaves.py'),
slavedest='oauth.txt',
workdir='.',
flunkOnFailure=False,
))
self.runScript()
@@ -4684,16 +4590,22 @@ class ScriptFactory(RequestSortingBuildF
self.addStep(MockInstall(
target=self.mock_target,
packages=self.mock_packages,
timeout=2700,
))
if self.tooltool_manifest_src:
+ self.addStep(SetProperty(
+ name='set_toolsdir',
+ command=['bash', '-c', 'pwd'],
+ property='toolsdir',
+ workdir='scripts',
+ ))
self.addTooltoolStep(workdir='build')
def runScript(self, env=None):
if not env:
env = self.env
self.preRunScript()
self.addStep(MockCommand(
name="run_script",
@@ -4776,15 +4688,30 @@ class SigningScriptFactory(ScriptFactory
))
self.addStep(SigningServerAuthenication(
servers=self.signingServers,
server_cert=SIGNING_SERVER_CERT,
slavedest=token,
workdir='.',
name='download_token',
))
+ # toolsdir, basedir
+ if self.tools_repo_cache:
+ self.addStep(SetProperty(
+ name='set_toolsdir',
+ command=['bash', '-c', 'pwd'],
+ property='toolsdir',
+ workdir=self.tools_repo_cache
+ ))
+ else:
+ self.addStep(SetProperty(
+ name='set_toolsdir',
+ command=self.get_basedir_cmd,
+ property='toolsdir',
+ workdir='scripts',
+ ))
signing_env = self.env.copy()
signing_env['MOZ_SIGN_CMD'] = WithProperties(get_signing_cmd(
self.signingServers, self.env.get('PYTHON26')))
signing_env['MOZ_SIGNING_SERVERS'] = ",".join(
"%s:%s" % (":".join(s[3]), s[0]) for s in self.signingServers)
ScriptFactory.runScript(self, env=signing_env)