Bug 1438839 - Finish the flake8 fixes on tools/ r?ahal
MozReview-Commit-ID: F9c4laxe8Pt
--- a/tools/jprof/split-profile.py
+++ b/tools/jprof/split-profile.py
@@ -28,17 +28,19 @@
# split profiles into segments. It takes as input a single text file,
# and from that text file creates a series of jprof profiles in the
# directory the text file is in.
#
# The input file format looks like the following:
#
# poll g_main_poll
# GetRuleCascade CSSRuleProcessor::GetRuleCascade(nsPresContext *, nsAtom *)
-# RuleProcessorData RuleProcessorData::RuleProcessorData(nsPresContext *, nsIContent *, nsRuleWalker *, nsCompatibility *)
+# RuleProcessorData RuleProcessorData::RuleProcessorData
+# (nsPresContext *, nsIContent *, nsRuleWalker *, nsCompatibility *)
+#
#
# From this input file, the script will construct a profile called
# jprof-0.html that contains the whole profile, a profile called
# jprof-1-poll.html that includes only stacks with g_main_poll, a
# profile called jprof-2-GetRuleCascade.html that includes only stacks
# that have GetRuleCascade and do not have g_main_poll, a profile called
# jprof-3-RuleProcessorData.html that includes only stacks that have the
# RuleProcessorData constructor and do not have GetRuleCascade or
--- a/tools/power/mach_commands.py
+++ b/tools/power/mach_commands.py
@@ -129,12 +129,12 @@ class MachCommands(MachCommandBase):
# - 'Terminal' is for the terminal. If no browser is running from
# within the terminal, it will show up unnecessarily. This is a
# minor disadvantage of this very simple parsing strategy.
#
# - 'WindowServer' is for the WindowServer.
#
# - 'kernel' is for the kernel.
#
- if re.search(r'(^Name|firefox|plugin-container|Safari\b|WebKit|Chrome|Terminal|WindowServer|kernel)', line):
+ if re.search(r'(^Name|firefox|plugin-container|Safari\b|WebKit|Chrome|Terminal|WindowServer|kernel)', line): # NOQA: E501
print(line)
return 0
--- a/tools/update-packaging/make_incremental_updates.py
+++ b/tools/update-packaging/make_incremental_updates.py
@@ -198,25 +198,26 @@ def copy_file(src_file_abs_path, dst_fil
dst_file_dir = os.path.dirname(dst_file_abs_path)
if not os.path.exists(dst_file_dir):
os.makedirs(dst_file_dir)
# Copy the file over
shutil.copy2(src_file_abs_path, dst_file_abs_path)
def xz_file(filename):
- """ XZ compresses the file in place. The original file is replaced with the xz compressed version of itself
- assumes the path is absolute"""
+ """ XZ compresses the file in place. The original file is replaced
+ with the xz compressed version of itself assumes the path is absolute"""
exec_shell_cmd('xz --compress --x86 --lzma2 --format=xz --check=crc64 "' + filename + '"')
os.rename(filename + ".xz", filename)
def xzunzip_file(filename):
- """ xz decompresses the file in palce. The original file is replaced with a xz decompressed version of itself.
- doesn't matter if the filename ends in .xz or not"""
+ """ xz decompresses the file in palce. The original file is replaced
+ with a xz decompressed version of itself. doesn't matter if the
+ filename ends in .xz or not"""
if not filename.endswith(".xz"):
os.rename(filename, filename + ".xz")
filename = filename + ".xz"
exec_shell_cmd('xz -d "' + filename + '"')
def extract_mar(filename, work_dir):
""" Extracts the marfile intot he work_dir
@@ -285,24 +286,26 @@ def create_partial_patch_for_file(from_m
file_in_manifest_name = to_marfile_entry.name
patch_info.append_add_instruction(file_in_manifest_name)
# Copy the pre-calculated file into our new patch work aread
copy_file(src_file_abs_path, os.path.join(patch_info.work_dir, file_in_manifest_name))
patch_info.archive_files.append('"' + file_in_manifest_name + '"')
def create_add_patch_for_file(to_marfile_entry, patch_info):
- """ Copy the file to the working dir, add the add instruction, and add it to the list of archive files """
+ """ Copy the file to the working dir, add the add instruction,
+ and add it to the list of archive files """
copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
patch_info.append_add_instruction(to_marfile_entry.name)
patch_info.archive_files.append('"' + to_marfile_entry.name + '"')
def create_add_if_not_patch_for_file(to_marfile_entry, patch_info):
- """ Copy the file to the working dir, add the add-if-not instruction, and add it to the list of archive files """
+ """ Copy the file to the working dir, add the add-if-not instruction,
+ and add it to the list of archive files """
copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
patch_info.append_add_if_not_instruction(to_marfile_entry.name)
patch_info.archive_files.append('"' + to_marfile_entry.name + '"')
def process_explicit_remove_files(dir_path, patch_info):
""" Looks for a 'removed-files' file in the dir_path. If the removed-files does not exist
this will throw. If found adds the removed-files
@@ -330,17 +333,18 @@ def process_explicit_remove_files(dir_pa
# Exclude any blank and comment lines.
if line and not line.startswith("#"):
# Python on windows uses \ for path separators and the update
# manifests expects / for path separators on all platforms.
line = line.replace("\\", "/")
patch_info.append_remove_instruction(line)
-def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates, add_if_not_list):
+def create_partial_patch(from_dir_path, to_dir_path, patch_filename,
+ shas, patch_info, forced_updates, add_if_not_list):
""" Builds a partial patch by comparing the files in from_dir_path to those of to_dir_path"""
# Cannocolize the paths for safey
from_dir_path = os.path.abspath(from_dir_path)
to_dir_path = os.path.abspath(to_dir_path)
# Create a hashtable of the from and to directories
from_dir_hash, from_file_set, from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
to_dir_hash, to_file_set, to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
# Create a list of the forced updates
@@ -467,28 +471,28 @@ def get_buildid(work_dir):
def decode_filename(filepath):
""" Breaks filename/dir structure into component parts based on regex
for example: firefox-3.0b3pre.en-US.linux-i686.complete.mar
Or linux-i686/en-US/firefox-3.0b3.complete.mar
Returns dict with keys product, version, locale, platform, type
"""
try:
- m = re.search(
- '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)',
- os.path.basename(filepath))
- return m.groupdict()
+ m = re.search(
+ '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)', # NOQA: E501
+ os.path.basename(filepath))
+ return m.groupdict()
except Exception as exc:
- try:
- m = re.search(
- '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar',
- filepath)
- return m.groupdict()
- except:
- raise Exception("could not parse filepath %s: %s" % (filepath, exc))
+ try:
+ m = re.search(
+ '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar', # NOQA: E501
+ filepath)
+ return m.groupdict()
+ except Exception:
+ raise Exception("could not parse filepath %s: %s" % (filepath, exc))
def create_partial_patches(patches):
""" Given the patches generates a set of partial patches"""
shas = {}
work_dir_root = None
metadata = []
@@ -524,18 +528,24 @@ def create_partial_patches(patches):
extract_mar(to_filename, work_dir_to)
to_decoded = decode_filename(from_filename)
to_buildid = get_buildid(work_dir_to)
to_shasum = hashlib.sha1(open(to_filename, 'rb').read()).hexdigest()
to_size = str(os.path.getsize(to_filename))
mar_extract_time = time.time()
- partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, [
- 'update.manifest', 'updatev2.manifest', 'updatev3.manifest'], []), forced_updates, ['channel-prefs.js', 'update-settings.ini'])
+ partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename,
+ shas, PatchInfo(work_dir, [
+ 'update.manifest',
+ 'updatev2.manifest',
+ 'updatev3.manifest'
+ ], []),
+ forced_updates,
+ ['channel-prefs.js', 'update-settings.ini'])
partial_shasum = hashlib.sha1(open(partial_filename, "rb").read()).hexdigest()
partial_size = str(os.path.getsize(partial_filename))
metadata.append({
'to_filename': os.path.basename(to_filename),
'from_filename': os.path.basename(from_filename),
'partial_filename': os.path.basename(partial_filename),
'to_buildid': to_buildid,
@@ -546,18 +556,18 @@ def create_partial_patches(patches):
'to_size': to_size,
'from_size': from_size,
'partial_size': partial_size,
'to_version': to_decoded['version'],
'from_version': from_decoded['version'],
'locale': from_decoded['locale'],
'platform': from_decoded['platform'],
})
- print("done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),
- str(len(patches)), mar_extract_time - startTime, time.time() - mar_extract_time, time.time() - startTime))
+ print("done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num), # NOQA: E501
+ str(len(patches)), mar_extract_time - startTime, time.time() - mar_extract_time, time.time() - startTime)) # NOQA: E501
patch_num += 1
return metadata
finally:
# If we fail or get a ctrl-c during run be sure to clean up temp dir
if (work_dir_root and os.path.exists(work_dir_root)):
shutil.rmtree(work_dir_root)
--- a/tools/update-packaging/test_make_incremental_updates.py
+++ b/tools/update-packaging/test_make_incremental_updates.py
@@ -28,37 +28,41 @@ class TestPatchInfo(unittest.TestCase):
def test_append_add_instruction(self):
self.patch_info.append_add_instruction('file.test')
self.assertEquals(['add "file.test"'], self.patch_info.manifestv2)
self.assertEquals(['add "file.test"'], self.patch_info.manifestv3)
def test_append_add_if_instruction(self):
self.patch_info.append_add_instruction('distribution/extensions/extension/file.test')
self.assertEquals(
- ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2)
+ ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], # NOQA: E501
+ self.patch_info.manifestv2)
self.assertEquals(
- ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3)
+ ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], # NOQA: E501
+ self.patch_info.manifestv3)
def test_append_add_if_not_instruction(self):
self.patch_info.append_add_if_not_instruction('file.test')
self.assertEquals([], self.patch_info.manifestv2)
self.assertEquals(['add-if-not "file.test" "file.test"'], self.patch_info.manifestv3)
def test_append_patch_instruction(self):
self.patch_info.append_patch_instruction('file.test', 'patchname')
self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv2)
self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv3)
def test_append_patch_if_instruction(self):
self.patch_info.append_patch_instruction(
'distribution/extensions/extension/file.test', 'patchname')
self.assertEquals(
- ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2)
+ ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], # NOQA: E501
+ self.patch_info.manifestv2)
self.assertEquals(
- ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3)
+ ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], # NOQA: E501
+ self.patch_info.manifestv3)
def test_append_remove_instruction(self):
self.patch_info.append_remove_instruction('file.test')
self.assertEquals(['remove "file.test"'], self.patch_info.manifestv2)
self.assertEquals(['remove "file.test"'], self.patch_info.manifestv3)
def test_append_rmdir_instruction(self):
self.patch_info.append_remove_instruction('dirtest/')
@@ -105,17 +109,19 @@ class TestMarFileEntry(unittest.TestCase
self.assertEquals(goodSha, sha)
"""
class TestMakeIncrementalUpdates(unittest.TestCase):
def setUp(self):
work_dir = '.'
self.patch_info = PatchInfo(
- work_dir, ['update.manifest', 'updatev2.manifest', 'updatev3.manifest'], ['/readme.txt'])
+ work_dir,
+ ['update.manifest', 'updatev2.manifest', 'updatev3.manifest'],
+ ['/readme.txt'])
root_path = '/'
filename = 'test.file'
self.mar_file_entry = MarFileEntry(root_path, filename)
""" FIXME makes direct shell calls, need refactoring
def test_exec_shell_cmd(self):
mkup.exec_shell_cmd('echo test')
@@ -127,26 +133,28 @@ class TestMakeIncrementalUpdates(unittes
def test_bunzip_file(self):
mkup.bunzip_file('filename')
def test_extract_mar(self):
mkup.extract_mar('filename', 'work_dir')
def test_create_partial_patch_for_file(self):
- mkup.create_partial_patch_for_file('from_marfile_entry', 'to_marfile_entry', 'shas', self.patch_info)
+ mkup.create_partial_patch_for_file('from_marfile_entry', 'to_marfile_entry', 'shas',
+ self.patch_info)
def test_create_add_patch_for_file(self):
mkup.create_add_patch_for_file('to_marfile_entry', self.patch_info)
def test_process_explicit_remove_files(self):
mkup.process_explicit_remove_files('dir_path', self.patch_info)
def test_create_partial_patch(self):
- mkup.create_partial_patch('from_dir_path', 'to_dir_path', 'patch_filename', 'shas', self.patch_info, 'forced_updates')
+ mkup.create_partial_patch('from_dir_path', 'to_dir_path', 'patch_filename', 'shas',
+ self.patch_info, 'forced_updates')
def test_create_partial_patches(patches):
mkup.create_partial_patches('patches')
"""
""" FIXME touches the filesystem, need refactoring
def test_get_buildid(self):