Bug 1438839 - Finish the flake8 fixes on tools/ r?ahal draft
authorSylvestre Ledru <sledru@mozilla.com>
Thu, 15 Mar 2018 11:31:12 +0100
changeset 767912 dc4c6b053aa27d7e42c27b5991ba15676f09857e
parent 767911 2dc2d60128cd754b009e37792a885aba8a4031e2
push id102736
push usersledru@mozilla.com
push dateThu, 15 Mar 2018 10:37:33 +0000
reviewersahal
bugs1438839
milestone61.0a1
Bug 1438839 - Finish the flake8 fixes on tools/ r?ahal MozReview-Commit-ID: F9c4laxe8Pt
tools/jprof/split-profile.py
tools/power/mach_commands.py
tools/update-packaging/make_incremental_updates.py
tools/update-packaging/test_make_incremental_updates.py
--- a/tools/jprof/split-profile.py
+++ b/tools/jprof/split-profile.py
@@ -28,17 +28,19 @@
 # split profiles into segments.  It takes as input a single text file,
 # and from that text file creates a series of jprof profiles in the
 # directory the text file is in.
 #
 # The input file format looks like the following:
 #
 #   poll g_main_poll
 #   GetRuleCascade CSSRuleProcessor::GetRuleCascade(nsPresContext *, nsAtom *)
-#   RuleProcessorData RuleProcessorData::RuleProcessorData(nsPresContext *, nsIContent *, nsRuleWalker *, nsCompatibility *)
+#   RuleProcessorData RuleProcessorData::RuleProcessorData
+#        (nsPresContext *, nsIContent *, nsRuleWalker *, nsCompatibility *)
+#
 #
 # From this input file, the script will construct a profile called
 # jprof-0.html that contains the whole profile, a profile called
 # jprof-1-poll.html that includes only stacks with g_main_poll, a
 # profile called jprof-2-GetRuleCascade.html that includes only stacks
 # that have GetRuleCascade and do not have g_main_poll, a profile called
 # jprof-3-RuleProcessorData.html that includes only stacks that have the
 # RuleProcessorData constructor and do not have GetRuleCascade or
--- a/tools/power/mach_commands.py
+++ b/tools/power/mach_commands.py
@@ -129,12 +129,12 @@ class MachCommands(MachCommandBase):
             # - 'Terminal' is for the terminal. If no browser is running from
             #   within the terminal, it will show up unnecessarily. This is a
             #   minor disadvantage of this very simple parsing strategy.
             #
             # - 'WindowServer' is for the WindowServer.
             #
             # - 'kernel' is for the kernel.
             #
-            if re.search(r'(^Name|firefox|plugin-container|Safari\b|WebKit|Chrome|Terminal|WindowServer|kernel)', line):
+            if re.search(r'(^Name|firefox|plugin-container|Safari\b|WebKit|Chrome|Terminal|WindowServer|kernel)', line):  # NOQA: E501
                 print(line)
 
         return 0
--- a/tools/update-packaging/make_incremental_updates.py
+++ b/tools/update-packaging/make_incremental_updates.py
@@ -198,25 +198,26 @@ def copy_file(src_file_abs_path, dst_fil
     dst_file_dir = os.path.dirname(dst_file_abs_path)
     if not os.path.exists(dst_file_dir):
         os.makedirs(dst_file_dir)
     # Copy the file over
     shutil.copy2(src_file_abs_path, dst_file_abs_path)
 
 
 def xz_file(filename):
-    """ XZ compresses the file in place.  The original file is replaced with the xz compressed version of itself
-        assumes the path is absolute"""
+    """ XZ compresses the file in place.  The original file is replaced
+        with the xz compressed version of itself assumes the path is absolute"""
     exec_shell_cmd('xz --compress --x86 --lzma2 --format=xz --check=crc64 "' + filename + '"')
     os.rename(filename + ".xz", filename)
 
 
 def xzunzip_file(filename):
-    """ xz decompresses the file in palce.  The original file is replaced with a xz decompressed version of itself.
-        doesn't matter if the filename ends in .xz or not"""
+    """ xz decompresses the file in palce.  The original file is replaced
+        with a xz decompressed version of itself. doesn't matter if the
+        filename ends in .xz or not"""
     if not filename.endswith(".xz"):
         os.rename(filename, filename + ".xz")
         filename = filename + ".xz"
     exec_shell_cmd('xz -d "' + filename + '"')
 
 
 def extract_mar(filename, work_dir):
     """ Extracts the marfile intot he work_dir
@@ -285,24 +286,26 @@ def create_partial_patch_for_file(from_m
             file_in_manifest_name = to_marfile_entry.name
             patch_info.append_add_instruction(file_in_manifest_name)
         # Copy the pre-calculated file into our new patch work aread
         copy_file(src_file_abs_path, os.path.join(patch_info.work_dir, file_in_manifest_name))
         patch_info.archive_files.append('"' + file_in_manifest_name + '"')
 
 
 def create_add_patch_for_file(to_marfile_entry, patch_info):
-    """  Copy the file to the working dir, add the add instruction, and add it to the list of archive files """
+    """  Copy the file to the working dir, add the add instruction,
+         and add it to the list of archive files """
     copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
     patch_info.append_add_instruction(to_marfile_entry.name)
     patch_info.archive_files.append('"' + to_marfile_entry.name + '"')
 
 
 def create_add_if_not_patch_for_file(to_marfile_entry, patch_info):
-    """  Copy the file to the working dir, add the add-if-not instruction, and add it to the list of archive files """
+    """  Copy the file to the working dir, add the add-if-not instruction,
+         and add it to the list of archive files """
     copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
     patch_info.append_add_if_not_instruction(to_marfile_entry.name)
     patch_info.archive_files.append('"' + to_marfile_entry.name + '"')
 
 
 def process_explicit_remove_files(dir_path, patch_info):
     """ Looks for a 'removed-files' file in the dir_path.  If the removed-files does not exist
     this will throw.  If found adds the removed-files
@@ -330,17 +333,18 @@ def process_explicit_remove_files(dir_pa
             # Exclude any blank and comment lines.
             if line and not line.startswith("#"):
                 # Python on windows uses \ for path separators and the update
                 # manifests expects / for path separators on all platforms.
                 line = line.replace("\\", "/")
                 patch_info.append_remove_instruction(line)
 
 
-def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates, add_if_not_list):
+def create_partial_patch(from_dir_path, to_dir_path, patch_filename,
+                         shas, patch_info, forced_updates, add_if_not_list):
     """ Builds a partial patch by comparing the files in from_dir_path to those of to_dir_path"""
     # Cannocolize the paths for safey
     from_dir_path = os.path.abspath(from_dir_path)
     to_dir_path = os.path.abspath(to_dir_path)
     # Create a hashtable of the from  and to directories
     from_dir_hash, from_file_set, from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
     to_dir_hash, to_file_set, to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
     # Create a list of the forced updates
@@ -467,28 +471,28 @@ def get_buildid(work_dir):
 
 def decode_filename(filepath):
     """ Breaks filename/dir structure into component parts based on regex
         for example: firefox-3.0b3pre.en-US.linux-i686.complete.mar
         Or linux-i686/en-US/firefox-3.0b3.complete.mar
         Returns dict with keys product, version, locale, platform, type
     """
     try:
-      m = re.search(
-        '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)',
-      os.path.basename(filepath))
-      return m.groupdict()
+        m = re.search(
+            '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)',  # NOQA: E501
+            os.path.basename(filepath))
+        return m.groupdict()
     except Exception as exc:
-      try:
-        m = re.search(
-          '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar',
-        filepath)
-        return m.groupdict()
-      except:
-        raise Exception("could not parse filepath %s: %s" % (filepath, exc))
+        try:
+            m = re.search(
+                '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar',  # NOQA: E501
+                filepath)
+            return m.groupdict()
+        except Exception:
+            raise Exception("could not parse filepath %s: %s" % (filepath, exc))
 
 
 def create_partial_patches(patches):
     """ Given the patches generates a set of partial patches"""
     shas = {}
 
     work_dir_root = None
     metadata = []
@@ -524,18 +528,24 @@ def create_partial_patches(patches):
             extract_mar(to_filename, work_dir_to)
             to_decoded = decode_filename(from_filename)
             to_buildid = get_buildid(work_dir_to)
             to_shasum = hashlib.sha1(open(to_filename, 'rb').read()).hexdigest()
             to_size = str(os.path.getsize(to_filename))
 
             mar_extract_time = time.time()
 
-            partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, [
-                                                    'update.manifest', 'updatev2.manifest', 'updatev3.manifest'], []), forced_updates, ['channel-prefs.js', 'update-settings.ini'])
+            partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename,
+                                                    shas, PatchInfo(work_dir, [
+                                                        'update.manifest',
+                                                        'updatev2.manifest',
+                                                        'updatev3.manifest'
+                                                    ], []),
+                                                    forced_updates,
+                                                    ['channel-prefs.js', 'update-settings.ini'])
             partial_shasum = hashlib.sha1(open(partial_filename, "rb").read()).hexdigest()
             partial_size = str(os.path.getsize(partial_filename))
 
             metadata.append({
                 'to_filename': os.path.basename(to_filename),
                 'from_filename': os.path.basename(from_filename),
                 'partial_filename': os.path.basename(partial_filename),
                 'to_buildid': to_buildid,
@@ -546,18 +556,18 @@ def create_partial_patches(patches):
                 'to_size': to_size,
                 'from_size': from_size,
                 'partial_size': partial_size,
                 'to_version': to_decoded['version'],
                 'from_version': from_decoded['version'],
                 'locale': from_decoded['locale'],
                 'platform': from_decoded['platform'],
             })
-            print("done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),
-                                                                                        str(len(patches)), mar_extract_time - startTime, time.time() - mar_extract_time, time.time() - startTime))
+            print("done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),  # NOQA: E501
+                                                                                        str(len(patches)), mar_extract_time - startTime, time.time() - mar_extract_time, time.time() - startTime))  # NOQA: E501
             patch_num += 1
         return metadata
     finally:
         # If we fail or get a ctrl-c during run be sure to clean up temp dir
         if (work_dir_root and os.path.exists(work_dir_root)):
             shutil.rmtree(work_dir_root)
 
 
--- a/tools/update-packaging/test_make_incremental_updates.py
+++ b/tools/update-packaging/test_make_incremental_updates.py
@@ -28,37 +28,41 @@ class TestPatchInfo(unittest.TestCase):
     def test_append_add_instruction(self):
         self.patch_info.append_add_instruction('file.test')
         self.assertEquals(['add "file.test"'], self.patch_info.manifestv2)
         self.assertEquals(['add "file.test"'], self.patch_info.manifestv3)
 
     def test_append_add_if_instruction(self):
         self.patch_info.append_add_instruction('distribution/extensions/extension/file.test')
         self.assertEquals(
-            ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2)
+            ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'],  # NOQA: E501
+            self.patch_info.manifestv2)
         self.assertEquals(
-            ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3)
+            ['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'],  # NOQA: E501
+            self.patch_info.manifestv3)
 
     def test_append_add_if_not_instruction(self):
         self.patch_info.append_add_if_not_instruction('file.test')
         self.assertEquals([], self.patch_info.manifestv2)
         self.assertEquals(['add-if-not "file.test" "file.test"'], self.patch_info.manifestv3)
 
     def test_append_patch_instruction(self):
         self.patch_info.append_patch_instruction('file.test', 'patchname')
         self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv2)
         self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv3)
 
     def test_append_patch_if_instruction(self):
         self.patch_info.append_patch_instruction(
             'distribution/extensions/extension/file.test', 'patchname')
         self.assertEquals(
-            ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2)
+            ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'],  # NOQA: E501
+            self.patch_info.manifestv2)
         self.assertEquals(
-            ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3)
+            ['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'],  # NOQA: E501
+            self.patch_info.manifestv3)
 
     def test_append_remove_instruction(self):
         self.patch_info.append_remove_instruction('file.test')
         self.assertEquals(['remove "file.test"'], self.patch_info.manifestv2)
         self.assertEquals(['remove "file.test"'], self.patch_info.manifestv3)
 
     def test_append_rmdir_instruction(self):
         self.patch_info.append_remove_instruction('dirtest/')
@@ -105,17 +109,19 @@ class TestMarFileEntry(unittest.TestCase
         self.assertEquals(goodSha, sha)
 """
 
 
 class TestMakeIncrementalUpdates(unittest.TestCase):
     def setUp(self):
         work_dir = '.'
         self.patch_info = PatchInfo(
-            work_dir, ['update.manifest', 'updatev2.manifest', 'updatev3.manifest'], ['/readme.txt'])
+            work_dir,
+            ['update.manifest', 'updatev2.manifest', 'updatev3.manifest'],
+            ['/readme.txt'])
         root_path = '/'
         filename = 'test.file'
         self.mar_file_entry = MarFileEntry(root_path, filename)
 
     """ FIXME makes direct shell calls, need refactoring
     def test_exec_shell_cmd(self):
         mkup.exec_shell_cmd('echo test')
 
@@ -127,26 +133,28 @@ class TestMakeIncrementalUpdates(unittes
 
     def test_bunzip_file(self):
         mkup.bunzip_file('filename')
 
     def test_extract_mar(self):
         mkup.extract_mar('filename', 'work_dir')
 
     def test_create_partial_patch_for_file(self):
-        mkup.create_partial_patch_for_file('from_marfile_entry', 'to_marfile_entry', 'shas', self.patch_info)
+        mkup.create_partial_patch_for_file('from_marfile_entry', 'to_marfile_entry', 'shas',
+                                           self.patch_info)
 
     def test_create_add_patch_for_file(self):
         mkup.create_add_patch_for_file('to_marfile_entry', self.patch_info)
 
     def test_process_explicit_remove_files(self):
         mkup.process_explicit_remove_files('dir_path', self.patch_info)
 
     def test_create_partial_patch(self):
-        mkup.create_partial_patch('from_dir_path', 'to_dir_path', 'patch_filename', 'shas', self.patch_info, 'forced_updates')
+        mkup.create_partial_patch('from_dir_path', 'to_dir_path', 'patch_filename', 'shas',
+                                  self.patch_info, 'forced_updates')
 
     def test_create_partial_patches(patches):
         mkup.create_partial_patches('patches')
 
     """
 
     """ FIXME touches the filesystem, need refactoring
     def test_get_buildid(self):