--- a/mobile/android/mach_commands.py
+++ b/mobile/android/mach_commands.py
@@ -209,8 +209,326 @@ class AutophoneCommands(MachCommandBase)
return 2
if not runner.configure():
runner.save_config()
return 3
runner.save_config()
runner.launch_autophone()
runner.command_prompts()
return 0
+
+import subprocess
+
+# This unorthodox structure is so these can be accessed via multiprocessing.
+javap = [None]
+compiled_mach = {}
+compiled_gradle = {}
+
+def javap_comparator(name):
+ if name not in compiled_mach:
+ return 0
+ if name not in compiled_gradle:
+ return 0
+ # Short circuit if we can.
+ if compiled_mach[name]['sha256'] == compiled_gradle[name]['sha256']:
+ return 0
+
+ if javap[0]:
+ a = subprocess.check_output([javap[0], '-private', '-constants', compiled_mach[name]['path']])
+ b = subprocess.check_output([javap[0], '-private', '-constants', compiled_gradle[name]['path']])
+ if a == b:
+ return 0
+
+ return name
+
+# Modified from http://stackoverflow.com/a/20380514.
+def png_dimensions(fileobj):
+ import imghdr
+ import struct
+
+ head = fileobj.read(24)
+ if len(head) != 24:
+ return None
+ if imghdr.what(None, head) == 'png':
+ check = struct.unpack('>i', head[4:8])[0]
+ if check != 0x0d0a1a0a:
+ return
+ width, height = struct.unpack('>ii', head[16:24])
+ return (width, height)
+
+ return None
+
+@CommandProvider
+class MachCommands(MachCommandBase):
+ # TODO: explain that match is not anchored to the beginning of the given path.
+ @CommandArgument('--filter', default=None,
+ help='Only compare build outputs with relative paths that match the given regular expression.')
+ @CommandArgument('--compare-merged-manifests', action='store_true',
+ help='Compare the merged manifests of Mach and Gradle build outputs.')
+ @CommandArgument('--compare-merged-resources', action='store_true',
+ help='Compare merged resources of Mach and Gradle build outputs.')
+ @CommandArgument('--compare-resource-ids', action='store_true',
+ help='Compare allocated resource IDs of Mach and Gradle build outputs.')
+ @CommandArgument('--compare-resource-contents', action='store_true',
+ help='Compare resource contents of Mach and Gradle build outputs.')
+ @CommandArgument('--compare-classes', action='store_true',
+ help='Compare compiled .class output of Mach and Gradle build outputs.')
+ @CommandArgument('--compare-javap', action='store_true',
+ help='Compare javap output of Mach and Gradle build outputs.')
+ @CommandArgument('--verbose', action='store_true',
+ help='Log informative status messages.')
+ @Command('compare-mach-and-gradle', category='devenv',
+ description='Compare Mach and Gradle build outputs.',
+ conditions=[conditions.is_android])
+ def compare(self,
+ compare_merged_manifests=True,
+ compare_merged_resources=True,
+ compare_resource_ids=True,
+ compare_resource_contents=True,
+ compare_classes=False, filter=None,
+ compare_javap=False,
+ verbose=False):
+ from collections import defaultdict, OrderedDict
+ import hashlib
+ import itertools
+ import multiprocessing
+ import re
+ from mozbuild.util import hash_file
+ from mozpack.files import FileFinder
+
+ retval = 0
+
+ if compare_merged_manifests:
+ if verbose:
+ print("Comparing manifests included in Mach and Gradle builds")
+
+ mach = mozpath.join(self.topobjdir,
+ 'mobile', 'android', 'base',
+ 'AndroidManifest.xml')
+
+ gradle = mozpath.join(self.topobjdir,
+ 'gradle', 'build', 'mobile', 'android', 'app',
+ 'intermediates', 'manifests', 'full', 'officialAustralis', 'release', 'AndroidManifest.xml')
+
+ print("python xmldiff.py 'diff -U2' {0} {1}".format(mach, gradle))
+
+ if compare_merged_resources:
+ if verbose:
+ print("Comparing the set of resource names after merging produced by Mach and Gradle builds")
+
+ mach = defaultdict(set)
+ gradle = defaultdict(set)
+
+ finder = FileFinder(mozpath.join(self.topobjdir,
+ 'mobile', 'android', 'base', 'merged'),
+ find_executables=False)
+
+ for path, _ in finder.find('*/*'):
+ dir, name = path.split('/')
+ mach[name].add(dir)
+
+ finder = FileFinder(mozpath.join(self.topobjdir,
+ 'gradle', 'build', 'mobile', 'android', 'app', 'intermediates', 'res', 'merged', 'officialAustralis', 'release'),
+ find_executables=False)
+
+ for path, _ in finder.find('*/*'):
+ dir, name = path.split('/')
+ gradle[name].add(dir)
+
+ mach_only = list(sorted(set(mach.keys()).difference(set(gradle.keys()))))
+ for name in mach_only:
+ retval |= 0b010000000
+ print("mach-only:", name)
+
+ gradle_only = list(sorted(set(gradle.keys()).difference(set(mach.keys()))))
+ for name in gradle_only:
+ retval |= 0b100000000
+ print("Gradle-only:", name)
+
+ both = list(sorted(set(mach.keys()).intersection(set(gradle.keys()))))
+ for name in both:
+ if mach[name] != gradle[name]:
+ retval |= 0b001000000
+ print("Different:", name)
+ print(list(sorted(mach[name])))
+ print(list(sorted(gradle[name])))
+
+ if verbose and not (retval & 0b111000000):
+ print("The sets of resource names after merging are identical!")
+
+ if compare_resource_contents:
+ if verbose:
+ print("Comparing the resource contents produced by Mach and Gradle builds")
+
+ # The builds produce .ap_ files containing all of the
+ # resources. These .ap_ files are really ZIP files. We
+ # Walk the .ap_ files, verifying the contents are the
+ # same. Since aapt sorts the resources, we should even
+ # have the same order; we won't have the exact same ZIP
+ # files, however, since the embedded timestamps will be
+ # different.
+ import zipfile
+ mach = zipfile.ZipFile(mozpath.join(self.topobjdir,
+ 'mobile', 'android', 'base',
+ 'gecko.ap_'))
+
+ gradle = zipfile.ZipFile(mozpath.join(self.topobjdir,
+ 'gradle', 'build', 'mobile', 'android',
+ 'app', 'intermediates', 'res', 'resources-official-australis-release.ap_'))
+
+ mach_only = list(sorted(set(mach.namelist()).difference(set(gradle.namelist()))))
+ for name in mach_only:
+ retval |= 0b010000000
+ print("mach-only:", name)
+
+ gradle_only = list(sorted(set(gradle.namelist()).difference(set(mach.namelist()))))
+ for name in gradle_only:
+ retval |= 0b100000000
+ print("Gradle-only:", name)
+
+ both = list(sorted(set(mach.namelist()).intersection(set(gradle.namelist()))))
+
+ for name in both:
+ m = mach.getinfo(name)
+ g = gradle.getinfo(name)
+ if m.CRC != g.CRC or m.file_size != g.file_size or m.compress_size != g.compress_size:
+ retval |= 0b001000000
+ if name.endswith('.png'):
+ print("Different:", name, (m.CRC, g.CRC), (m.file_size, g.file_size), (m.compress_size, g.compress_size), (png_dimensions(mach.open(name)), png_dimensions(gradle.open(name))))
+ else:
+ print("Different:", name)
+
+
+ if compare_resource_ids:
+ if verbose:
+ print("Comparing resource IDs allocated in Mach and Gradle build outputs")
+
+ fm = open(mozpath.join(self.topobjdir,
+ 'mobile', 'android', 'base',
+ 'R.txt'))
+
+ fg = open(mozpath.join(self.topobjdir,
+ 'gradle', 'build', 'mobile', 'android', 'app', 'intermediates', 'symbols', 'officialAustralis', 'release', 'R.txt'))
+
+ mach = OrderedDict([])
+ for line in fm:
+ (type, dir, name, id) = line.split(None, 3)
+ if not type.endswith("[]"):
+ mach[name] = (type, dir, int(id, base=16))
+ else:
+ # Handle arrays later.
+ pass
+
+ gradle = OrderedDict([])
+ for line in fg:
+ (type, dir, name, id) = line.split(None, 3)
+ if not type.endswith("[]"):
+ gradle[name] = (type, dir, int(id, base=16))
+ else:
+ # Handle arrays later.
+ pass
+
+ mach_only = list(sorted(set(mach.keys()).difference(set(gradle.keys()))))
+ for name in mach_only:
+ retval |= 0b010000
+ print("mach-only:", name)
+
+ gradle_only = list(sorted(set(gradle.keys()).difference(set(mach.keys()))))
+ for name in gradle_only:
+ retval |= 0b100000
+ print("Gradle-only:", name)
+
+ both = list(sorted(set(mach.keys()).intersection(set(gradle.keys()))))
+
+ jumps = []
+
+ offset = 0
+ for i, (m, g) in enumerate(zip(mach.items(), gradle.items())):
+ (mname, (mtype, mdir, mid)) = m
+ (gname, (gtype, gdir, gid)) = g
+
+ if mname != gname:
+ retval |= 0b001000
+ print("Different resource order at index:", hex(i), mname, gname)
+
+ if mid != gid + offset:
+ retval |= 0b001000
+
+ print("Different resource ids at index:", hex(i), (mname, hex(mid)), (gname, hex(gid)))
+ jumps.append((m, g))
+ offset = mid - gid
+
+ for jump in jumps:
+ print(jump)
+
+ if verbose and not (retval & 0b000111000):
+ print("The resource ID maps produced in R.txt are identical!")
+
+
+ if compare_classes or compare_javap:
+ if verbose:
+ print("Comparing compiled .class output of Mach and Gradle build outputs{0}"
+ .format("" if not filter else "matching regular expression {0}".format(filter)))
+
+ if filter:
+ filter = re.compile(filter)
+
+ for part in ('base', 'geckoview', 'javaaddons', 'stumbler', 'thirdparty'):
+ finder = FileFinder(mozpath.join(self.topobjdir,
+ 'mobile', 'android', part),
+ find_executables=False)
+
+ for path, _ in finder.find('**/*-classes/**/*.class'):
+ _, name = path.split('-classes/', 1)
+
+ if filter and not filter.search(name):
+ continue
+
+ compiled_mach[name] = {}
+ compiled_mach[name]['path'] = mozpath.join(finder.base, path)
+ compiled_mach[name]['sha256'] = hash_file(mozpath.join(finder.base, path))
+
+ for (part, product) in (('app', 'officialAustralis'),
+ ('geckoview', ''),
+ ('thirdparty', '')):
+ finder = FileFinder(mozpath.join(self.topobjdir,
+ 'gradle', 'build', 'mobile', 'android', part,
+ 'intermediates', 'classes',
+ product,
+ 'release'),
+ find_executables=False)
+ for path, _ in finder.find('**/*.class'):
+ name = path
+
+ if filter and not filter.search(name):
+ continue
+
+ compiled_gradle[name] = {}
+ compiled_gradle[name]['path'] = mozpath.join(finder.base, path)
+ compiled_gradle[name]['sha256'] = hash_file(mozpath.join(finder.base, path))
+
+ mach_only = list(sorted(set(compiled_mach.keys()).difference(set(compiled_gradle.keys()))))
+ for name in mach_only:
+ retval |= 0b010
+ print("mach-only:", name)
+
+ gradle_only = list(sorted(set(compiled_gradle.keys()).difference(set(compiled_mach.keys()))))
+ for name in gradle_only:
+ retval |= 0b100
+ print("Gradle-only:", name)
+
+ both = list(sorted(set(compiled_mach.keys()).intersection(set(compiled_gradle.keys()))))
+
+ if compare_javap:
+ javap[0] = mozpath.join(os.path.dirname(self.substs['JAVA']), 'javap')
+
+ pool = multiprocessing.Pool(4)
+ for name in pool.imap(javap_comparator, both):
+ if name:
+ retval |= 0b001
+ print("Different:", name)
+ print(compiled_mach[name]['path'])
+ print(compiled_gradle[name]['path'])
+
+ if verbose and not (retval & 0b000000111):
+ print("Compiled .class files are identical!")
+
+ return retval
new file mode 100644
--- /dev/null
+++ b/xmldiff.py
@@ -0,0 +1,165 @@
+##########################################################################
+#
+# xmldiff
+#
+# Simple utility script to enable a diff of two XML files in a way
+# that ignores the order or attributes and elements.
+#
+# Dale Lane (email@dalelane.co.uk)
+# 6 Oct 2014
+#
+##########################################################################
+#
+# Overview
+# The approach is to sort both files by attribute and element, and
+# then reuse an existing diff implementation on the sorted files.
+#
+# Arguments
+# <diffcommand> the command that should be run to diff the sorted files
+# <filename1> the first XML file to diff
+# <filename2> the second XML file to diff
+#
+# Background
+# http://dalelane.co.uk/blog/?p=3225
+#
+##########################################################################
+
+import os, sys, subprocess, platform
+import lxml.etree as le
+from operator import attrgetter
+
+le.register_namespace('android', 'http://schemas.android.com/apk/res/android')
+
+#
+# Check required arguments
+if len(sys.argv) != 4:
+ print ("Usage: python xmldiff.py <diffcommand> <filename1> <filename2>")
+ quit()
+
+
+#
+# Prepares the location of the temporary file that will be created by xmldiff
+def createFileObj(prefix, name):
+ return {
+ "filename" : os.path.abspath(name),
+ "tmpfilename" : "." + prefix + "." + os.path.basename(name)
+ }
+
+
+#
+# Function to sort XML elements by id
+# (where the elements have an 'id' attribute that can be cast to an int)
+def sortbyid(elem):
+ return elem.get('{http://schemas.android.com/apk/res/android}name') or \
+ elem.get('{http://schemas.android.com/apk/res/android}label')
+ # if name:
+ # return name
+ # return 0
+
+
+#
+# Function to sort XML elements by their text contents
+def sortbytext(elem):
+ text = elem.text
+ if text:
+ return text
+ else:
+ return ''
+
+
+#
+# Function to sort XML attributes alphabetically by key
+# The original item is left unmodified, and it's attributes are
+# copied to the provided sorteditem
+def sortAttrs(item, sorteditem):
+ attrkeys = sorted(item.keys())
+ for key in attrkeys:
+ sorteditem.set(key, item.get(key))
+
+
+#
+# Function to sort XML elements
+# The sorted elements will be added as children of the provided newroot
+# This is a recursive function, and will be called on each of the children
+# of items.
+def sortElements(items, newroot):
+ # The intended sort order is to sort by XML element name
+ # If more than one element has the same name, we want to
+ # sort by their text contents.
+ # If more than one element has the same name and they do
+ # not contain any text contents, we want to sort by the
+ # value of their ID attribute.
+ # If more than one element has the same name, but has
+ # no text contents or ID attribute, their order is left
+ # unmodified.
+ #
+ # We do this by performing three sorts in the reverse order
+ items = sorted(items, key=sortbytext)
+ items = sorted(items, key=sortbyid)
+ items = sorted(items, key=attrgetter('tag'))
+
+ # Once sorted, we sort each of the items
+ for item in items:
+ if item.tag is le.Comment:
+ continue
+
+ # Create a new item to represent the sorted version
+ # of the next item, and copy the tag name and contents
+ newitem = le.Element(item.tag)
+ if item.text and item.text.isspace() == False:
+ newitem.text = item.text
+
+ # Copy the attributes (sorted by key) to the new item
+ sortAttrs(item, newitem)
+
+ # Copy the children of item (sorted) to the new item
+ sortElements(list(item), newitem)
+
+ # Append this sorted item to the sorted root
+ newroot.append(newitem)
+
+
+#
+# Function to sort the provided XML file
+# fileobj.filename will be left untouched
+# A new sorted copy of it will be created at fileobj.tmpfilename
+def sortFile(fileobj):
+ with open(fileobj['filename'], 'r') as original:
+ # parse the XML file and get a pointer to the top
+ xmldoc = le.parse(original)
+ xmlroot = xmldoc.getroot()
+
+ # create a new XML element that will be the top of
+ # the sorted copy of the XML file
+ newxmlroot = le.Element(xmlroot.tag)
+
+ # create the sorted copy of the XML file
+ sortAttrs(xmlroot, newxmlroot)
+ sortElements(list(xmlroot), newxmlroot)
+
+ # write the sorted XML file to the temp file
+ newtree = le.ElementTree(newxmlroot)
+ with open(fileobj['tmpfilename'], 'wb') as newfile:
+ newtree.write(newfile, pretty_print=True)
+
+
+#
+# sort each of the specified files
+filefrom = createFileObj("from", sys.argv[2])
+sortFile(filefrom)
+fileto = createFileObj("to", sys.argv[3])
+sortFile(fileto)
+
+#
+# invoke the requested diff command to compare the two sorted files
+if platform.system() == "Windows":
+ sp = subprocess.Popen([ "cmd", "/c", sys.argv[1] + " " + filefrom['tmpfilename'] + " " + fileto['tmpfilename'] ])
+ sp.communicate()
+else:
+ sp = subprocess.Popen([ "/bin/bash", "-i", "-c", sys.argv[1] + " " + os.path.abspath(filefrom['tmpfilename']) + " " + os.path.abspath(fileto['tmpfilename']) ])
+ sp.communicate()
+
+#
+# cleanup - delete the temporary sorted files after the diff terminates
+os.remove(filefrom['tmpfilename'])
+os.remove(fileto['tmpfilename'])