--- a/toolkit/mozapps/installer/find-dupes.py
+++ b/toolkit/mozapps/installer/find-dupes.py
@@ -8,48 +8,294 @@ from mozpack.packager.unpack import Unpa
from mozpack.files import DeflatedFile
from collections import OrderedDict
'''
Find files duplicated in a given packaged directory, independently of its
package format.
'''
+# Known duplicate files
+# This list is ideally empty, but some existing files will be grandfathered in
+# See bug 1303184
+#
+# PLEASE DO NOT ADD MORE EXCEPTIONS TO THIS LIST
+#
+ALLOWED_DUPES = set([
+ # updater on osx is bug 1311194
+ 'LaunchServices/org.mozilla.updater',
+ 'updater.app/Contents/MacOS/org.mozilla.updater',
+ 'updater.app/Contents/PkgInfo',
+ 'browser/chrome.manifest',
+ # browser branding / themes is bug 1313106
+ 'browser/chrome/browser/content/branding/icon128.png',
+ 'browser/chrome/browser/content/branding/icon16.png',
+ 'browser/chrome/browser/content/branding/icon32.png',
+ 'browser/chrome/browser/content/branding/icon48.png',
+ 'browser/chrome/browser/content/browser/defaultthemes/5.footer.png',
+ 'browser/chrome/browser/content/browser/defaultthemes/5.header.png',
+ 'browser/chrome/browser/content/browser/extension.svg',
+ 'browser/chrome/browser/content/browser/places/bookmarkProperties.xul',
+ 'browser/chrome/browser/content/browser/places/bookmarkProperties2.xul',
+ 'browser/chrome/browser/skin/classic/browser/addons/addon-install-confirm.svg',
+ 'browser/chrome/browser/skin/classic/browser/connection-secure.svg',
+ 'browser/chrome/browser/skin/classic/browser/controlcenter/warning-gray.svg',
+ 'browser/chrome/browser/skin/classic/browser/newtab/close.png',
+ 'browser/chrome/browser/skin/classic/browser/theme-switcher-icon.png',
+ # devtools reduction is bug 1311178
+ 'browser/chrome/devtools/content/dom/content/dom-view.css',
+ 'browser/chrome/devtools/content/dom/dom.html',
+ 'browser/chrome/devtools/content/dom/main.js',
+ 'browser/chrome/devtools/content/framework/toolbox-options.js',
+ 'browser/chrome/devtools/content/inspector/fonts/fonts.js',
+ 'browser/chrome/devtools/content/inspector/inspector.xhtml',
+ 'browser/chrome/devtools/content/memory/initializer.js',
+ 'browser/chrome/devtools/content/projecteditor/lib/helpers/readdir.js',
+ 'browser/chrome/devtools/content/shared/frame-script-utils.js',
+ 'browser/chrome/devtools/content/shared/theme-switching.js',
+ 'browser/chrome/devtools/modules/devtools/client/dom/content/dom-view.css',
+ 'browser/chrome/devtools/modules/devtools/client/dom/dom.html',
+ 'browser/chrome/devtools/modules/devtools/client/dom/main.js',
+ 'browser/chrome/devtools/modules/devtools/client/framework/toolbox-options.js',
+ 'browser/chrome/devtools/modules/devtools/client/inspector/fonts/fonts.js',
+ 'browser/chrome/devtools/modules/devtools/client/inspector/inspector.xhtml',
+ 'browser/chrome/devtools/modules/devtools/client/jsonview/css/controls.png',
+ 'browser/chrome/devtools/modules/devtools/client/jsonview/css/controls@2x.png',
+ 'browser/chrome/devtools/modules/devtools/client/memory/initializer.js',
+ 'browser/chrome/devtools/modules/devtools/client/projecteditor/lib/helpers/readdir.js',
+ 'browser/chrome/devtools/modules/devtools/client/shared/frame-script-utils.js',
+ 'browser/chrome/devtools/modules/devtools/client/shared/theme-switching.js',
+ 'browser/chrome/devtools/modules/devtools/client/themes/common.css',
+ 'browser/chrome/devtools/modules/devtools/client/themes/variables.css',
+ 'browser/chrome/devtools/skin/common.css',
+ 'browser/chrome/devtools/skin/images/command-scratchpad.svg',
+ 'browser/chrome/devtools/skin/images/controls.png',
+ 'browser/chrome/devtools/skin/images/controls@2x.png',
+ 'browser/chrome/devtools/skin/images/debugger-blackbox.svg',
+ 'browser/chrome/devtools/skin/images/debugger-prettyprint.svg',
+ 'browser/chrome/devtools/skin/images/filetypes/store.svg',
+ 'browser/chrome/devtools/skin/images/itemToggle.svg',
+ 'browser/chrome/devtools/skin/images/security-state-broken.svg',
+ 'browser/chrome/devtools/skin/images/security-state-local.svg',
+ 'browser/chrome/devtools/skin/images/security-state-secure.svg',
+ 'browser/chrome/devtools/skin/images/tabs-icon.svg',
+ 'browser/chrome/devtools/skin/images/tool-scratchpad.svg',
+ 'browser/chrome/devtools/skin/images/tool-storage.svg',
+ 'browser/chrome/devtools/skin/images/tool-styleeditor.svg',
+ 'browser/chrome/devtools/skin/promisedebugger.css',
+ 'browser/chrome/devtools/skin/variables.css',
+ 'modules/devtools/Console.jsm',
+ 'modules/devtools/Loader.jsm',
+ 'modules/devtools/Simulator.jsm',
+ 'modules/devtools/shared/Console.jsm',
+ 'modules/devtools/shared/Loader.jsm',
+ 'modules/devtools/shared/apps/Simulator.jsm',
+ 'browser/modules/devtools/client/framework/gDevTools.jsm',
+ 'browser/modules/devtools/gDevTools.jsm',
+ 'browser/chrome/icons/default/default16.png',
+ 'browser/chrome/icons/default/default32.png',
+ 'browser/chrome/icons/default/default48.png',
+ 'browser/chrome/pdfjs/content/web/images/findbarButton-next-rtl.png',
+ 'browser/chrome/pdfjs/content/web/images/findbarButton-next-rtl@2x.png',
+ 'browser/chrome/pdfjs/content/web/images/findbarButton-next.png',
+ 'browser/chrome/pdfjs/content/web/images/findbarButton-next@2x.png',
+ 'browser/chrome/pdfjs/content/web/images/findbarButton-previous-rtl.png',
+ 'browser/chrome/pdfjs/content/web/images/findbarButton-previous-rtl@2x.png',
+ 'browser/chrome/pdfjs/content/web/images/findbarButton-previous.png',
+ 'browser/chrome/pdfjs/content/web/images/findbarButton-previous@2x.png',
+ 'browser/extensions/{972ce4c6-7e08-4474-a285-3208198ce6fd}/icon.png',
+ 'browser/features/firefox@getpocket.com/chrome/skin/linux/menuPanel.png',
+ 'browser/features/firefox@getpocket.com/chrome/skin/linux/menuPanel@2x.png',
+ 'browser/features/firefox@getpocket.com/chrome/skin/windows/menuPanel.png',
+ 'browser/features/firefox@getpocket.com/chrome/skin/windows/menuPanel@2x.png',
+ # flyweb reduction is bug 1313107
+ 'browser/features/flyweb@mozilla.org/chrome/skin/linux/flyweb.css',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/linux/icon-16.png',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/linux/icon-32-anchored.png',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/linux/icon-32.png',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/linux/icon-64-anchored.png',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/linux/icon-64.png',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/osx/flyweb.css',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/osx/icon-32-anchored.png',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/osx/icon-64-anchored.png',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/windows/flyweb.css',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/windows/icon-16.png',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/windows/icon-32-anchored.png',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/windows/icon-32.png',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/windows/icon-64-anchored.png',
+ 'browser/features/flyweb@mozilla.org/chrome/skin/windows/icon-64.png',
+ 'browser/icons/mozicon128.png',
+ 'chrome.manifest',
+ 'chrome/en-US/locale/en-US/browser/overrides/AccessFu.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/about.dtd',
+ 'chrome/en-US/locale/en-US/browser/overrides/aboutAbout.dtd',
+ 'chrome/en-US/locale/en-US/browser/overrides/aboutReader.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/aboutRights.dtd',
+ 'chrome/en-US/locale/en-US/browser/overrides/charsetMenu.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/commonDialogs.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/crashreporter/crashes.dtd',
+ 'chrome/en-US/locale/en-US/browser/overrides/crashreporter/crashes.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/dom/dom.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/global.dtd',
+ 'chrome/en-US/locale/en-US/browser/overrides/global/aboutSupport.dtd',
+ 'chrome/en-US/locale/en-US/browser/overrides/global/aboutSupport.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/global/aboutTelemetry.dtd',
+ 'chrome/en-US/locale/en-US/browser/overrides/global/aboutTelemetry.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/global/aboutWebrtc.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/global/mozilla.dtd',
+ 'chrome/en-US/locale/en-US/browser/overrides/intl.css',
+ 'chrome/en-US/locale/en-US/browser/overrides/intl.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/passwordmgr.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/plugins.properties',
+ 'chrome/en-US/locale/en-US/browser/overrides/plugins/pluginproblem.dtd',
+ 'chrome/en-US/locale/en-US/browser/overrides/search/search.properties',
+ 'chrome/en-US/locale/en-US/global-platform/mac/intl.properties',
+ 'chrome/en-US/locale/en-US/global-platform/unix/accessible.properties',
+ 'chrome/en-US/locale/en-US/global-platform/unix/intl.properties',
+ 'chrome/en-US/locale/en-US/global-platform/unix/platformKeys.properties',
+ 'chrome/en-US/locale/en-US/global-platform/win/accessible.properties',
+ 'chrome/en-US/locale/en-US/global-platform/win/intl.properties',
+ 'chrome/en-US/locale/en-US/global-platform/win/platformKeys.properties',
+ 'chrome/en-US/locale/en-US/global/AccessFu.properties',
+ 'chrome/en-US/locale/en-US/global/about.dtd',
+ 'chrome/en-US/locale/en-US/global/aboutAbout.dtd',
+ 'chrome/en-US/locale/en-US/global/aboutReader.properties',
+ 'chrome/en-US/locale/en-US/global/aboutRights.dtd',
+ 'chrome/en-US/locale/en-US/global/aboutSupport.dtd',
+ 'chrome/en-US/locale/en-US/global/aboutSupport.properties',
+ 'chrome/en-US/locale/en-US/global/aboutTelemetry.dtd',
+ 'chrome/en-US/locale/en-US/global/aboutTelemetry.properties',
+ 'chrome/en-US/locale/en-US/global/aboutWebrtc.properties',
+ 'chrome/en-US/locale/en-US/global/charsetMenu.properties',
+ 'chrome/en-US/locale/en-US/global/commonDialogs.properties',
+ 'chrome/en-US/locale/en-US/global/crashes.dtd',
+ 'chrome/en-US/locale/en-US/global/crashes.properties',
+ 'chrome/en-US/locale/en-US/global/dom/dom.properties',
+ 'chrome/en-US/locale/en-US/global/global.dtd',
+ 'chrome/en-US/locale/en-US/global/intl.css',
+ 'chrome/en-US/locale/en-US/global/intl.properties',
+ 'chrome/en-US/locale/en-US/global/mozilla.dtd',
+ 'chrome/en-US/locale/en-US/global/plugins.properties',
+ 'chrome/en-US/locale/en-US/global/search/search.properties',
+ 'chrome/en-US/locale/en-US/passwordmgr/passwordmgr.properties',
+ 'chrome/en-US/locale/en-US/pluginproblem/pluginproblem.dtd',
+ 'chrome/toolkit/skin/classic/global/autocomplete.css',
+ 'chrome/toolkit/skin/classic/global/button.css',
+ 'chrome/toolkit/skin/classic/global/checkbox.css',
+ 'chrome/toolkit/skin/classic/global/dialog.css',
+ 'chrome/toolkit/skin/classic/global/dropmarker.css',
+ 'chrome/toolkit/skin/classic/global/global.css',
+ 'chrome/toolkit/skin/classic/global/groupbox.css',
+ 'chrome/toolkit/skin/classic/global/icons/close-XPVista7.png',
+ 'chrome/toolkit/skin/classic/global/icons/tabprompts-bgtexture.png',
+ 'chrome/toolkit/skin/classic/global/listbox.css',
+ 'chrome/toolkit/skin/classic/global/media/clicktoplay-bgtexture.png',
+ 'chrome/toolkit/skin/classic/global/menu.css',
+ 'chrome/toolkit/skin/classic/global/menulist.css',
+ 'chrome/toolkit/skin/classic/global/numberbox.css',
+ 'chrome/toolkit/skin/classic/global/popup.css',
+ 'chrome/toolkit/skin/classic/global/preferences.css',
+ 'chrome/toolkit/skin/classic/global/progressmeter.css',
+ 'chrome/toolkit/skin/classic/global/radio.css',
+ 'chrome/toolkit/skin/classic/global/resizer.css',
+ 'chrome/toolkit/skin/classic/global/richlistbox.css',
+ 'chrome/toolkit/skin/classic/global/scale.css',
+ 'chrome/toolkit/skin/classic/global/scrollbars.css',
+ 'chrome/toolkit/skin/classic/global/scrollbox.css',
+ 'chrome/toolkit/skin/classic/global/spinbuttons.css',
+ 'chrome/toolkit/skin/classic/global/splitter.css',
+ 'chrome/toolkit/skin/classic/global/tabbox.css',
+ 'chrome/toolkit/skin/classic/global/textbox.css',
+ 'chrome/toolkit/skin/classic/global/toolbar.css',
+ 'chrome/toolkit/skin/classic/global/toolbarbutton.css',
+ 'chrome/toolkit/skin/classic/global/tree.css',
+ 'chrome/toolkit/skin/classic/global/wizard.css',
+ 'chrome/toolkit/skin/classic/mozapps/downloads/buttons.png',
+ 'chrome/toolkit/skin/classic/mozapps/downloads/downloadButtons-XP.png',
+ 'chrome/toolkit/skin/classic/mozapps/downloads/downloadButtons.png',
+ 'chrome/toolkit/skin/classic/mozapps/extensions/category-dictionaries.png',
+ 'chrome/toolkit/skin/classic/mozapps/extensions/category-experiments.png',
+ 'chrome/toolkit/skin/classic/mozapps/extensions/dictionaryGeneric.png',
+ 'chrome/toolkit/skin/classic/mozapps/extensions/experimentGeneric.png',
+ 'chrome/toolkit/skin/classic/mozapps/update/buttons.png',
+ 'chrome/toolkit/skin/classic/mozapps/update/downloadButtons-XP.png',
+ 'chrome/toolkit/skin/classic/mozapps/update/downloadButtons.png',
+ 'components/FxAccountsPush.js',
+ 'crashreporter.app/Contents/Resources/English.lproj/MainMenu.nib/classes.nib',
+ 'crashreporter.app/Contents/Resources/English.lproj/MainMenuRTL.nib/classes.nib',
+ # firefox/firefox-bin is bug 658850
+ 'firefox',
+ 'firefox-bin',
+ 'modules/FxAccountsPush.js',
+ 'modules/commonjs/index.js',
+ 'modules/commonjs/sdk/ui/button/view/events.js',
+ 'modules/commonjs/sdk/ui/state/events.js',
+ 'plugin-container.app/Contents/PkgInfo',
+ 'res/table-remove-column-active.gif',
+ 'res/table-remove-column-hover.gif',
+ 'res/table-remove-column.gif',
+ 'res/table-remove-row-active.gif',
+ 'res/table-remove-row-hover.gif',
+ 'res/table-remove-row.gif',
+])
+
+
+def normalize_osx_path(p):
+ '''
+ Strips the first 3 elements of an OSX app path
+
+ >>> normalize_osx_path('Nightly.app/foo/bar/baz')
+ 'baz'
+ '''
+ bits = p.split('/')
+ if len(bits) > 3 and bits[0].endswith('.app'):
+ return '/'.join(bits[3:])
+ return p
+
def find_dupes(source):
md5s = OrderedDict()
for p, f in UnpackFinder(source):
content = f.open().read()
m = hashlib.md5(content).digest()
if m not in md5s:
if isinstance(f, DeflatedFile):
compressed = f.file.compressed_size
else:
compressed = len(content)
md5s[m] = (len(content), compressed, [])
md5s[m][2].append(p)
total = 0
total_compressed = 0
num_dupes = 0
+ unexpected_dupes = []
for m, (size, compressed, paths) in sorted(md5s.iteritems(),
key=lambda x: x[1][1]):
if len(paths) > 1:
print 'Duplicates %d bytes%s%s:' % (size,
' (%d compressed)' % compressed if compressed != size else '',
' (%d times)' % (len(paths) - 1) if len(paths) > 2 else '')
print ''.join(' %s\n' % p for p in paths)
total += (len(paths) - 1) * size
total_compressed += (len(paths) - 1) * compressed
num_dupes += 1
+
+ unexpected_dupes.extend([p for p in paths if normalize_osx_path(p) not in ALLOWED_DUPES])
+
if num_dupes:
print "WARNING: Found %d duplicated files taking %d bytes (%s)" % \
(num_dupes, total,
'%d compressed' % total_compressed if total_compressed != total
else 'uncompressed')
+ if unexpected_dupes:
+ print "ERROR: The following duplicated files are not allowed:"
+ print "\n".join(unexpected_dupes)
+ exit(1)
+
def main():
if len(sys.argv) != 2:
import os
print >>sys.stderr, "Usage: %s directory" % \
os.path.basename(sys.argv[0])
sys.exit(1)
