generate-hg-s3-bundles: remove code generating non-generaldelta bundles (bug 1354311); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Thu, 06 Apr 2017 15:06:56 -0700
changeset 10726 540c42d69dd9220639e5e7231656e1d6f5cde910
parent 10725 65a542b7aac87ad75d70b130edf4856fdf1d8041
child 10727 eb24dbfa2361318387d241e881d85233027c4dae
push id1613
push userbmo:gps@mozilla.com
push dateFri, 07 Apr 2017 00:08:28 +0000
reviewersglob
bugs1354311
generate-hg-s3-bundles: remove code generating non-generaldelta bundles (bug 1354311); r?glob We no longer produce bundles that don't use generaldelta. So remove code related to it. The "all" option was only used by generaldelta code, which is why some bundle type entries for generaldelta have been updated in this change. MozReview-Commit-ID: Limo3AzJzQO
scripts/generate-hg-s3-bundles
--- a/scripts/generate-hg-s3-bundles
+++ b/scripts/generate-hg-s3-bundles
@@ -24,37 +24,33 @@ import concurrent.futures as futures
 HG = '/var/hg/venv_tools/bin/hg'
 HG_BUNDLE = '/var/hg/venv_hg_pre/bin/hg'
 PUSH_REPO = '/var/hg/version-control-tools/scripts/push-repo.sh'
 
 # The types of bundles to generate.
 #
 # Define in order bundles should be listed in manifest.
 CREATES = [
-    ('gzip', ['bundle', '-a', '-t', 'gzip-v1'], {}),
-    ('gzip-v2', ['bundle', '-a', '-t', 'gzip-v2'], {'gd': True}),
+    ('gzip-v2', ['bundle', '-a', '-t', 'gzip-v2'], {}),
     # ``zstd`` uses default compression settings and is reasonably fast.
     # ``zstd-max`` uses the highest available compression settings and is
     # absurdly slow. But it produces significantly smaller bundles. Level 20
     # (and not higher) is used because it is the largest level supported
     # by the zstd library in 32-bit processes.
     ('zstd', ['bundle', '-a', '-t', 'zstd-v2'], {'all': True}),
     ('zstd-max', ['--config', 'experimental.bundlecomplevel=20',
                   'bundle', '-a', '-t', 'zstd-v2'], {'all': True}),
-    ('packed1', ['debugcreatestreamclonebundle'], {}),
-    ('packed1-gd', ['debugcreatestreamclonebundle'], {'gd': True}),
+    ('packed1-gd', ['debugcreatestreamclonebundle'], {}),
 ]
 
 CLONEBUNDLES_ORDER = [
     ('zstd-max', 'BUNDLESPEC=zstd-v2'),
     ('zstd', 'BUNDLESPEC=zstd-v2'),
     ('gzip-v2', 'BUNDLESPEC=gzip-v2'),
-    ('gzip', 'BUNDLESPEC=gzip-v1'),
     ('packed1-gd', 'BUNDLESPEC=none-packed1;requirements%3Dgeneraldelta%2Crevlogv1'),
-    ('packed1', 'BUNDLESPEC=none-packed1;requirements%3Drevlogv1'),
 ]
 
 # Defines hostname and bucket where uploads should go.
 HOSTS = (
     # We list Oregon (us-west-2) before N. California (us-west-1) because it is
     # cheaper.
     ('s3-us-west-2.amazonaws.com', 'moz-hg-bundles-us-west-2', 'us-west-2'),
     ('s3-us-west-1.amazonaws.com', 'moz-hg-bundles-us-west-1', 'us-west-1'),
@@ -103,36 +99,32 @@ HTML_INDEX = '''
        For more, see
        <a href="https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/bundleclone.html">the official docs</a>.
     </p>
     <table border="1">
       <tr>
         <th>Repository</th>
         <th>zstd</th>
         <th>zstd (max)</th>
-        <th>gzip (v1)</th>
         <th>gzip (v2)</th>
-        <th>stream</th>
         <th>stream (generaldelta)</th>
       </tr>
       %s
     </table>
     <p>This page generated at %s.</p>
   </body>
 </html>
 '''.strip()
 
 HTML_ENTRY = '''
 <tr>
   <td>{repo}</td>
   <td class="numeric">{zstd_entry}</td>
   <td class="numeric">{zstd_max_entry}</td>
-  <td class="numeric">{gzip_entry}</td>
   <td class="numeric">{gzip_v2_entry}</td>
-  <td class="numeric">{packed1_entry}</a></td>
   <td class="numeric">{packed1_gd_entry}</td>
 </tr>
 '''.strip()
 
 
 def upload_to_s3(host, bucket_name, local_path, remote_path):
     """Upload a file to S3."""
     attempt = 0
@@ -276,28 +268,16 @@ def generate_bundles(repo, upload=True, 
         for t, args, opts in CREATES:
             # Only generate 1 of zstd or zstd-max since they are redundant.
             if t == 'zstd' and zstd_max:
                 continue
 
             if t == 'zstd-max' and not zstd_max:
                 continue
 
-            # The "all" opt allows generation for generaldelta and
-            # non-generaldelta repos. If not set, we key off the generaldelta
-            # state and the "gd" key to determine whether to generate. What
-            # we're trying to prevent is v1 bundle types being generated for
-            # generaldelta repos, which could require a lot of CPU usage to
-            # recalculate deltas.
-            if not opts.get('all'):
-                if generaldelta and not opts.get('gd'):
-                    continue
-                elif not generaldelta and opts.get('gd'):
-                    continue
-
             final_path, remote_path = bundle_paths(bundle_path, repo, tip, t)
             temp_path = '%s.tmp' % final_path
 
             # Record that this bundle is relevant.
             bundles.append((t, final_path, remote_path))
 
             if os.path.exists(final_path):
                 print('bundle already exists, skipping: %s' % final_path)
@@ -386,24 +366,23 @@ def generate_bundles(repo, upload=True, 
 def generate_index(repos):
     """Upload an index HTML page describing available bundles."""
     entries = []
 
     for repo in sorted(repos):
         p = repos[repo]
 
         # Should only be for bundles with copyfrom.
-        if 'gzip' not in p and 'gzip-v2' not in p:
+        if 'gzip-v2' not in p:
             print('ignoring repo %s in index because no gzip bundle' % repo)
             continue
 
         opts = {'repo': repo}
 
-        for k in ('gzip', 'gzip-v2', 'packed1', 'packed1-gd',
-                  'zstd', 'zstd-max'):
+        for k in ('gzip-v2', 'packed1-gd', 'zstd', 'zstd-max'):
             key = '%s_entry' % k.replace('-', '_')
             if k in p:
                 opts[key] = '<a href="{path}">{size:,}</a>'.format(
                     path=p[k][0],
                     size=p[k][1],
                 )
                 opts['basename'] = os.path.basename(p[k][0])
             else: