generate-hg-s3-bundles: stop supporting legacy stream bundles (bug 1352494); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Fri, 31 Mar 2017 10:41:38 -0700
changeset 10598 11f958e4114b972c096ebccffcb24c008a3f9560
parent 10597 c4250dd5507b8d8d371fcfbc19fe54b12a88306f
child 10599 19f058e492c1b74aeec543e7f36593733515d278
push id1596
push userbmo:gps@mozilla.com
push dateFri, 31 Mar 2017 21:38:22 +0000
reviewersglob
bugs1352494
generate-hg-s3-bundles: stop supporting legacy stream bundles (bug 1352494); r?glob We just stopped generating them in the manifest file. So let's stop supporting their generation. MozReview-Commit-ID: 9s72AS3DLAW
hgserver/tests/test-clonebundles.t
scripts/generate-hg-s3-bundles
--- a/hgserver/tests/test-clonebundles.t
+++ b/hgserver/tests/test-clonebundles.t
@@ -256,69 +256,44 @@ bzip2 bundles created when requested
   https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 ec2region=us-east-1
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 ec2region=eu-central-1
   https://hg.cdn.mozilla.net/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1
   https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1
 
-Legacy stream bundles only generated when requested
-
-  $ cd mozilla-central
-  $ echo legacystream > foo
-  $ hg commit -m legacystream
-  $ hg push > /dev/null
-  $ cd ..
-
-  $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central legacy_stream' --no-upload > /dev/null
-  $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini
-
-  $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=bundles
-  200
-  
-  https://hg.cdn.mozilla.net/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg compression=gzip cdn=true requiresni=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=us-west-2 compression=gzip
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=us-west-1 compression=gzip
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=us-east-1 compression=gzip
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=eu-central-1 compression=gzip
-  https://hg.cdn.mozilla.net/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg stream=revlogv1 cdn=true requiresni=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-west-2 stream=revlogv1
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-west-1 stream=revlogv1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-east-1 stream=revlogv1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=eu-central-1 stream=revlogv1
-
 zstd-max bundles created when requested
 
   $ cd mozilla-central
   $ echo ztd-max > foo
   $ hg commit -m zstd-max
   $ hg push >/dev/null
   $ cd ..
   $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central zstd_max' --no-upload > /dev/null
   $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini
 
   $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles
   200
   
-  https://hg.cdn.mozilla.net/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
-  https://hg.cdn.mozilla.net/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1
-  https://hg.cdn.mozilla.net/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1
+  https://hg.cdn.mozilla.net/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.zstd-max.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
+  https://hg.cdn.mozilla.net/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1
+  https://hg.cdn.mozilla.net/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1
 
 Generaldelta repos should create zstd-v2, gzip-v2, and streamclone bundles only
 
   $ hgmo create-repo generaldelta scm_level_1
   (recorded repository creation in replication log)
   $ hg -q clone ssh://${SSH_SERVER}:${SSH_PORT}/generaldelta
   $ cd generaldelta
   $ touch foo
--- a/scripts/generate-hg-s3-bundles
+++ b/scripts/generate-hg-s3-bundles
@@ -33,25 +33,23 @@ CREATES = [
     ('gzip-v2', ['bundle', '-a', '-t', 'gzip-v2'], {'gd': True}),
     ('bzip2', ['bundle', '-a', '-t', 'bzip2-v1'], {}),
     # ``zstd`` uses default compression settings and is reasonably fast.
     # ``zstd-max`` uses the highest available compression settings and is
     # absurdly slow. But it produces significantly smaller bundles.
     ('zstd', ['bundle', '-a', '-t', 'zstd-v2'], {'all': True}),
     ('zstd-max', ['--config', 'experimental.bundlecomplevel=21',
                   'bundle', '-a', '-t', 'zstd-v2'], {'all': True}),
-    ('stream-legacy', ['streambundle'], {}),
     ('packed1', ['streambundle', '--type', 's1'], {}),
     ('packed1-gd', ['streambundle', '--type', 's1'], {'gd': True}),
 ]
 
 BUNDLECLONE_ORDER = [
     ('gzip', 'compression=gzip'),
     ('bzip2', 'compression=bzip2'),
-    ('stream-legacy', 'stream=revlogv1'),
 ]
 
 CLONEBUNDLES_ORDER = [
     ('zstd-max', 'BUNDLESPEC=zstd-v2'),
     ('zstd', 'BUNDLESPEC=zstd-v2'),
     ('gzip-v2', 'BUNDLESPEC=gzip-v2'),
     ('gzip', 'BUNDLESPEC=gzip-v1'),
     ('bzip2', 'BUNDLESPEC=bzip2-v1'),
@@ -115,17 +113,16 @@ HTML_INDEX = '''
         <th>Repository</th>
         <th>zstd</th>
         <th>zstd (max)</th>
         <th>gzip (v1)</th>
         <th>gzip (v2)</th>
         <th>bz2 (v1)</th>
         <th>stream</th>
         <th>stream (generaldelta)</th>
-        <th>stream-legacy</th>
       </tr>
       %s
     </table>
     <p>This page generated at %s.</p>
   </body>
 </html>
 '''.strip()
 
@@ -134,17 +131,16 @@ HTML_ENTRY = '''
   <td>{repo}</td>
   <td class="numeric">{zstd_entry}</td>
   <td class="numeric">{zstd_max_entry}</td>
   <td class="numeric">{gzip_entry}</td>
   <td class="numeric">{gzip_v2_entry}</td>
   <td class="numeric">{bzip2_entry}</td>
   <td class="numeric">{packed1_entry}</a></td>
   <td class="numeric">{packed1_gd_entry}</td>
-  <td class="numeric">{stream_legacy_entry}</td>
 </tr>
 '''.strip()
 
 
 def upload_to_s3(host, bucket_name, local_path, remote_path):
     """Upload a file to S3."""
     attempt = 0
     while attempt < 3:
@@ -198,23 +194,20 @@ def generate_bundle(repo, temp_path, fin
     args = [HG_BUNDLE,
             '--config', 'extensions.vcsreplicator=!',
             '-R', repo] + extra_args + [temp_path]
     subprocess.check_call(args)
     os.rename(temp_path, final_path)
 
 
 def generate_bundles(repo, upload=True, bzip2=False, copyfrom=None,
-                     legacy_stream=False, zstd_max=False):
+                     zstd_max=False):
     """Generate bundle files for a repository at a path.
 
     ``bzip2`` denotes whether to generate bzip2 bundles.
-    ``legacy_stream`` denotes whether to generate legacy stream bundles
-    (required for compatibility with the "bundleclone" extension, which
-    pre-dated Mercurial's built-in support for "clonebundles.").
     ``zstd_max`` denotes whether to generate zstd bundles with maximum
     compression.
     """
     # Copy manifest files from the source repository listed. Don't return
     # anything because we don't need to list bundles since this repo isn't
     # canonical.
     if copyfrom:
         # We assume all paths are pinned from a common root.
@@ -289,19 +282,16 @@ def generate_bundles(repo, upload=True, 
 
     bundles = []
     fs = []
     with futures.ThreadPoolExecutor(CONCURRENT_THREADS) as e:
         for t, args, opts in CREATES:
             if t == 'bzip2' and not bzip2:
                 continue
 
-            if t == 'stream-legacy' and not legacy_stream:
-                continue
-
             # Only generate 1 of zstd or zstd-max since they are redundant.
             if t == 'zstd' and zstd_max:
                 continue
 
             if t == 'zstd-max' and not zstd_max:
                 continue
 
             # The "all" opt allows generation for generaldelta and
@@ -436,17 +426,17 @@ def generate_index(repos):
         # Should only be for bundles with copyfrom.
         if 'gzip' not in p and 'gzip-v2' not in p:
             print('ignoring repo %s in index because no gzip bundle' % repo)
             continue
 
         opts = {'repo': repo}
 
         for k in ('bzip2', 'gzip', 'gzip-v2', 'packed1', 'packed1-gd',
-                  'stream-legacy', 'zstd', 'zstd-max'):
+                  'zstd', 'zstd-max'):
             key = '%s_entry' % k.replace('-', '_')
             if k in p:
                 opts[key] = '<a href="{path}">{size:,}</a>'.format(
                     path=p[k][0],
                     size=p[k][1],
                 )
                 opts['basename'] = os.path.basename(p[k][0])
             else: