author | Gregory Szorc <gps@mozilla.com> |
Fri, 31 Mar 2017 10:41:38 -0700 | |
changeset 10598 | 11f958e4114b972c096ebccffcb24c008a3f9560 |
parent 10597 | c4250dd5507b8d8d371fcfbc19fe54b12a88306f |
child 10599 | 19f058e492c1b74aeec543e7f36593733515d278 |
push id | 1596 |
push user | bmo:gps@mozilla.com |
push date | Fri, 31 Mar 2017 21:38:22 +0000 |
reviewers | glob |
bugs | 1352494 |
hgserver/tests/test-clonebundles.t | file | annotate | diff | comparison | revisions | |
scripts/generate-hg-s3-bundles | file | annotate | diff | comparison | revisions |
--- a/hgserver/tests/test-clonebundles.t +++ b/hgserver/tests/test-clonebundles.t @@ -256,69 +256,44 @@ bzip2 bundles created when requested https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 ec2region=us-east-1 https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 ec2region=eu-central-1 https://hg.cdn.mozilla.net/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2 https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1 https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1 https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1 -Legacy stream bundles only generated when requested - - $ cd mozilla-central - $ echo legacystream > foo - $ hg commit -m legacystream - $ hg push > /dev/null - $ cd .. - - $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central legacy_stream' --no-upload > /dev/null - $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini - - $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=bundles - 200 - - https://hg.cdn.mozilla.net/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg compression=gzip cdn=true requiresni=true - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=us-west-2 compression=gzip - https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=us-west-1 compression=gzip - https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=us-east-1 compression=gzip - https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=eu-central-1 compression=gzip - https://hg.cdn.mozilla.net/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg stream=revlogv1 cdn=true requiresni=true - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-west-2 stream=revlogv1 - https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-west-1 stream=revlogv1 - https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-east-1 stream=revlogv1 - https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=eu-central-1 stream=revlogv1 - zstd-max bundles created when requested $ cd mozilla-central $ echo ztd-max > foo $ hg commit -m zstd-max $ hg push >/dev/null $ cd .. $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central zstd_max' --no-upload > /dev/null $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles 200 - https://hg.cdn.mozilla.net/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2 - https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1 - https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1 - https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1 - https://hg.cdn.mozilla.net/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2 - https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1 - https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1 - https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1 - https://hg.cdn.mozilla.net/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2 - https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1 - https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1 - https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1 + https://hg.cdn.mozilla.net/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.zstd-max.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1 + https://hg.cdn.mozilla.net/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1 + https://hg.cdn.mozilla.net/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/0a63a7e04e27e4e600286c04bb070171a6e5259d.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1 Generaldelta repos should create zstd-v2, gzip-v2, and streamclone bundles only $ hgmo create-repo generaldelta scm_level_1 (recorded repository creation in replication log) $ hg -q clone ssh://${SSH_SERVER}:${SSH_PORT}/generaldelta $ cd generaldelta $ touch foo
--- a/scripts/generate-hg-s3-bundles +++ b/scripts/generate-hg-s3-bundles @@ -33,25 +33,23 @@ CREATES = [ ('gzip-v2', ['bundle', '-a', '-t', 'gzip-v2'], {'gd': True}), ('bzip2', ['bundle', '-a', '-t', 'bzip2-v1'], {}), # ``zstd`` uses default compression settings and is reasonably fast. # ``zstd-max`` uses the highest available compression settings and is # absurdly slow. But it produces significantly smaller bundles. ('zstd', ['bundle', '-a', '-t', 'zstd-v2'], {'all': True}), ('zstd-max', ['--config', 'experimental.bundlecomplevel=21', 'bundle', '-a', '-t', 'zstd-v2'], {'all': True}), - ('stream-legacy', ['streambundle'], {}), ('packed1', ['streambundle', '--type', 's1'], {}), ('packed1-gd', ['streambundle', '--type', 's1'], {'gd': True}), ] BUNDLECLONE_ORDER = [ ('gzip', 'compression=gzip'), ('bzip2', 'compression=bzip2'), - ('stream-legacy', 'stream=revlogv1'), ] CLONEBUNDLES_ORDER = [ ('zstd-max', 'BUNDLESPEC=zstd-v2'), ('zstd', 'BUNDLESPEC=zstd-v2'), ('gzip-v2', 'BUNDLESPEC=gzip-v2'), ('gzip', 'BUNDLESPEC=gzip-v1'), ('bzip2', 'BUNDLESPEC=bzip2-v1'), @@ -115,17 +113,16 @@ HTML_INDEX = ''' <th>Repository</th> <th>zstd</th> <th>zstd (max)</th> <th>gzip (v1)</th> <th>gzip (v2)</th> <th>bz2 (v1)</th> <th>stream</th> <th>stream (generaldelta)</th> - <th>stream-legacy</th> </tr> %s </table> <p>This page generated at %s.</p> </body> </html> '''.strip() @@ -134,17 +131,16 @@ HTML_ENTRY = ''' <td>{repo}</td> <td class="numeric">{zstd_entry}</td> <td class="numeric">{zstd_max_entry}</td> <td class="numeric">{gzip_entry}</td> <td class="numeric">{gzip_v2_entry}</td> <td class="numeric">{bzip2_entry}</td> <td class="numeric">{packed1_entry}</a></td> <td class="numeric">{packed1_gd_entry}</td> - <td class="numeric">{stream_legacy_entry}</td> </tr> '''.strip() def upload_to_s3(host, bucket_name, local_path, remote_path): """Upload a file to S3.""" attempt = 0 while attempt < 3: @@ -198,23 +194,20 @@ def generate_bundle(repo, temp_path, fin args = [HG_BUNDLE, '--config', 'extensions.vcsreplicator=!', '-R', repo] + extra_args + [temp_path] subprocess.check_call(args) os.rename(temp_path, final_path) def generate_bundles(repo, upload=True, bzip2=False, copyfrom=None, - legacy_stream=False, zstd_max=False): + zstd_max=False): """Generate bundle files for a repository at a path. ``bzip2`` denotes whether to generate bzip2 bundles. - ``legacy_stream`` denotes whether to generate legacy stream bundles - (required for compatibility with the "bundleclone" extension, which - pre-dated Mercurial's built-in support for "clonebundles."). ``zstd_max`` denotes whether to generate zstd bundles with maximum compression. """ # Copy manifest files from the source repository listed. Don't return # anything because we don't need to list bundles since this repo isn't # canonical. if copyfrom: # We assume all paths are pinned from a common root. @@ -289,19 +282,16 @@ def generate_bundles(repo, upload=True, bundles = [] fs = [] with futures.ThreadPoolExecutor(CONCURRENT_THREADS) as e: for t, args, opts in CREATES: if t == 'bzip2' and not bzip2: continue - if t == 'stream-legacy' and not legacy_stream: - continue - # Only generate 1 of zstd or zstd-max since they are redundant. if t == 'zstd' and zstd_max: continue if t == 'zstd-max' and not zstd_max: continue # The "all" opt allows generation for generaldelta and @@ -436,17 +426,17 @@ def generate_index(repos): # Should only be for bundles with copyfrom. if 'gzip' not in p and 'gzip-v2' not in p: print('ignoring repo %s in index because no gzip bundle' % repo) continue opts = {'repo': repo} for k in ('bzip2', 'gzip', 'gzip-v2', 'packed1', 'packed1-gd', - 'stream-legacy', 'zstd', 'zstd-max'): + 'zstd', 'zstd-max'): key = '%s_entry' % k.replace('-', '_') if k in p: opts[key] = '<a href="{path}">{size:,}</a>'.format( path=p[k][0], size=p[k][1], ) opts['basename'] = os.path.basename(p[k][0]) else: