author | Gregory Szorc <gps@mozilla.com> |
Fri, 27 Jan 2017 15:35:10 -0800 | |
changeset 10289 | 3b630e5f5407559fe1d68372089a17af2b1c4881 |
parent 10288 | a65c30536781cd049ba2186273c5e151f4c0da5f |
child 10290 | 6c17418adc339e9a8cc2e6f627399e3e6ca7c38a |
push id | 1505 |
push user | bmo:gps@mozilla.com |
push date | Wed, 01 Feb 2017 19:22:14 +0000 |
reviewers | glob |
bugs | 1331084 |
hgserver/tests/test-clonebundles.t | file | annotate | diff | comparison | revisions | |
scripts/generate-hg-s3-bundles | file | annotate | diff | comparison | revisions |
--- a/hgserver/tests/test-clonebundles.t +++ b/hgserver/tests/test-clonebundles.t @@ -254,16 +254,103 @@ Legacy stream bundles only generated whe https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=us-east-1 compression=gzip https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=eu-central-1 compression=gzip https://hg.cdn.mozilla.net/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg stream=revlogv1 cdn=true requiresni=true https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-west-2 stream=revlogv1 https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-west-1 stream=revlogv1 https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-east-1 stream=revlogv1 https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=eu-central-1 stream=revlogv1 +zstd bundles created when requested + + $ cd mozilla-central + $ echo ztd > foo + $ hg commit -m zstd + $ hg push >/dev/null + $ cd .. + $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central zstd' --no-upload > /dev/null + $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini + + $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles + 200 + + https://hg.cdn.mozilla.net/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1 + https://hg.cdn.mozilla.net/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1 + https://hg.cdn.mozilla.net/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1 + +zstd-max bundles created when requested + + $ cd mozilla-central + $ echo ztd-max > foo + $ hg commit -m zstd-max + $ hg push >/dev/null + $ cd .. + $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central zstd_max' --no-upload > /dev/null + $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini + + $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles + 200 + + https://hg.cdn.mozilla.net/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.zstd-max.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1 + https://hg.cdn.mozilla.net/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1 + https://hg.cdn.mozilla.net/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1 + +only zstd-max created if both zstd and zstd-max set + + $ cd mozilla-central + $ echo ztd_and_max > foo + $ hg commit -m zstd-and-max + $ hg push >/dev/null + $ cd .. + $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central zstd zstd_max' --no-upload > /dev/null + $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini + + $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles + 200 + + https://hg.cdn.mozilla.net/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.zstd-max.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1 + https://hg.cdn.mozilla.net/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1 + https://hg.cdn.mozilla.net/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true + https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2 + https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1 + https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1 + https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1 + Generaldelta repos should create gzip-v2 and streamclone bundles only $ hgmo create-repo generaldelta scm_level_1 --generaldelta (recorded repository creation in replication log) $ hg -q clone ssh://${SSH_SERVER}:${SSH_PORT}/generaldelta $ cd generaldelta $ touch foo $ hg -q commit -A -m initial
--- a/scripts/generate-hg-s3-bundles +++ b/scripts/generate-hg-s3-bundles @@ -25,28 +25,36 @@ PUSH_REPO = '/var/hg/version-control-too # The types of bundles to generate. # # Define in order bundles should be listed in manifest. CREATES = [ ('gzip', ['bundle', '-a', '-t', 'gzip-v1'], {}), ('gzip-v2', ['bundle', '-a', '-t', 'gzip-v2'], {'gd': True}), ('bzip2', ['bundle', '-a', '-t', 'bzip2-v1'], {}), + # ``zstd`` uses default compression settings and is reasonably fast. + # ``zstd-max`` uses the highest available compression settings and is + # absurdly slow. But it produces significantly smaller bundles. + ('zstd', ['bundle', '-a', '-t', 'zstd-v2'], {'all': True}), + ('zstd-max', ['--config', 'experimental.bundlecomplevel=22', + 'bundle', '-a', '-t', 'zstd-v2'], {'all': True}), ('stream-legacy', ['streambundle'], {}), ('packed1', ['streambundle', '--type', 's1'], {}), ('packed1-gd', ['streambundle', '--type', 's1'], {'gd': True}), ] BUNDLECLONE_ORDER = [ ('gzip', 'compression=gzip'), ('bzip2', 'compression=bzip2'), ('stream-legacy', 'stream=revlogv1'), ] CLONEBUNDLES_ORDER = [ + ('zstd-max', 'BUNDLESPEC=zstd-v2'), + ('zstd', 'BUNDLESPEC=zstd-v2'), ('gzip-v2', 'BUNDLESPEC=gzip-v2'), ('gzip', 'BUNDLESPEC=gzip-v1'), ('bzip2', 'BUNDLESPEC=bzip2-v1'), ('packed1-gd', 'BUNDLESPEC=none-packed1;requirements%3Dgeneraldelta%2Crevlogv1'), ('packed1', 'BUNDLESPEC=none-packed1;requirements%3Drevlogv1'), ] # Defines hostname and bucket where uploads should go. @@ -98,16 +106,18 @@ HTML_INDEX = ''' </p> <p> For more, see <a href="https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/bundleclone.html">the official docs</a>. </p> <table border="1"> <tr> <th>Repository</th> + <th>zstd</th> + <th>zstd (max)</th> <th>gzip (v1)</th> <th>gzip (v2)</th> <th>bz2 (v1)</th> <th>stream</th> <th>stream (generaldelta)</th> <th>stream-legacy</th> </tr> %s @@ -115,16 +125,18 @@ HTML_INDEX = ''' <p>This page generated at %s.</p> </body> </html> '''.strip() HTML_ENTRY = ''' <tr> <td>{repo}</td> + <td class="numeric">{zstd_entry}</td> + <td class="numeric">{zstd_max_entry}</td> <td class="numeric">{gzip_entry}</td> <td class="numeric">{gzip_v2_entry}</td> <td class="numeric">{bzip2_entry}</td> <td class="numeric">{packed1_entry}</a></td> <td class="numeric">{packed1_gd_entry}</td> <td class="numeric">{stream_legacy_entry}</td> </tr> '''.strip() @@ -167,23 +179,26 @@ def generate_bundle(repo, temp_path, fin args = [HG_BUNDLE, '--config', 'extensions.vcsreplicator=!', '-R', repo] + extra_args + [temp_path] subprocess.check_call(args) os.rename(temp_path, final_path) def generate_bundles(repo, upload=True, bzip2=False, copyfrom=None, - legacy_stream=False): + legacy_stream=False, zstd=False, zstd_max=False): """Generate bundle files for a repository at a path. ``bzip2`` denotes whether to generate bzip2 bundles. ``legacy_stream`` denotes whether to generate legacy stream bundles (required for compatibility with the "bundleclone" extension, which pre-dated Mercurial's built-in support for "clonebundles."). + ``zstd`` denotes whether to generate zstd bundles. + ``zstd_max`` denotes whether to generate zstd bundles with maximum + compression. """ # Copy manifest files from the source repository listed. Don't return # anything because we don't need to list bundles since this repo isn't # canonical. if copyfrom: # We assume all paths are pinned from a common root. assert not os.path.isabs(copyfrom) source_repo = os.path.join('/repo/hg/mozilla', copyfrom) @@ -259,24 +274,34 @@ def generate_bundles(repo, upload=True, with futures.ThreadPoolExecutor(CONCURRENT_THREADS) as e: for t, args, opts in CREATES: if t == 'bzip2' and not bzip2: continue if t == 'stream-legacy' and not legacy_stream: continue - # When the repo is generaldelta, only allow generation of - # bundles that are compatible with generaldelta. If we don't do - # this, CPU usage could blow up attempting to convert a - # generaldelta repo to a non-generaldelta compatible bundle. - if generaldelta and not opts.get('gd'): + # Only generate 1 of zstd or zstd-max since they are redundant. + if t == 'zstd' and (not zstd or zstd_max): + continue + + if t == 'zstd-max' and not zstd_max: continue - elif not generaldelta and opts.get('gd'): - continue + + # The "all" opt allows generation for generaldelta and + # non-generaldelta repos. If not set, we key off the generaldelta + # state and the "gd" key to determine whether to generate. What + # we're trying to prevent is v1 bundle types being generated for + # generaldelta repos, which could require a lot of CPU usage to + # recalculate deltas. + if not opts.get('all'): + if generaldelta and not opts.get('gd'): + continue + elif not generaldelta and opts.get('gd'): + continue final_path, remote_path = bundle_paths(bundle_path, repo, tip, t) temp_path = '%s.tmp' % final_path # Record that this bundle is relevant. bundles.append((t, final_path, remote_path)) if os.path.exists(final_path): @@ -392,17 +417,18 @@ def generate_index(repos): # Should only be for bundles with copyfrom. if 'gzip' not in p and 'gzip-v2' not in p: print('ignoring repo %s in index because no gzip bundle' % repo) continue opts = {'repo': repo} - for k in ('bzip2', 'gzip', 'gzip-v2', 'packed1', 'packed1-gd', 'stream-legacy'): + for k in ('bzip2', 'gzip', 'gzip-v2', 'packed1', 'packed1-gd', + 'stream-legacy', 'zstd', 'zstd-max'): key = '%s_entry' % k.replace('-', '_') if k in p: opts[key] = '<a href="{path}">{size:,}</a>'.format( path=p[k][0], size=p[k][1], ) opts['basename'] = os.path.basename(p[k][0]) else: