author | Gregory Szorc <gps@mozilla.com> |
Wed, 05 Apr 2017 09:13:36 -0700 | |
changeset 10721 | fa8a20bda4d98df98455f8faaa5bc856463ecce0 |
parent 10720 | e3a0477faab962b4b2e7c512e2cb598135854571 |
child 10722 | 35bb8636c0dcb15a18ffa4c44bd03e2058071e43 |
push id | 1613 |
push user | bmo:gps@mozilla.com |
push date | Fri, 07 Apr 2017 00:08:28 +0000 |
reviewers | glob |
bugs | 1354311 |
hgserver/tests/test-clonebundles.t | file | annotate | diff | comparison | revisions | |
scripts/generate-hg-s3-bundles | file | annotate | diff | comparison | revisions |
--- a/hgserver/tests/test-clonebundles.t +++ b/hgserver/tests/test-clonebundles.t @@ -189,49 +189,23 @@ The copyfrom=x field copies bundles from https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1 https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1 https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2 https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1 https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1 https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1 -bzip2 bundles created when requested +Create a commit for bzip2 to keep hashes stable $ cd mozilla-central $ echo bzip2 > foo $ hg commit -m bzip2 $ hg push >/dev/null $ cd .. - $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central bzip2' --no-upload > /dev/null - $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini - - $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles - 200 - - https://hg.cdn.mozilla.net/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2 - https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1 - https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1 - https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1 - https://hg.cdn.mozilla.net/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2 - https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1 - https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1 - https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1 - https://hg.cdn.mozilla.net/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 REQUIRESNI=true cdn=true - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 ec2region=us-west-2 - https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 ec2region=us-west-1 - https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 ec2region=us-east-1 - https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 ec2region=eu-central-1 - https://hg.cdn.mozilla.net/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2 - https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1 - https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1 - https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1 zstd-max bundles created when requested $ cd mozilla-central $ echo ztd-max > foo $ hg commit -m zstd-max $ hg push >/dev/null $ cd ..
--- a/scripts/generate-hg-s3-bundles +++ b/scripts/generate-hg-s3-bundles @@ -26,17 +26,16 @@ HG_BUNDLE = '/var/hg/venv_hg_pre/bin/hg' PUSH_REPO = '/var/hg/version-control-tools/scripts/push-repo.sh' # The types of bundles to generate. # # Define in order bundles should be listed in manifest. CREATES = [ ('gzip', ['bundle', '-a', '-t', 'gzip-v1'], {}), ('gzip-v2', ['bundle', '-a', '-t', 'gzip-v2'], {'gd': True}), - ('bzip2', ['bundle', '-a', '-t', 'bzip2-v1'], {}), # ``zstd`` uses default compression settings and is reasonably fast. # ``zstd-max`` uses the highest available compression settings and is # absurdly slow. But it produces significantly smaller bundles. Level 20 # (and not higher) is used because it is the largest level supported # by the zstd library in 32-bit processes. ('zstd', ['bundle', '-a', '-t', 'zstd-v2'], {'all': True}), ('zstd-max', ['--config', 'experimental.bundlecomplevel=20', 'bundle', '-a', '-t', 'zstd-v2'], {'all': True}), @@ -44,17 +43,16 @@ CREATES = [ ('packed1-gd', ['debugcreatestreamclonebundle'], {'gd': True}), ] CLONEBUNDLES_ORDER = [ ('zstd-max', 'BUNDLESPEC=zstd-v2'), ('zstd', 'BUNDLESPEC=zstd-v2'), ('gzip-v2', 'BUNDLESPEC=gzip-v2'), ('gzip', 'BUNDLESPEC=gzip-v1'), - ('bzip2', 'BUNDLESPEC=bzip2-v1'), ('packed1-gd', 'BUNDLESPEC=none-packed1;requirements%3Dgeneraldelta%2Crevlogv1'), ('packed1', 'BUNDLESPEC=none-packed1;requirements%3Drevlogv1'), ] # Defines hostname and bucket where uploads should go. HOSTS = ( # We list Oregon (us-west-2) before N. California (us-west-1) because it is # cheaper. @@ -107,17 +105,16 @@ HTML_INDEX = ''' </p> <table border="1"> <tr> <th>Repository</th> <th>zstd</th> <th>zstd (max)</th> <th>gzip (v1)</th> <th>gzip (v2)</th> - <th>bz2 (v1)</th> <th>stream</th> <th>stream (generaldelta)</th> </tr> %s </table> <p>This page generated at %s.</p> </body> </html> @@ -125,17 +122,16 @@ HTML_INDEX = ''' HTML_ENTRY = ''' <tr> <td>{repo}</td> <td class="numeric">{zstd_entry}</td> <td class="numeric">{zstd_max_entry}</td> <td class="numeric">{gzip_entry}</td> <td class="numeric">{gzip_v2_entry}</td> - <td class="numeric">{bzip2_entry}</td> <td class="numeric">{packed1_entry}</a></td> <td class="numeric">{packed1_gd_entry}</td> </tr> '''.strip() def upload_to_s3(host, bucket_name, local_path, remote_path): """Upload a file to S3.""" @@ -190,21 +186,19 @@ def generate_bundle(repo, temp_path, fin """ args = [HG_BUNDLE, '--config', 'extensions.vcsreplicator=!', '-R', repo] + extra_args + [temp_path] subprocess.check_call(args) os.rename(temp_path, final_path) -def generate_bundles(repo, upload=True, bzip2=False, copyfrom=None, - zstd_max=False): +def generate_bundles(repo, upload=True, copyfrom=None, zstd_max=False): """Generate bundle files for a repository at a path. - ``bzip2`` denotes whether to generate bzip2 bundles. ``zstd_max`` denotes whether to generate zstd bundles with maximum compression. """ # Copy manifest files from the source repository listed. Don't return # anything because we don't need to list bundles since this repo isn't # canonical. if copyfrom: # We assume all paths are pinned from a common root. @@ -275,19 +269,16 @@ def generate_bundles(repo, upload=True, # We write to temporary files then move them into place after generation. # This is because an aborted bundle process may result in a partial file, # which may confuse our don't-write-if-file-exists logic. bundles = [] fs = [] with futures.ThreadPoolExecutor(CONCURRENT_THREADS) as e: for t, args, opts in CREATES: - if t == 'bzip2' and not bzip2: - continue - # Only generate 1 of zstd or zstd-max since they are redundant. if t == 'zstd' and zstd_max: continue if t == 'zstd-max' and not zstd_max: continue # The "all" opt allows generation for generaldelta and @@ -401,17 +392,17 @@ def generate_index(repos): # Should only be for bundles with copyfrom. if 'gzip' not in p and 'gzip-v2' not in p: print('ignoring repo %s in index because no gzip bundle' % repo) continue opts = {'repo': repo} - for k in ('bzip2', 'gzip', 'gzip-v2', 'packed1', 'packed1-gd', + for k in ('gzip', 'gzip-v2', 'packed1', 'packed1-gd', 'zstd', 'zstd-max'): key = '%s_entry' % k.replace('-', '_') if k in p: opts[key] = '<a href="{path}">{size:,}</a>'.format( path=p[k][0], size=p[k][1], ) opts['basename'] = os.path.basename(p[k][0])