author | Gregory Szorc <gps@mozilla.com> |
Fri, 31 Mar 2017 10:45:18 -0700 | |
changeset 10599 | 19f058e492c1b74aeec543e7f36593733515d278 |
parent 10598 | 11f958e4114b972c096ebccffcb24c008a3f9560 |
child 10600 | a2ffb994cc579262d075733145b6d5560f08f622 |
push id | 1596 |
push user | bmo:gps@mozilla.com |
push date | Fri, 31 Mar 2017 21:38:22 +0000 |
reviewers | glob |
bugs | 1352494 |
hgserver/tests/test-clonebundles.t | file | annotate | diff | comparison | revisions | |
scripts/generate-hg-s3-bundles | file | annotate | diff | comparison | revisions |
--- a/hgserver/tests/test-clonebundles.t +++ b/hgserver/tests/test-clonebundles.t @@ -102,24 +102,16 @@ Cloning will fetch bundle HTTP error fetching bundle: HTTP Error 403: Forbidden falling back to normal clone requesting all changes adding changesets adding manifests adding file changes added 1 changesets with 1 changes to 1 files -#else - - $ hg --config extensions.bundleclone=$TESTDIR/hgext/bundleclone clone -U ${HGWEB_0_URL}mozilla-central bundleclone - downloading bundle https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg - abort: HTTP error fetching bundle: HTTP Error 403: Forbidden - (consider contacting the server operator if this error persists) - [255] - #endif The full manifest is fetched normally $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles 200 https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true @@ -133,41 +125,26 @@ The full manifest is fetched normally https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1 https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1 https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2 https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1 https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1 https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1 - $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=bundles - 200 - - https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg compression=gzip cdn=true requiresni=true - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=us-west-2 compression=gzip - https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=us-west-1 compression=gzip - https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=us-east-1 compression=gzip - https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=eu-central-1 compression=gzip - Fetching with an AWS us-west-2 IP will limit to same region URLs $ http --no-headers --request-header "X-Cluster-Client-IP: 54.245.168.15" ${HGWEB_0_URL}mozilla-central?cmd=clonebundles 200 https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2 https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2 https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2 - $ http --no-headers --request-header "X-Cluster-Client-IP: 54.245.168.15" ${HGWEB_0_URL}mozilla-central?cmd=bundles - 200 - - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=us-west-2 compression=gzip - - Fetching with an AWS IP from "other" region returns full list $ http --no-headers --request-header "X-Cluster-Client-IP: 54.248.220.10" ${HGWEB_0_URL}mozilla-central?cmd=clonebundles 200 https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2 https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1 @@ -179,38 +156,28 @@ Fetching with an AWS IP from "other" reg https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1 https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1 https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2 https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1 https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1 https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1 - $ http --no-headers --request-header "X-Cluster-Client-IP: 54.248.220.10" ${HGWEB_0_URL}mozilla-central?cmd=bundles - 200 - - https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg compression=gzip cdn=true requiresni=true - https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=us-west-2 compression=gzip - https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=us-west-1 compression=gzip - https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=us-east-1 compression=gzip - https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=eu-central-1 compression=gzip - The copyfrom=x field copies bundles from another repo $ hgmo create-repo try scm_level_1 --no-generaldelta (recorded repository creation in replication log) $ hg -q clone ssh://${SSH_SERVER}:${SSH_PORT}/try $ cd try $ touch foo $ hg -q commit -A -m initial $ hg push > /dev/null $ cd .. $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles --no-upload 'try copyfrom=mozilla-central' - copying /repo/hg/mozilla/mozilla-central/.hg/bundleclone.manifest -> /repo/hg/mozilla/try/.hg/bundleclone.manifest copying /repo/hg/mozilla/mozilla-central/.hg/clonebundles.manifest -> /repo/hg/mozilla/try/.hg/clonebundles.manifest ignoring repo try in index because no gzip bundle $ http --no-headers ${HGWEB_0_URL}try?cmd=clonebundles 200 https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2 https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
--- a/scripts/generate-hg-s3-bundles +++ b/scripts/generate-hg-s3-bundles @@ -37,21 +37,16 @@ CREATES = [ # absurdly slow. But it produces significantly smaller bundles. ('zstd', ['bundle', '-a', '-t', 'zstd-v2'], {'all': True}), ('zstd-max', ['--config', 'experimental.bundlecomplevel=21', 'bundle', '-a', '-t', 'zstd-v2'], {'all': True}), ('packed1', ['streambundle', '--type', 's1'], {}), ('packed1-gd', ['streambundle', '--type', 's1'], {'gd': True}), ] -BUNDLECLONE_ORDER = [ - ('gzip', 'compression=gzip'), - ('bzip2', 'compression=bzip2'), -] - CLONEBUNDLES_ORDER = [ ('zstd-max', 'BUNDLESPEC=zstd-v2'), ('zstd', 'BUNDLESPEC=zstd-v2'), ('gzip-v2', 'BUNDLESPEC=gzip-v2'), ('gzip', 'BUNDLESPEC=gzip-v1'), ('bzip2', 'BUNDLESPEC=bzip2-v1'), ('packed1-gd', 'BUNDLESPEC=none-packed1;requirements%3Dgeneraldelta%2Crevlogv1'), ('packed1', 'BUNDLESPEC=none-packed1;requirements%3Drevlogv1'), @@ -209,24 +204,23 @@ def generate_bundles(repo, upload=True, # Copy manifest files from the source repository listed. Don't return # anything because we don't need to list bundles since this repo isn't # canonical. if copyfrom: # We assume all paths are pinned from a common root. assert not os.path.isabs(copyfrom) source_repo = os.path.join('/repo/hg/mozilla', copyfrom) dest_repo = os.path.join('/repo/hg/mozilla', repo) - for p in ('bundleclone.manifest', 'clonebundles.manifest'): - source = os.path.join(source_repo, '.hg', p) - dest = os.path.join(dest_repo, '.hg', p) + source = os.path.join(source_repo, '.hg', 'clonebundles.manifest') + dest = os.path.join(dest_repo, '.hg', 'clonebundles.manifest') - print('copying %s -> %s' % (source, dest)) + print('copying %s -> %s' % (source, dest)) - # copy2 copies metadata. - shutil.copy2(source, dest) + # copy2 copies metadata. + shutil.copy2(source, dest) # Replicate manifest to mirrors. subprocess.check_call([PUSH_REPO], cwd=dest_repo) return {} assert not os.path.isabs(repo) repo_full = os.path.join('/repo/hg/mozilla', repo) @@ -360,59 +354,39 @@ def generate_bundles(repo, upload=True, # Now assemble a manifest listing each bundle. paths = {} for t, final_path, remote_path in bundles: paths[t] = (remote_path, os.path.getsize(final_path)) bundle_types = set(t[0] for t in bundles) - bundleclone_manifest = [] - for t, params in BUNDLECLONE_ORDER: - if t not in bundle_types: - continue - - final_path, remote_path = bundle_paths(bundle_path, repo, tip, t) - - bundleclone_manifest.append('%s/%s %s cdn=true requiresni=true' % ( - CDN, remote_path, params)) - - for host, bucket, name in HOSTS: - entry = 'https://%s/%s/%s ec2region=%s %s' % ( - host, bucket, remote_path, name, params) - bundleclone_manifest.append(entry) - clonebundles_manifest = [] for t, params in CLONEBUNDLES_ORDER: if t not in bundle_types: continue final_path, remote_path = bundle_paths(bundle_path, repo, tip, t) clonebundles_manifest.append('%s/%s %s REQUIRESNI=true cdn=true' % ( CDN, remote_path, params)) for host, bucket, name in HOSTS: entry = 'https://%s/%s/%s %s ec2region=%s' % ( host, bucket, remote_path, params, name) clonebundles_manifest.append(entry) - bundleclone_path = os.path.join(repo_full, '.hg', 'bundleclone.manifest') clonebundles_path = os.path.join(repo_full, '.hg', 'clonebundles.manifest') - with open(bundleclone_path, 'wb') as fh: - fh.write('\n'.join(bundleclone_manifest)) with open(clonebundles_path, 'wb') as fh: fh.write('\n'.join(clonebundles_manifest)) # Ensure manifest is owned by same user who owns repo and has sane # permissions. # TODO we can't do this yet since the "hg" user isn't a member of the # scm_* groups. - #os.chown(bundleclone_path, uid, gid) #os.chown(clonebundles_path, uid, gid) - os.chmod(bundleclone_path, 0664) os.chmod(clonebundles_path, 0664) # Replicate manifest to mirrors. subprocess.check_call([HG, 'replicatesync'], cwd=repo_full) return paths