hgserver: generate zstd bundles by default (bug 1336190); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Fri, 03 Feb 2017 19:13:46 -0800
changeset 10312 3ff92adb242468e5c292e48a1bcb29433d488f77
parent 10308 6d618ac81fcf64602fa2e90a84f69dd4069f07cc
push id1510
push userbmo:gps@mozilla.com
push dateSat, 04 Feb 2017 03:14:23 +0000
reviewersglob
bugs1336190
hgserver: generate zstd bundles by default (bug 1336190); r?glob zstd-max bundles for mozilla-central, mozilla-unified, and other repos appear to be working just fine. So let's enable zstd bundle generation on all repos by default. zstd bundles will be preferred over other bundles (unless in an EC2 region, which continues to favor stream clone bundles). If the zstd_max option is present, we will generate zstd bundles with maximum compression instead of the default compression. MozReview-Commit-ID: 8FKZMMFZYKM
hgserver/tests/test-clonebundles.t
scripts/generate-hg-s3-bundles
--- a/hgserver/tests/test-clonebundles.t
+++ b/hgserver/tests/test-clonebundles.t
@@ -27,25 +27,30 @@ Ensure bundle creation script raises dur
   OSError: [Errno 2] No such file or directory: '/repo/hg/mozilla/missing/.hg'
   [1]
 
 And raises during upload since we don't have credentials in the test env
 
   $ hgmo exec hgssh sudo -u hg SINGLE_THREADED=1 /var/hg/venv_tools/bin/python -u /var/hg/version-control-tools/scripts/generate-hg-s3-bundles mozilla-central
   tip is 77538e1ce4bec5f7aac58a7ceca2da0e38e90a72
   1 changesets found
+  1 changesets found
   writing 328 bytes for 3 files
   bundle requirements: revlogv1
   uploading to s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg
+  uploading to s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg
   uploading to s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg
   uploading to s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg
+  uploading to s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg
   uploading to s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg
   uploading to s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg
+  uploading to s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg
   uploading to s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg
   uploading to s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg
+  uploading to s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg
   uploading to s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg
   Traceback (most recent call last):
     File "/var/hg/version-control-tools/scripts/generate-hg-s3-bundles", line \d+, in <module> (re)
       paths[repo] = generate_bundles(repo, upload=upload, **opts)
     File "/var/hg/version-control-tools/scripts/generate-hg-s3-bundles", line \d+, in generate_bundles (re)
       f.result()
     File "/var/hg/venv_tools/lib/python2.7/site-packages/concurrent/futures/_base.py", line \d+, in result (re)
       return self.__get_result()
@@ -70,16 +75,17 @@ The manifest should be empty because the
   
 
 An index.html and bundles.json document should be produced
 
   $ hgmo exec hgssh sudo -u hg SINGLE_THREADED=1 /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles mozilla-central --no-upload
   wrote synchronization message into replication log
   tip is 77538e1ce4bec5f7aac58a7ceca2da0e38e90a72
   bundle already exists, skipping: /repo/hg/bundles/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg
+  bundle already exists, skipping: /repo/hg/bundles/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg
   bundle already exists, skipping: /repo/hg/bundles/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg
   $ hgmo exec hgssh ls /repo/hg/bundles
   bundles.json
   index.html
   mozilla-central
   repos
 
 Create a clonebundles manifest
@@ -111,16 +117,21 @@ Cloning will fetch bundle
 
 #endif
 
 The full manifest is fetched normally
 
   $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles
   200
   
+  https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
   https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1
   https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1
   https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1
@@ -137,30 +148,36 @@ The full manifest is fetched normally
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=eu-central-1 compression=gzip
 
 Fetching with an AWS us-west-2 IP will limit to same region URLs
 
   $ http --no-headers --request-header "X-Cluster-Client-IP: 54.245.168.15" ${HGWEB_0_URL}mozilla-central?cmd=clonebundles
   200
   
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2
   
 
   $ http --no-headers --request-header "X-Cluster-Client-IP: 54.245.168.15" ${HGWEB_0_URL}mozilla-central?cmd=bundles
   200
   
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=us-west-2 compression=gzip
   
 
 Fetching with an AWS IP from "other" region returns full list
 
   $ http --no-headers --request-header "X-Cluster-Client-IP: 54.248.220.10" ${HGWEB_0_URL}mozilla-central?cmd=clonebundles
   200
   
+  https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
   https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1
   https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1
   https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1
@@ -189,16 +206,21 @@ The copyfrom=x field copies bundles from
 
   $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles --no-upload 'try copyfrom=mozilla-central'
   copying /repo/hg/mozilla/mozilla-central/.hg/bundleclone.manifest -> /repo/hg/mozilla/try/.hg/bundleclone.manifest
   copying /repo/hg/mozilla/mozilla-central/.hg/clonebundles.manifest -> /repo/hg/mozilla/try/.hg/clonebundles.manifest
   ignoring repo try in index because no gzip bundle
   $ http --no-headers ${HGWEB_0_URL}try?cmd=clonebundles
   200
   
+  https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
   https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1
   https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1
   https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1
@@ -213,16 +235,21 @@ bzip2 bundles created when requested
   $ hg push >/dev/null
   $ cd ..
   $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central bzip2' --no-upload > /dev/null
   $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini
 
   $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles
   200
   
+  https://hg.cdn.mozilla.net/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
   https://hg.cdn.mozilla.net/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1
   https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1
   https://hg.cdn.mozilla.net/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/70b5a2a0a3ef0e272c12bb90a28c0fb534724368.bzip2.hg BUNDLESPEC=bzip2-v1 ec2region=us-west-1
@@ -254,116 +281,63 @@ Legacy stream bundles only generated whe
   https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=us-east-1 compression=gzip
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.gzip.hg ec2region=eu-central-1 compression=gzip
   https://hg.cdn.mozilla.net/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg stream=revlogv1 cdn=true requiresni=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-west-2 stream=revlogv1
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-west-1 stream=revlogv1
   https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=us-east-1 stream=revlogv1
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/4123d33678728ad98862cdac91d6a3f447a0271a.stream-legacy.hg ec2region=eu-central-1 stream=revlogv1
 
-zstd bundles created when requested
-
-  $ cd mozilla-central
-  $ echo ztd > foo
-  $ hg commit -m zstd
-  $ hg push >/dev/null
-  $ cd ..
-  $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central zstd' --no-upload > /dev/null
-  $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini
-
-  $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles
-  200
-  
-  https://hg.cdn.mozilla.net/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
-  https://hg.cdn.mozilla.net/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1
-  https://hg.cdn.mozilla.net/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/04de8c7145565ffe061b878c1ec4c197654513ab.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1
-
 zstd-max bundles created when requested
 
   $ cd mozilla-central
   $ echo ztd-max > foo
   $ hg commit -m zstd-max
   $ hg push >/dev/null
   $ cd ..
   $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central zstd_max' --no-upload > /dev/null
   $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini
 
   $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles
   200
   
-  https://hg.cdn.mozilla.net/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.zstd-max.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
-  https://hg.cdn.mozilla.net/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1
-  https://hg.cdn.mozilla.net/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/3c010f45b766cb8c9f24041a078c58576a22fbd8.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1
-
-only zstd-max created if both zstd and zstd-max set
+  https://hg.cdn.mozilla.net/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
+  https://hg.cdn.mozilla.net/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1
+  https://hg.cdn.mozilla.net/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1
 
-  $ cd mozilla-central
-  $ echo ztd_and_max > foo
-  $ hg commit -m zstd-and-max
-  $ hg push >/dev/null
-  $ cd ..
-  $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'mozilla-central zstd zstd_max' --no-upload > /dev/null
-  $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini
-
-  $ http --no-headers ${HGWEB_0_URL}mozilla-central?cmd=clonebundles
-  200
-  
-  https://hg.cdn.mozilla.net/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.zstd-max.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.zstd-max.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
-  https://hg.cdn.mozilla.net/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.gzip.hg BUNDLESPEC=gzip-v1 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.gzip.hg BUNDLESPEC=gzip-v1 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.gzip.hg BUNDLESPEC=gzip-v1 ec2region=eu-central-1
-  https://hg.cdn.mozilla.net/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true
-  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
-  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1
-  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1
-  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/e5b830f6d033ca863c8a42952b5c2733305a8eed.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1
-
-Generaldelta repos should create gzip-v2 and streamclone bundles only
+Generaldelta repos should create zstd-v2, gzip-v2, and streamclone bundles only
 
   $ hgmo create-repo generaldelta scm_level_1 --generaldelta
   (recorded repository creation in replication log)
   $ hg -q clone ssh://${SSH_SERVER}:${SSH_PORT}/generaldelta
   $ cd generaldelta
   $ touch foo
   $ hg -q commit -A -m initial
   $ hg push > /dev/null
   $ cd ..
 
   $ hgmo exec hgssh sudo -u hg /var/hg/venv_tools/bin/python /var/hg/version-control-tools/scripts/generate-hg-s3-bundles 'generaldelta' --no-upload > /dev/null
   $ hgmo exec hgssh cat /repo/hg/mozilla/generaldelta/.hg/clonebundles.manifest
+  https://hg.cdn.mozilla.net/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
   https://hg.cdn.mozilla.net/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip-v2.hg BUNDLESPEC=gzip-v2 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip-v2.hg BUNDLESPEC=gzip-v2 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip-v2.hg BUNDLESPEC=gzip-v2 ec2region=us-west-1
   https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip-v2.hg BUNDLESPEC=gzip-v2 ec2region=us-east-1
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip-v2.hg BUNDLESPEC=gzip-v2 ec2region=eu-central-1
   https://hg.cdn.mozilla.net/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1-gd.hg BUNDLESPEC=none-packed1;requirements%3Dgeneraldelta%2Crevlogv1 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1-gd.hg BUNDLESPEC=none-packed1;requirements%3Dgeneraldelta%2Crevlogv1 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/generaldelta/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.packed1-gd.hg BUNDLESPEC=none-packed1;requirements%3Dgeneraldelta%2Crevlogv1 ec2region=us-west-1
--- a/scripts/generate-hg-s3-bundles
+++ b/scripts/generate-hg-s3-bundles
@@ -179,24 +179,23 @@ def generate_bundle(repo, temp_path, fin
     args = [HG_BUNDLE,
             '--config', 'extensions.vcsreplicator=!',
             '-R', repo] + extra_args + [temp_path]
     subprocess.check_call(args)
     os.rename(temp_path, final_path)
 
 
 def generate_bundles(repo, upload=True, bzip2=False, copyfrom=None,
-                     legacy_stream=False, zstd=False, zstd_max=False):
+                     legacy_stream=False, zstd_max=False):
     """Generate bundle files for a repository at a path.
 
     ``bzip2`` denotes whether to generate bzip2 bundles.
     ``legacy_stream`` denotes whether to generate legacy stream bundles
     (required for compatibility with the "bundleclone" extension, which
     pre-dated Mercurial's built-in support for "clonebundles.").
-    ``zstd`` denotes whether to generate zstd bundles.
     ``zstd_max`` denotes whether to generate zstd bundles with maximum
     compression.
     """
     # Copy manifest files from the source repository listed. Don't return
     # anything because we don't need to list bundles since this repo isn't
     # canonical.
     if copyfrom:
         # We assume all paths are pinned from a common root.
@@ -275,17 +274,17 @@ def generate_bundles(repo, upload=True, 
         for t, args, opts in CREATES:
             if t == 'bzip2' and not bzip2:
                 continue
 
             if t == 'stream-legacy' and not legacy_stream:
                 continue
 
             # Only generate 1 of zstd or zstd-max since they are redundant.
-            if t == 'zstd' and (not zstd or zstd_max):
+            if t == 'zstd' and zstd_max:
                 continue
 
             if t == 'zstd-max' and not zstd_max:
                 continue
 
             # The "all" opt allows generation for generaldelta and
             # non-generaldelta repos. If not set, we key off the generaldelta
             # state and the "gd" key to determine whether to generate. What