hgserver: create repos as generaldelta by default (bug 1351848); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Wed, 29 Mar 2017 17:08:46 -0700
changeset 10546 59663fa84d31b4105cfcde046cfb462f3ba406a7
parent 10544 5be525a6578fba4156d506fbcec7223065c02c35
child 10547 0425e163637e9aecaee2e407ee0d27b323742750
push id1592
push userbmo:gps@mozilla.com
push dateThu, 30 Mar 2017 00:44:12 +0000
reviewersglob
bugs1351848
hgserver: create repos as generaldelta by default (bug 1351848); r?glob This commit changes the `create-repo` helper script to create repos as generaldelta by default. The --generaldelta argument to the script has been invented to --no-generaldelta and various uses of this argument have been updated. There is some minor test fallout from this change. The only test fallout we should care about are tests explicitly testing generaldelta versus non-generaldelta. I believe I took care to preserve the existing behavior of these tests. MozReview-Commit-ID: 9MAAMhidxmd
ansible/roles/docker-hg-ssh/files/create-repo
hgserver/tests/test-clonebundles.t
hgserver/tests/test-obsolescence.t
pylib/vcsreplicator/tests/test-changegroup-replicate.t
pylib/vcsreplicator/tests/test-hgrc-replicate.t
pylib/vcsreplicator/tests/test-init.t
pylib/vcsreplicator/tests/test-pushaggregator.t
pylib/vcsreplicator/tests/test-sync.t
testing/vcttesting/hgmo.py
testing/vcttesting/hgmo_mach_commands.py
--- a/ansible/roles/docker-hg-ssh/files/create-repo
+++ b/ansible/roles/docker-hg-ssh/files/create-repo
@@ -9,27 +9,29 @@ REPO=$1
 GROUP=$2
 
 for arg in "$@"
 do
     case $arg in
         --non-publishing)
         NONPUBLISHING=1
         ;;
-        --generaldelta)
-        GENERALDELTA=1
+        --no-generaldelta)
+        NO_GENERALDELTA=1
         ;;
         *)
         ;;
     esac
 done
 
 INITFLAGS=
-if [ -n "${GENERALDELTA}" ]; then
-  INITFLAGS="--config format.generaldelta=true"
+if [ -n "${NO_GENERALDELTA}" ]; then
+  INITFLAGS="--config format.usegeneraldelta=false"
+else
+  INITFLAGS="--config format.usegeneraldelta=true"
 fi
 
 /var/hg/venv_tools/bin/hg init /repo/hg/mozilla/${REPO} ${INITFLAGS}
 chown -R hg:${GROUP} /repo/hg/mozilla/${REPO}
 find /repo/hg/mozilla/${REPO} | xargs chmod g+w
 find /repo/hg/mozilla/${REPO} -type d | xargs chmod g+s
 
 if [ -n "${NONPUBLISHING}" ]; then
--- a/hgserver/tests/test-clonebundles.t
+++ b/hgserver/tests/test-clonebundles.t
@@ -1,17 +1,17 @@
 #require hgmodocker
 
   $ . $TESTDIR/hgserver/tests/helpers.sh
   $ hgmoenv
   $ standarduser
 
 Create and seed repository
 
-  $ hgmo create-repo mozilla-central scm_level_1
+  $ hgmo create-repo mozilla-central scm_level_1 --no-generaldelta
   (recorded repository creation in replication log)
 
   $ hg clone ssh://${SSH_SERVER}:${SSH_PORT}/mozilla-central > /dev/null
   $ cd mozilla-central
   $ touch foo
   $ hg -q commit -A -m initial
   $ hg push > /dev/null
   $ cd ..
@@ -190,17 +190,17 @@ Fetching with an AWS IP from "other" reg
   https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg compression=gzip cdn=true requiresni=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=us-west-2 compression=gzip
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=us-west-1 compression=gzip
   https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=us-east-1 compression=gzip
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.gzip.hg ec2region=eu-central-1 compression=gzip
 
 The copyfrom=x field copies bundles from another repo
 
-  $ hgmo create-repo try scm_level_1
+  $ hgmo create-repo try scm_level_1 --no-generaldelta
   (recorded repository creation in replication log)
   $ hg -q clone ssh://${SSH_SERVER}:${SSH_PORT}/try
   $ cd try
   $ touch foo
   $ hg -q commit -A -m initial
   $ hg push > /dev/null
   $ cd ..
 
@@ -312,17 +312,17 @@ zstd-max bundles created when requested
   https://hg.cdn.mozilla.net/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 REQUIRESNI=true cdn=true
   https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-2
   https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-west-1
   https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=us-east-1
   https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/934273ae8830f7bf6f12950ba87e02185a177467.packed1.hg BUNDLESPEC=none-packed1;requirements%3Drevlogv1 ec2region=eu-central-1
 
 Generaldelta repos should create zstd-v2, gzip-v2, and streamclone bundles only
 
-  $ hgmo create-repo generaldelta scm_level_1 --generaldelta
+  $ hgmo create-repo generaldelta scm_level_1
   (recorded repository creation in replication log)
   $ hg -q clone ssh://${SSH_SERVER}:${SSH_PORT}/generaldelta
   $ cd generaldelta
   $ touch foo
   $ hg -q commit -A -m initial
   $ hg push > /dev/null
   $ cd ..
 
--- a/hgserver/tests/test-obsolescence.t
+++ b/hgserver/tests/test-obsolescence.t
@@ -302,17 +302,17 @@ and we're not in the allowed user list
 
   $ hgmo exec hgssh /var/hg/venv_pash/bin/hg -R /repo/hg/mozilla/integration/autoland debugobsolete
 
 hgweb advertise marker exchange
 
   $ http --no-headers "${HGWEB_0_URL}integration/autoland?cmd=capabilities"
   200
   
-  lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch stream bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Aobsmarkers%3DV0%2CV1%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps unbundle=HG10GZ,HG10BZ,HG10UN httpheader=6144 pushlog
+  lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Aobsmarkers%3DV0%2CV1%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps unbundle=HG10GZ,HG10BZ,HG10UN httpheader=6144 pushlog
 
 Allow this user to send obsolescence markers (since the per-repo hgrc will get replicated
 and take precedence on the mirror, we need to add the allowed user from the replication
 processes on both server and mirror)
 
   $ hgmo exec hgssh /set-hgrc-option integration/autoland obshacks obsolescenceexchangeusers "user@example.com,vcs-sync@mozilla.com,hg"
   $ hgmo exec hgssh /var/hg/venv_pash/bin/hg -R /repo/hg/mozilla/integration/autoland replicatehgrc
   recorded hgrc in replication log
@@ -347,13 +347,13 @@ And they should get replicated to mirror
   5fb779ae39de4af3229a53c35d46117e98fb5f83 d57129f00b2f329fc2cf3371a0c28796bcfbde1c 0 (*) {'user': 'Test User <someone@example.com>'} (glob)
   d57129f00b2f329fc2cf3371a0c28796bcfbde1c 9e2d548e5f1f94b9172cfeb77b53f5943722b594 0 (*) {'user': 'Test User <someone@example.com>'} (glob)
 
 hgweb should still advertise marker exchange
 
   $ http --no-headers "${HGWEB_0_URL}integration/autoland?cmd=capabilities"
   200
   
-  lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch stream bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Aobsmarkers%3DV0%2CV1%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps unbundle=HG10GZ,HG10BZ,HG10UN httpheader=6144 pushlog
+  lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Aobsmarkers%3DV0%2CV1%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps unbundle=HG10GZ,HG10BZ,HG10UN httpheader=6144 pushlog
 
   $ cd ..
 
   $ hgmo clean
--- a/pylib/vcsreplicator/tests/test-changegroup-replicate.t
+++ b/pylib/vcsreplicator/tests/test-changegroup-replicate.t
@@ -27,17 +27,17 @@ Pushing the initial commit will result i
   remote: recorded push in pushlog
   remote: 
   remote: View your change here:
   remote:   https://hg.mozilla.org/mozilla-central/rev/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72
   remote: recorded changegroup in replication log in \d\.\d+s (re)
 
   $ consumer --dump --partition 2
   - _created: \d+\.\d+ (re)
-    generaldelta: false
+    generaldelta: true
     name: hg-repo-init-2
     path: '{moz}/mozilla-central'
   - _created: \d+\.\d+ (re)
     name: heartbeat-1
   - _created: \d+\.\d+ (re)
     name: heartbeat-1
   - _created: \d+\.\d+ (re)
     heads:
--- a/pylib/vcsreplicator/tests/test-hgrc-replicate.t
+++ b/pylib/vcsreplicator/tests/test-hgrc-replicate.t
@@ -9,17 +9,17 @@
 hgrc file content is sent in a message
 
   $ hgmo exec hgssh /set-hgrc-option mozilla-central hooks dummy value
   $ hgmo exec hgssh /var/hg/venv_pash/bin/hg -R /repo/hg/mozilla/mozilla-central replicatehgrc
   recorded hgrc in replication log
 
   $ consumer --dump --partition 2
   - _created: \d+\.\d+ (re)
-    generaldelta: false
+    generaldelta: true
     name: hg-repo-init-2
     path: '{moz}/mozilla-central'
   - _created: \d+\.\d+ (re)
     content: '[hooks]
   
       dummy = value
   
   
@@ -37,17 +37,17 @@ hgrc should have been written on client
 Replicating hgrc without hgrc file will delete file
 
   $ hgmo exec hgssh rm /repo/hg/mozilla/mozilla-central/.hg/hgrc
   $ hgmo exec hgssh /var/hg/venv_pash/bin/hg -R /repo/hg/mozilla/mozilla-central replicatehgrc
   recorded hgrc in replication log
 
   $ consumer --dump --partition 2
   - _created: \d+\.\d+ (re)
-    generaldelta: false
+    generaldelta: true
     name: hg-repo-init-2
     path: '{moz}/mozilla-central'
   - _created: \d+\.\d+ (re)
     content: '[hooks]
   
       dummy = value
   
   
@@ -68,17 +68,17 @@ Unicode in hgrc is preserved
 
   $ docker exec ${SSH_CID} /set-hgrc-option mozilla-central hooks dummy 'こんにちは'
 
   $ hgmo exec hgssh /var/hg/venv_pash/bin/hg -R /repo/hg/mozilla/mozilla-central replicatehgrc
   recorded hgrc in replication log
 
   $ consumer --dump --partition 2
   - _created: \d+\.\d+ (re)
-    generaldelta: false
+    generaldelta: true
     name: hg-repo-init-2
     path: '{moz}/mozilla-central'
   - _created: \d+\.\d+ (re)
     content: '[hooks]
   
       dummy = value
   
   
--- a/pylib/vcsreplicator/tests/test-init.t
+++ b/pylib/vcsreplicator/tests/test-init.t
@@ -2,17 +2,17 @@
 
   $ . $TESTDIR/pylib/vcsreplicator/tests/helpers.sh
   $ vcsrenv
 
 Creating a repository should record an event saying so
 
   $ hgmo exec hgweb0 ls /repo/hg/mozilla
 
-  $ hgmo create-repo mozilla-central scm_level_3
+  $ hgmo create-repo mozilla-central scm_level_3 --no-generaldelta
   (recorded repository creation in replication log)
 
   $ hgmo exec hgssh cat /repo/hg/mozilla/mozilla-central/.hg/requires
   dotencode
   fncache
   revlogv1
   store
 
@@ -57,17 +57,17 @@ Creating a repository should record an e
   $ hgmo exec hgweb0 cat /repo/hg/mozilla/mozilla-central/.hg/requires
   dotencode
   fncache
   revlogv1
   store
 
 generaldelta is preserved
 
-  $ hgmo create-repo mcgd scm_level_3 --generaldelta
+  $ hgmo create-repo mcgd scm_level_3
   (recorded repository creation in replication log)
 
   $ hgmo exec hgssh cat /repo/hg/mozilla/mcgd/.hg/requires
   dotencode
   fncache
   generaldelta
   revlogv1
   store
--- a/pylib/vcsreplicator/tests/test-pushaggregator.t
+++ b/pylib/vcsreplicator/tests/test-pushaggregator.t
@@ -18,17 +18,17 @@ The aggregate topic should contain a hea
   $ paconsumer --dump
   - _created: \d+\.\d+ (re)
     _original_created: \d+\.\d+ (re)
     _original_partition: 0
     name: heartbeat-1
   - _created: \d+\.\d+ (re)
     _original_created: \d+\.\d+ (re)
     _original_partition: 2
-    generaldelta: false
+    generaldelta: true
     name: hg-repo-init-2
     path: '{moz}/mozilla-central'
 
   $ hg -q clone ssh://${SSH_SERVER}:${SSH_PORT}/mozilla-central
   $ cd mozilla-central
 
   $ touch foo
   $ hg -q commit -A -m initial
@@ -167,29 +167,29 @@ Aggregation of messages from multiple pa
     name: heartbeat-1
   - _created: \d+\.\d+ (re)
     _original_created: \d+\.\d+ (re)
     _original_partition: 0
     name: heartbeat-1
   - _created: \d+\.\d+ (re)
     _original_created: \d+\.\d+ (re)
     _original_partition: 2
-    generaldelta: false
+    generaldelta: true
     name: hg-repo-init-2
     path: '{moz}/mc2'
   - _created: \d+\.\d+ (re)
     _original_created: \d+\.\d+ (re)
     _original_partition: 4
-    generaldelta: false
+    generaldelta: true
     name: hg-repo-init-2
     path: '{moz}/try'
   - _created: \d+\.\d+ (re)
     _original_created: \d+\.\d+ (re)
     _original_partition: 7
-    generaldelta: false
+    generaldelta: true
     name: hg-repo-init-2
     path: '{moz}/users/foo'
 
 --max-polls argument exits process after N intervals
 
   $ hgmo exec hgssh /var/hg/venv_tools/bin/vcsreplicator-aggregator --max-polls 1 /etc/mercurial/pushdataaggregator.ini
   vcsreplicator.aggregator hit max polls threshold; exiting
   vcsreplicator.aggregator executing loop exiting gracefully
--- a/pylib/vcsreplicator/tests/test-sync.t
+++ b/pylib/vcsreplicator/tests/test-sync.t
@@ -10,33 +10,34 @@
 
   $ hgmo exec hgssh /set-hgrc-option mozilla-central hooks foo bar
 
   $ hgmo exec hgssh /var/hg/venv_tools/bin/hg -R /repo/hg/mozilla/mozilla-central replicatesync
   wrote synchronization message into replication log
 
   $ consumer --dump --partition 2
   - _created: \d+\.\d+ (re)
-    generaldelta: false
+    generaldelta: true
     name: hg-repo-init-2
     path: '{moz}/mozilla-central'
   - _created: \d+\.\d+ (re)
     heads:
     - '0000000000000000000000000000000000000000'
     hgrc: '[hooks]
   
       foo = bar
   
   
       '
     name: hg-repo-sync-1
     path: '{moz}/mozilla-central'
     requirements:
     - dotencode
     - fncache
+    - generaldelta
     - revlogv1
     - store
 
   $ hgmo exec hgweb0 /var/hg/venv_replication/bin/vcsreplicator-consumer --wait-for-no-lag /etc/mercurial/vcsreplicator.ini
 
   $ hgmo exec hgweb0 cat /var/log/vcsreplicator/consumer.log
   No handlers could be found for logger "kafka.conn"
   No handlers could be found for logger "kafka.conn"
--- a/testing/vcttesting/hgmo.py
+++ b/testing/vcttesting/hgmo.py
@@ -488,28 +488,28 @@ class HgCluster(object):
             f_host_rsa_key = e.submit(self._d.get_file_content, master_id,
                                       '/etc/mercurial/ssh/ssh_host_rsa_key.pub')
 
         host_ed25519_key = ' '.join(f_host_ed25519_key.result().split()[0:2])
         host_rsa_key = ' '.join(f_host_rsa_key.result().split()[0:2])
 
         return f_private_key.result(), f_public_key.result(), host_ed25519_key, host_rsa_key
 
-    def create_repo(self, name, group='scm_level_1', generaldelta=False):
+    def create_repo(self, name, group='scm_level_1', no_generaldelta=False):
         """Create a repository on the cluster.
 
         ``path`` is the path fragment the repository would be accessed under
         at https://hg.mozilla.org. e.g. ``hgcustom/version-control-tools``.
 
         The repository will be owned by the specified ``group``.
         """
         cmd = ['/create-repo', name, group]
 
-        if generaldelta:
-            cmd.append('--generaldelta')
+        if no_generaldelta:
+            cmd.append('--no-generaldelta')
 
         return self._d.execute(self.master_id, cmd, stdout=True, stderr=True)
 
     def aggregate_code_coverage(self, destdir):
         master_map = {}
         for host, container in self.MASTER_FILE_MAP.items():
             master_map[container] = os.path.join(ROOT, host)
 
--- a/testing/vcttesting/hgmo_mach_commands.py
+++ b/testing/vcttesting/hgmo_mach_commands.py
@@ -133,20 +133,21 @@ class HgmoCommands(object):
         self.c.ldap.add_user_to_group(email, group)
 
     @Command('create-repo', category='hgmo',
              description='Create a repository in the cluster')
     @CommandArgument('name',
                      help='Name of repository to create')
     @CommandArgument('group', default='scm_level_1',
                      help='LDAP group that owns repo')
-    @CommandArgument('--generaldelta', action='store_true',
-                     help='Create repository with generaldelta storage')
-    def create_repo(self, name, group, generaldelta=True):
-        out = self.c.create_repo(name, group=group, generaldelta=generaldelta)
+    @CommandArgument('--no-generaldelta', action='store_true',
+                     help='Create repository without generaldelta storage')
+    def create_repo(self, name, group, no_generaldelta=False):
+        out = self.c.create_repo(name, group=group,
+                                 no_generaldelta=no_generaldelta)
         if out:
             sys.stdout.write(out)
 
     @Command('aggregate-code-coverage', category='hgmo',
              description='Aggregate code coverage results to a directory')
     @CommandArgument('destdir',
                      help='Directory where to save code coverage files')
     def aggregate_code_coverage(self, destdir):