--- a/scripts/generate-hg-s3-bundles
+++ b/scripts/generate-hg-s3-bundles
@@ -24,37 +24,33 @@ import concurrent.futures as futures
HG = '/var/hg/venv_tools/bin/hg'
HG_BUNDLE = '/var/hg/venv_hg_pre/bin/hg'
PUSH_REPO = '/var/hg/version-control-tools/scripts/push-repo.sh'
# The types of bundles to generate.
#
# Define in order bundles should be listed in manifest.
CREATES = [
- ('gzip', ['bundle', '-a', '-t', 'gzip-v1'], {}),
- ('gzip-v2', ['bundle', '-a', '-t', 'gzip-v2'], {'gd': True}),
+ ('gzip-v2', ['bundle', '-a', '-t', 'gzip-v2'], {}),
# ``zstd`` uses default compression settings and is reasonably fast.
# ``zstd-max`` uses the highest available compression settings and is
# absurdly slow. But it produces significantly smaller bundles. Level 20
# (and not higher) is used because it is the largest level supported
# by the zstd library in 32-bit processes.
('zstd', ['bundle', '-a', '-t', 'zstd-v2'], {'all': True}),
('zstd-max', ['--config', 'experimental.bundlecomplevel=20',
'bundle', '-a', '-t', 'zstd-v2'], {'all': True}),
- ('packed1', ['debugcreatestreamclonebundle'], {}),
- ('packed1-gd', ['debugcreatestreamclonebundle'], {'gd': True}),
+ ('packed1-gd', ['debugcreatestreamclonebundle'], {}),
]
CLONEBUNDLES_ORDER = [
('zstd-max', 'BUNDLESPEC=zstd-v2'),
('zstd', 'BUNDLESPEC=zstd-v2'),
('gzip-v2', 'BUNDLESPEC=gzip-v2'),
- ('gzip', 'BUNDLESPEC=gzip-v1'),
('packed1-gd', 'BUNDLESPEC=none-packed1;requirements%3Dgeneraldelta%2Crevlogv1'),
- ('packed1', 'BUNDLESPEC=none-packed1;requirements%3Drevlogv1'),
]
# Defines hostname and bucket where uploads should go.
HOSTS = (
# We list Oregon (us-west-2) before N. California (us-west-1) because it is
# cheaper.
('s3-us-west-2.amazonaws.com', 'moz-hg-bundles-us-west-2', 'us-west-2'),
('s3-us-west-1.amazonaws.com', 'moz-hg-bundles-us-west-1', 'us-west-1'),
@@ -103,36 +99,32 @@ HTML_INDEX = '''
For more, see
<a href="https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/bundleclone.html">the official docs</a>.
</p>
<table border="1">
<tr>
<th>Repository</th>
<th>zstd</th>
<th>zstd (max)</th>
- <th>gzip (v1)</th>
<th>gzip (v2)</th>
- <th>stream</th>
<th>stream (generaldelta)</th>
</tr>
%s
</table>
<p>This page generated at %s.</p>
</body>
</html>
'''.strip()
HTML_ENTRY = '''
<tr>
<td>{repo}</td>
<td class="numeric">{zstd_entry}</td>
<td class="numeric">{zstd_max_entry}</td>
- <td class="numeric">{gzip_entry}</td>
<td class="numeric">{gzip_v2_entry}</td>
- <td class="numeric">{packed1_entry}</a></td>
<td class="numeric">{packed1_gd_entry}</td>
</tr>
'''.strip()
def upload_to_s3(host, bucket_name, local_path, remote_path):
"""Upload a file to S3."""
attempt = 0
@@ -276,28 +268,16 @@ def generate_bundles(repo, upload=True,
for t, args, opts in CREATES:
# Only generate 1 of zstd or zstd-max since they are redundant.
if t == 'zstd' and zstd_max:
continue
if t == 'zstd-max' and not zstd_max:
continue
- # The "all" opt allows generation for generaldelta and
- # non-generaldelta repos. If not set, we key off the generaldelta
- # state and the "gd" key to determine whether to generate. What
- # we're trying to prevent is v1 bundle types being generated for
- # generaldelta repos, which could require a lot of CPU usage to
- # recalculate deltas.
- if not opts.get('all'):
- if generaldelta and not opts.get('gd'):
- continue
- elif not generaldelta and opts.get('gd'):
- continue
-
final_path, remote_path = bundle_paths(bundle_path, repo, tip, t)
temp_path = '%s.tmp' % final_path
# Record that this bundle is relevant.
bundles.append((t, final_path, remote_path))
if os.path.exists(final_path):
print('bundle already exists, skipping: %s' % final_path)
@@ -386,24 +366,23 @@ def generate_bundles(repo, upload=True,
def generate_index(repos):
"""Upload an index HTML page describing available bundles."""
entries = []
for repo in sorted(repos):
p = repos[repo]
# Should only be for bundles with copyfrom.
- if 'gzip' not in p and 'gzip-v2' not in p:
+ if 'gzip-v2' not in p:
print('ignoring repo %s in index because no gzip bundle' % repo)
continue
opts = {'repo': repo}
- for k in ('gzip', 'gzip-v2', 'packed1', 'packed1-gd',
- 'zstd', 'zstd-max'):
+ for k in ('gzip-v2', 'packed1-gd', 'zstd', 'zstd-max'):
key = '%s_entry' % k.replace('-', '_')
if k in p:
opts[key] = '<a href="{path}">{size:,}</a>'.format(
path=p[k][0],
size=p[k][1],
)
opts['basename'] = os.path.basename(p[k][0])
else: