Bug 1390693 - Generate docs archive within mach command; r?dustin draft
authorGregory Szorc <gps@mozilla.com>
Thu, 24 Aug 2017 10:37:53 -0700
changeset 652430 1f7646e1d27bb3301c6ad422bedd54866950bc70
parent 652429 8b10c11e19d9ffe29ae030c2201cb6626ca449e7
child 652431 ecfcfc64013751517cb1d7bedef34c6916659f31
push id76055
push usergszorc@mozilla.com
push dateThu, 24 Aug 2017 20:42:35 +0000
reviewersdustin
bugs1390693
milestone57.0a1
Bug 1390693 - Generate docs archive within mach command; r?dustin By using mozpack, we get deterministic archives. This also makes the task command simpler. MozReview-Commit-ID: EPI7tuGQuso
taskcluster/ci/source-test/doc.yml
tools/docs/mach_commands.py
tools/docs/moztreedocs/__init__.py
--- a/taskcluster/ci/source-test/doc.yml
+++ b/taskcluster/ci/source-test/doc.yml
@@ -12,20 +12,18 @@ doc-generate:
         artifacts:
             - type: file
               name: public/docs.tar.gz
               path: /home/worker/checkouts/gecko/docs.tar.gz
     run:
         using: run-task
         command: >
             cd /home/worker/checkouts/gecko &&
-            ./mach doc --outdir docs-out --no-open &&
-            rm -rf docs-out/html/Mozilla_Source_Tree_Docs/_venv &&
-            mv docs-out/html/Mozilla_Source_Tree_Docs docs &&
-            tar -czf docs.tar.gz docs
+            ./mach doc --outdir docs-out --no-open --archive &&
+            mv docs-out/Mozilla_Source_Tree_Docs.tar.gz docs.tar.gz
     when:
         files-changed:
             - '**/*.py'
             - '**/*.rst'
             - 'tools/docs/**'
 
 doc-upload:
     description: Generate and upload the Sphinx documentation
--- a/tools/docs/mach_commands.py
+++ b/tools/docs/mach_commands.py
@@ -25,28 +25,32 @@ class Documentation(MachCommandBase):
     @Command('doc', category='devenv',
              description='Generate and display documentation from the tree.')
     @CommandArgument('what', nargs='*', metavar='DIRECTORY [, DIRECTORY]',
                      help='Path(s) to documentation to build and display.')
     @CommandArgument('--format', default='html',
                      help='Documentation format to write.')
     @CommandArgument('--outdir', default=None, metavar='DESTINATION',
                      help='Where to write output.')
+    @CommandArgument('--archive', action='store_true',
+                     help='Write a gzipped tarball of generated docs')
     @CommandArgument('--no-open', dest='auto_open', default=True,
                      action='store_false',
                      help="Don't automatically open HTML docs in a browser.")
     @CommandArgument('--http', const=':6666', metavar='ADDRESS', nargs='?',
                      help='Serve documentation on an HTTP server, '
                           'e.g. ":6666".')
-    def build_docs(self, what=None, format=None, outdir=None, auto_open=True, http=None):
+    def build_docs(self, what=None, format=None, outdir=None, auto_open=True,
+                   http=None, archive=False):
         self._activate_virtualenv()
         self.virtualenv_manager.install_pip_package('sphinx_rtd_theme==0.1.6')
 
         import sphinx
         import webbrowser
+        import moztreedocs
 
         if not outdir:
             outdir = os.path.join(self.topobjdir, 'docs')
         if not what:
             what = [os.path.join(self.topsrcdir, 'tools')]
 
         format_outdir = os.path.join(outdir, format)
 
@@ -71,16 +75,22 @@ class Documentation(MachCommandBase):
                 savedir,
             ]
             result = sphinx.build_main(args)
             if result != 0:
                 failed.append((path, 'sphinx return code %d' % result))
             else:
                 generated.append(savedir)
 
+            if archive:
+                archive_path = os.path.join(outdir,
+                                            '%s.tar.gz' %  project)
+                moztreedocs.create_tarball(archive_path, savedir)
+                print('Archived to %s' % archive_path)
+
             index_path = os.path.join(savedir, 'index.html')
             if not http and auto_open and os.path.isfile(index_path):
                 webbrowser.open(index_path)
 
         if generated:
             print('\nGenerated documentation:\n%s\n' % '\n'.join(generated))
 
         if failed:
--- a/tools/docs/moztreedocs/__init__.py
+++ b/tools/docs/moztreedocs/__init__.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import os
 
 from mozbuild.frontend.reader import BuildReader
+from mozpack.archive import create_tar_gz_from_files
 from mozpack.copier import FileCopier
 from mozpack.files import FileFinder
 from mozpack.manifests import InstallManifest
 
 import sphinx
 import sphinx.apidoc
 
 
@@ -119,8 +120,28 @@ class SphinxManager(object):
 
         packages = [os.path.basename(p) for p in self._python_package_dirs]
         packages = ['python/%s' % p for p in packages]
         packages = '\n   '.join(sorted(packages))
         data = data.format(indexes=indexes, python_packages=packages)
 
         with open(os.path.join(self._docs_dir, 'index.rst'), 'wb') as fh:
             fh.write(data)
+
+
+def distribution_files(root):
+    """Find all files suitable for distributing.
+
+    Given the path to generated Sphinx documentation, returns an iterable
+    of (path, BaseFile) for files that should be archived, uploaded, etc.
+    Paths are relative to given root directory.
+    """
+    finder = FileFinder(root, ignore=('_staging', '_venv'))
+    return finder.find('**')
+
+
+def create_tarball(filename, root):
+    """Create a tar.gz archive of docs in a directory."""
+    files = dict(distribution_files(root))
+
+    with open(filename, 'wb') as fh:
+        create_tar_gz_from_files(fh, files, filename=os.path.basename(filename),
+                                 compresslevel=6)