Bug 1436369 Docker image for periodic file updates r=catlee draft
authorSimon Fraser <sfraser@mozilla.com>
Wed, 21 Feb 2018 10:45:01 +0000
changeset 757892 01fe9c3ea96083c2511cc0ee170de90a5574f787
parent 757574 861067332bac96a44bbf41ef366f58a30476057b
child 757893 199e637679035ee654b71da1e78c11ecc5820a91
push id99868
push usersfraser@mozilla.com
push dateWed, 21 Feb 2018 13:42:36 +0000
reviewerscatlee
bugs1436369
milestone60.0a1
Bug 1436369 Docker image for periodic file updates r=catlee MozReview-Commit-ID: E6OFCGBTttX
taskcluster/docker/periodic_updates/.eslintrc.js
taskcluster/docker/periodic_updates/Dockerfile
taskcluster/docker/periodic_updates/README.md
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/periodic_updates/.eslintrc.js
@@ -0,0 +1,109 @@
+"use strict";
+
+module.exports = {
+  "globals": {
+    // JS files in this folder are commonly xpcshell scripts where |arguments|
+    // is defined in the global scope.
+    "arguments": false
+  },
+  "rules": {
+    // Enforce return statements in callbacks of array methods.
+    "array-callback-return": "error",
+
+    // Braces only needed for multi-line arrow function blocks
+    "arrow-body-style": ["error", "as-needed"],
+
+    // Verify calls of super() in constructors.
+    "constructor-super": "error",
+
+    // Require braces around blocks that start a new line
+    "curly": ["error", "multi-line"],
+
+    // Require default case in switch statements.
+    "default-case": "error",
+
+    // Always require parenthesis for new calls
+    "new-parens": "error",
+
+    // Disallow use of alert(), confirm(), and prompt().
+    "no-alert": "error",
+
+    // Disallow likely erroneous `switch` scoped lexical declarations in
+    // case/default clauses.
+    "no-case-declarations": "error",
+
+    // Disallow use of the console API.
+    "no-console": "error",
+
+    // Disallow constant expressions in conditions (except for loops).
+    "no-constant-condition": ["error", { "checkLoops": false }],
+
+    // Disallow extending of native objects.
+    "no-extend-native": "error",
+
+    // Disallow case statement fallthrough without explicit `// falls through`
+    // annotation.
+    "no-fallthrough": "error",
+
+    // No reassigning native JS objects or read only globals.
+    "no-global-assign": "error",
+
+    // Disallow use of assignment in return statement.
+    "no-return-assign": ["error", "always"],
+
+    // Disallow use of the comma operator.
+    "no-sequences": "error",
+
+    // Disallow template literal placeholder syntax in regular strings.
+    "no-template-curly-in-string": "error",
+
+    // Disallow use of this/super before calling super() in constructors.
+    "no-this-before-super": "error",
+
+    // Disallow throwing literals (eg. |throw "error"| instead of
+    // |throw new Error("error")|)
+    "no-throw-literal": "error",
+
+    // Disallow unmodified loop conditions.
+    "no-unmodified-loop-condition": "error",
+
+    // No expressions where a statement is expected
+    "no-unused-expressions": "error",
+
+    // Disallow unnecessary escape usage in strings and regular expressions.
+    "no-useless-escape": "error",
+
+    // Disallow blank line padding within blocks.
+    "padded-blocks": ["error", "never"],
+
+    // Enforce spacing after semicolons.
+    "semi-spacing": ["error", { "before": false, "after": true }],
+
+    // Never use spaces before named function parentheses, but always for async
+    // arrow functions.
+    "space-before-function-paren": ["error", {
+      "anonymous": "ignore",
+      "asyncArrow": "always",
+      "named": "never",
+    }],
+
+    // No space padding in parentheses
+    "space-in-parens": ["error", "never"],
+
+    // ++ and -- should not need spacing
+    "space-unary-ops": ["error", { "words": true, "nonwords": false }],
+
+    // Require "use strict" to be defined globally in the script.
+    "strict": ["error", "global"],
+
+    // Enforce valid JSDoc comments.
+    "valid-jsdoc": ["error", {
+      "requireParamDescription": false,
+      "requireReturn": false,
+      "requireReturnDescription": false,
+    }],
+
+    // Disallow Yoda conditions.
+    "yoda": ["error", "never"],
+  }
+};
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/periodic_updates/Dockerfile
@@ -0,0 +1,38 @@
+FROM ubuntu:bionic
+MAINTAINER Simon Fraser <sfraser@mozilla.com>
+
+# Required software
+ENV DEBIAN_FRONTEND noninteractive
+
+# Chain apt-get commands with apt-get clean in a single docker RUN
+# to make sure that files are removed within a single docker layer
+
+RUN apt-get update -q && \
+    apt-get install -y --no-install-recommends \
+    arcanist=0~git20170812-1 \
+    bzip2=1.0.6-8.1 \
+    ca-certificates=20170717 \
+    curl=7.58.0-2ubuntu1 \
+    jq=1.5+dfsg-2 \
+    libdbus-glib-1-2=0.110-2 \
+    libgtk-3-0=3.22.28-1ubuntu1  \
+    libxml2-utils=2.9.4+dfsg1-6.1ubuntu1 \
+    libxt6=1:1.1.5-1 \
+    mercurial=4.3.1-2 \
+    python3=3.6.4-1 \
+    shellcheck=0.4.6-1 \
+    unzip=6.0-21ubuntu1 \
+    wget=1.19.4-1ubuntu2 \
+    && apt-get clean
+
+RUN useradd -d /home/worker -s /bin/bash -m worker
+
+COPY runme.sh /
+COPY scripts/* /home/worker/scripts/
+
+ENV           HOME          /home/worker
+ENV           SHELL         /bin/bash
+ENV           USER          worker
+ENV           LOGNAME       worker
+
+CMD ["/runme.sh"]
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/periodic_updates/README.md
@@ -0,0 +1,93 @@
+
+==Periodic File Updates==
+
+This docker image examines the in-tree files for HSTS preload data, HPKP pinning and blocklist.xml, and
+will produce a diff for each necessary to update the in-tree files.
+
+If given a conduit API token, it will also use the arcanist client to submit the commits for review.
+
+
+==Quick Start==
+
+```sh
+docker build -t hsts-local --no-cache --rm .
+
+docker run -e DO_HSTS=1 -e DO_HPKP=1 -e DO_BLOCKLIST=1 -e PRODUCT="firefox" -e BRANCH="mozilla-central" -e USE_MOZILLA_CENTRAL=1 hsts-local
+```
+
+HSTS checks will only be run if the `DO_HSTS` environment variable is set.
+Likewise for `DO_HPKP` and the HPKP checks, and `DO_BLOCKLIST` and the
+blocklist checks. Environment variables are used rather than command line
+arguments to make constructing taskcluster tasks easier.
+
+==Background==
+
+These scripts have been moved from `https://hg.mozilla.org/build/tools/scripts/periodic_file_updates/` and
+`security/manager/tools/` in the main repos, as part of the buildbot to taskcluster migration.
+
+==HSTS Checks==
+
+`scripts/getHSTSPreloadList.js` will examine the current contents of
+nsSTSPreloadList.inc from whichever `BRANCH` is specified, add in the mandatory
+hosts, and those from the Chromium source, and check them all to see if their
+SSL configuration is valid, and whether or not they have the
+Strict-Transport-Security header set with an appropriate `max-age`. 
+
+This javascript has been modified to use async calls to improve performance.
+
+==HPKP Checks==
+
+`scripts/genHPKPStaticPins.js` will ensure the list of pinned public keys are
+up to date.
+
+==Example Taskcluster Task==
+
+https://tools.taskcluster.net/tasks/create
+
+```yaml
+provisionerId: aws-provisioner-v1
+workerType: gecko-1-b-linux
+retries: 0
+created: '2018-02-07T14:45:57.347Z'
+deadline: '2018-02-07T17:45:57.348Z'
+expires: '2019-02-07T17:45:57.348Z'
+scopes: []
+payload:
+  image: srfraser/hsts1
+  maxRunTime: 1800
+  artifacts:
+    public/build/nsSTSPreloadList.diff:
+      path: /home/worker/artifacts/nsSTSPreloadList.diff
+      expires: '2019-02-07T13:57:35.448Z'
+      type: file
+    public/build/StaticHPKPins.h.diff:
+      path: /home/worker/artifacts/StaticHPKPins.h.diff
+      expires: '2019-02-07T13:57:35.448Z'
+      type: file
+    public/build/blocklist.diff:
+      path: /home/worker/artifacts/blocklist.diff
+      expires: '2019-02-07T13:57:35.448Z'
+      type: file
+  env:
+    DO_HSTS: 1
+    DO_HPKP: 1
+    DO_BLOCKLIST: 1
+    PRODUCT: firefox
+    BRANCH: mozilla-central
+    USE_MOZILLA_CENTRAL: 1
+    REVIEWERS: catlee
+metadata:
+  name: Periodic updates testing
+  description: Produce diffs for HSTS and HPKP in-tree files.
+  owner: sfraser@mozilla.com
+  source: 'https://tools.taskcluster.net/task-creator/'
+tags: {}
+extra:
+  treeherder:
+    jobKind: test
+    machine:
+      platform: linux64
+    tier: 1
+    symbol: 'hsts'
+
+```