Bug 1333255 - Task for indexing other tasks. r?dustin draft
authorJonas Finnemann Jensen <jopsen@gmail.com>
Thu, 02 Feb 2017 13:16:30 -0800
changeset 469828 f8ca244c170f587ead852bb9fbdc99c9dec52f31
parent 464990 5a4412474c63e1d9e66036d603ac42e9cb2b9150
child 544318 e8ef9d53eb1006df100e15345b9a0677749466dc
push id43859
push userbmo:jopsen@gmail.com
push dateThu, 02 Feb 2017 21:24:09 +0000
reviewersdustin
bugs1333255
milestone54.0a1
Bug 1333255 - Task for indexing other tasks. r?dustin This introduces a new docker image: `index-task`, which given a taskId and a set of namespaces will index the given taskId under said namespaces. MozReview-Commit-ID: KPHVT0XPfsb
taskcluster/docker/index-task/Dockerfile
taskcluster/docker/index-task/README
taskcluster/docker/index-task/index.js
taskcluster/docker/index-task/npm-shrinkwrap.json
taskcluster/docker/index-task/package.json
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/index-task/Dockerfile
@@ -0,0 +1,11 @@
+FROM node:6-alpine
+
+ENV       NODE_ENV        production
+RUN       mkdir /app
+COPY      index.js            /app
+COPY      package.json        /app
+COPY      npm-shrinkwrap.json /app
+WORKDIR   /app
+RUN       npm install && npm cache clean
+
+ENTRYPOINT ["node", "index.js"]
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/index-task/README
@@ -0,0 +1,34 @@
+Index-Image
+===========
+
+This image is designed to be used for indexing other tasks. It takes a task
+definition as follows:
+```js
+{
+  ...,
+  scopes: [
+    'index:insert-task:my-index.namespace',
+    'index:insert-task:...',
+  ],
+  payload: {
+    image: '...',
+    env: {
+      TARGET_TASKID: '<taskId-to-be-indexed>',
+    },
+    command: [
+      'my-index.namespace',
+      '....',
+    ],
+    features: {
+      taskclusterProxy: true,
+    },
+    maxRunTime: 600,
+  },
+}
+```
+
+As can be seen the `taskId` to be indexed is given by the environment variable
+`TARGET_TASKID` and the `command` arguments specifies namespaces that it must
+be index under. It is **important** to also include scopes on the form
+`index:insert-task:<...>` for all namespaces `<...>` given as `command`
+arguments.
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/index-task/index.js
@@ -0,0 +1,44 @@
+let taskcluster = require('taskcluster-client');
+
+// Create instance of index client
+let index = new taskcluster.Index({
+  delayFactor:    750,  // Good solid delay for background process
+  retries:        8,    // A few extra retries for robustness
+  baseUrl:        'taskcluster/index/v1',
+});
+
+// Create queue instance for fetching taskId
+let queue = new taskcluster.Queue();
+
+// Load input
+let taskId = process.env.TARGET_TASKID;
+let namespaces = process.argv.slice(2);
+
+// Validate input
+if (!taskId) {
+  console.log('Expected target task as environment variable: TARGET_TASKID');
+  process.exit(1);
+}
+
+// Fetch task definition to get expiration and then insert into index
+queue.task(taskId).then(task => task.expires).then(expires => {
+  return Promise.all(namespaces.map(namespace => {
+    console.log('Inserting %s into index under: %s', taskId, namespace);
+    return index.insertTask(namespace, {
+      taskId,
+      rank: 0,
+      data: {},
+      expires,
+    });
+  }));
+}).then(() => {
+  console.log('indexing successfully completed.');
+  process.exit(0);
+}).catch(err => {
+  console.log('Error:\n%s', err);
+  if (err.stack) {
+    console.log('Stack:\n%s', err.stack());
+  }
+  console.log('Properties:\n%j', err);
+  throw err;
+}).catch(() => process.exit(1));
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/index-task/npm-shrinkwrap.json
@@ -0,0 +1,309 @@
+{
+  "dependencies": {
+    "amqplib": {
+      "version": "0.5.1",
+      "from": "amqplib@>=0.5.1 <0.6.0",
+      "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.5.1.tgz"
+    },
+    "asap": {
+      "version": "1.0.0",
+      "from": "asap@>=1.0.0 <1.1.0",
+      "resolved": "https://registry.npmjs.org/asap/-/asap-1.0.0.tgz"
+    },
+    "async": {
+      "version": "0.9.2",
+      "from": "async@>=0.9.0 <0.10.0",
+      "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz"
+    },
+    "bitsyntax": {
+      "version": "0.0.4",
+      "from": "bitsyntax@>=0.0.4 <0.1.0",
+      "resolved": "https://registry.npmjs.org/bitsyntax/-/bitsyntax-0.0.4.tgz"
+    },
+    "bluebird": {
+      "version": "3.4.7",
+      "from": "bluebird@>=3.4.6 <4.0.0",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz"
+    },
+    "boom": {
+      "version": "2.10.1",
+      "from": "boom@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz"
+    },
+    "buffer-more-ints": {
+      "version": "0.0.2",
+      "from": "buffer-more-ints@0.0.2",
+      "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-0.0.2.tgz"
+    },
+    "combined-stream": {
+      "version": "0.0.7",
+      "from": "combined-stream@>=0.0.4 <0.1.0",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz"
+    },
+    "component-emitter": {
+      "version": "1.2.1",
+      "from": "component-emitter@>=1.2.0 <1.3.0",
+      "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz"
+    },
+    "cookiejar": {
+      "version": "2.0.6",
+      "from": "cookiejar@2.0.6",
+      "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.6.tgz"
+    },
+    "core-util-is": {
+      "version": "1.0.2",
+      "from": "core-util-is@>=1.0.0 <1.1.0",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
+    },
+    "cryptiles": {
+      "version": "2.0.5",
+      "from": "cryptiles@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz"
+    },
+    "debug": {
+      "version": "2.6.0",
+      "from": "debug@>=2.1.3 <3.0.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.0.tgz"
+    },
+    "delayed-stream": {
+      "version": "0.0.5",
+      "from": "delayed-stream@0.0.5",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz"
+    },
+    "eventsource": {
+      "version": "0.1.6",
+      "from": "eventsource@0.1.6",
+      "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-0.1.6.tgz",
+      "optional": true
+    },
+    "extend": {
+      "version": "3.0.0",
+      "from": "extend@3.0.0",
+      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz"
+    },
+    "faye-websocket": {
+      "version": "0.11.1",
+      "from": "faye-websocket@>=0.11.0 <0.12.0",
+      "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.1.tgz",
+      "optional": true
+    },
+    "form-data": {
+      "version": "0.2.0",
+      "from": "form-data@0.2.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-0.2.0.tgz"
+    },
+    "formidable": {
+      "version": "1.0.17",
+      "from": "formidable@>=1.0.14 <1.1.0",
+      "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.17.tgz"
+    },
+    "hawk": {
+      "version": "2.3.1",
+      "from": "hawk@>=2.3.1 <3.0.0",
+      "resolved": "https://registry.npmjs.org/hawk/-/hawk-2.3.1.tgz"
+    },
+    "hoek": {
+      "version": "2.16.3",
+      "from": "hoek@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz"
+    },
+    "inherits": {
+      "version": "2.0.3",
+      "from": "inherits@>=2.0.1 <2.1.0",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
+    },
+    "isarray": {
+      "version": "0.0.1",
+      "from": "isarray@0.0.1",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
+    },
+    "json3": {
+      "version": "3.3.2",
+      "from": "json3@>=3.3.2 <4.0.0",
+      "resolved": "https://registry.npmjs.org/json3/-/json3-3.3.2.tgz",
+      "optional": true
+    },
+    "lodash": {
+      "version": "3.10.1",
+      "from": "lodash@>=3.6.0 <4.0.0",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz"
+    },
+    "methods": {
+      "version": "1.1.2",
+      "from": "methods@>=1.1.1 <1.2.0",
+      "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz"
+    },
+    "mime": {
+      "version": "1.3.4",
+      "from": "mime@1.3.4",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
+    },
+    "mime-db": {
+      "version": "1.12.0",
+      "from": "mime-db@>=1.12.0 <1.13.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.12.0.tgz"
+    },
+    "mime-types": {
+      "version": "2.0.14",
+      "from": "mime-types@>=2.0.3 <2.1.0",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.14.tgz"
+    },
+    "ms": {
+      "version": "0.7.2",
+      "from": "ms@0.7.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz"
+    },
+    "original": {
+      "version": "1.0.0",
+      "from": "original@>=0.0.5",
+      "resolved": "https://registry.npmjs.org/original/-/original-1.0.0.tgz",
+      "optional": true,
+      "dependencies": {
+        "url-parse": {
+          "version": "1.0.5",
+          "from": "url-parse@>=1.0.0 <1.1.0",
+          "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.0.5.tgz",
+          "optional": true
+        }
+      }
+    },
+    "promise": {
+      "version": "6.1.0",
+      "from": "promise@>=6.1.0 <7.0.0",
+      "resolved": "https://registry.npmjs.org/promise/-/promise-6.1.0.tgz"
+    },
+    "qs": {
+      "version": "2.3.3",
+      "from": "qs@2.3.3",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-2.3.3.tgz"
+    },
+    "querystringify": {
+      "version": "0.0.4",
+      "from": "querystringify@>=0.0.0 <0.1.0",
+      "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-0.0.4.tgz"
+    },
+    "readable-stream": {
+      "version": "1.1.14",
+      "from": "readable-stream@>=1.0.0 <2.0.0 >=1.1.9",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz"
+    },
+    "reduce-component": {
+      "version": "1.0.1",
+      "from": "reduce-component@1.0.1",
+      "resolved": "https://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz"
+    },
+    "requires-port": {
+      "version": "1.0.0",
+      "from": "requires-port@>=1.0.0 <1.1.0",
+      "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz"
+    },
+    "slugid": {
+      "version": "1.1.0",
+      "from": "slugid@>=1.1.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/slugid/-/slugid-1.1.0.tgz"
+    },
+    "sntp": {
+      "version": "1.0.9",
+      "from": "sntp@>=1.0.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz"
+    },
+    "sockjs-client": {
+      "version": "1.1.2",
+      "from": "sockjs-client@>=1.0.3 <2.0.0",
+      "resolved": "https://registry.npmjs.org/sockjs-client/-/sockjs-client-1.1.2.tgz",
+      "optional": true
+    },
+    "string_decoder": {
+      "version": "0.10.31",
+      "from": "string_decoder@>=0.10.0 <0.11.0",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
+    },
+    "superagent": {
+      "version": "1.7.2",
+      "from": "superagent@>=1.7.0 <1.8.0",
+      "resolved": "https://registry.npmjs.org/superagent/-/superagent-1.7.2.tgz",
+      "dependencies": {
+        "readable-stream": {
+          "version": "1.0.27-1",
+          "from": "readable-stream@1.0.27-1",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz"
+        }
+      }
+    },
+    "superagent-hawk": {
+      "version": "0.0.6",
+      "from": "superagent-hawk@>=0.0.6 <0.0.7",
+      "resolved": "https://registry.npmjs.org/superagent-hawk/-/superagent-hawk-0.0.6.tgz",
+      "dependencies": {
+        "boom": {
+          "version": "0.4.2",
+          "from": "boom@>=0.4.0 <0.5.0",
+          "resolved": "https://registry.npmjs.org/boom/-/boom-0.4.2.tgz"
+        },
+        "cryptiles": {
+          "version": "0.2.2",
+          "from": "cryptiles@>=0.2.0 <0.3.0",
+          "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-0.2.2.tgz"
+        },
+        "hawk": {
+          "version": "1.0.0",
+          "from": "hawk@>=1.0.0 <1.1.0",
+          "resolved": "https://registry.npmjs.org/hawk/-/hawk-1.0.0.tgz"
+        },
+        "hoek": {
+          "version": "0.9.1",
+          "from": "hoek@>=0.9.0 <0.10.0",
+          "resolved": "https://registry.npmjs.org/hoek/-/hoek-0.9.1.tgz"
+        },
+        "qs": {
+          "version": "0.6.6",
+          "from": "qs@>=0.6.6 <0.7.0",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-0.6.6.tgz"
+        },
+        "sntp": {
+          "version": "0.2.4",
+          "from": "sntp@>=0.2.0 <0.3.0",
+          "resolved": "https://registry.npmjs.org/sntp/-/sntp-0.2.4.tgz"
+        }
+      }
+    },
+    "superagent-promise": {
+      "version": "0.2.0",
+      "from": "superagent-promise@>=0.2.0 <0.3.0",
+      "resolved": "https://registry.npmjs.org/superagent-promise/-/superagent-promise-0.2.0.tgz"
+    },
+    "taskcluster-client": {
+      "version": "1.6.3",
+      "from": "taskcluster-client@>=1.6.2 <2.0.0",
+      "resolved": "https://registry.npmjs.org/taskcluster-client/-/taskcluster-client-1.6.3.tgz"
+    },
+    "url-join": {
+      "version": "0.0.1",
+      "from": "url-join@>=0.0.1 <0.0.2",
+      "resolved": "https://registry.npmjs.org/url-join/-/url-join-0.0.1.tgz"
+    },
+    "url-parse": {
+      "version": "1.1.7",
+      "from": "url-parse@>=1.1.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.1.7.tgz",
+      "optional": true
+    },
+    "uuid": {
+      "version": "2.0.3",
+      "from": "uuid@>=2.0.1 <3.0.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz"
+    },
+    "websocket-driver": {
+      "version": "0.6.5",
+      "from": "websocket-driver@>=0.5.1",
+      "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.6.5.tgz",
+      "optional": true
+    },
+    "websocket-extensions": {
+      "version": "0.1.1",
+      "from": "websocket-extensions@>=0.1.1",
+      "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.1.tgz",
+      "optional": true
+    }
+  }
+}
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/index-task/package.json
@@ -0,0 +1,9 @@
+{
+  "private": true,
+  "scripts":{
+    "start": "node index.js"
+  },
+  "dependencies": {
+    "taskcluster-client": "^1.6.2"
+  }
+}