Bug 1451040 - Download remote settings dumps regularly r?sfraser draft
authorMathieu Leplatre <mathieu@mozilla.com>
Wed, 16 May 2018 13:03:22 +0200
changeset 798094 79c1bcdb091f1c2c8fe1f353fd9601140a0e64e8
parent 796870 11ee70f24ea52c4dc4f113593c288f4a6dc92c55
push id110678
push usermleplatre@mozilla.com
push dateTue, 22 May 2018 12:59:07 +0000
reviewerssfraser
bugs1451040
milestone62.0a1
Bug 1451040 - Download remote settings dumps regularly r?sfraser MozReview-Commit-ID: BCCEplOq1O8
taskcluster/ci/repo-update/kind.yml
taskcluster/docker/periodic-updates/runme.sh
taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
--- a/taskcluster/ci/repo-update/kind.yml
+++ b/taskcluster/ci/repo-update/kind.yml
@@ -6,33 +6,34 @@ loader: taskgraph.loader.transform:loade
 
 transforms:
    - taskgraph.transforms.task:transforms
 
 
 jobs:
    hsts-hpkp-blocklist:
       name: periodic_file_update
-      description: HSTS, HPKP, and Blocklist update
+      description: HSTS, HPKP, Blocklist and remote settings update
       run-on-projects: []  # Only run via cron
       treeherder:
          kind: build
          platform: linux64/opt
          symbol: pfu
          tier: 1
       worker-type: aws-provisioner-v1/gecko-{level}-b-linux
       worker:
          implementation: docker-worker
          os: linux
          docker-image: {in-tree: periodic-updates}
          max-run-time: 3600  # Sometimes takes ~40 minutes
          env:
             DO_HSTS: "1"
             DO_HPKP: "1"
             DO_BLOCKLIST: "1"
+            DO_REMOTE_SETTINGS: "1"
             USE_MOZILLA_CENTRAL: "1"
             BRANCH: mozilla-central
             PRODUCT: firefox
             REVIEWERS: "mtabara, jlund"
          command:
             - /runme.sh
          taskcluster-proxy: true
          artifacts:
@@ -40,10 +41,13 @@ jobs:
               path: '/home/worker/artifacts/nsSTSPreloadList.diff'
               type: file
             - name: 'public/build/StaticHPKPins.h.diff'
               path: '/home/worker/artifacts/StaticHPKPins.h.diff'
               type: file
             - name: 'public/build/blocklist.diff'
               path: '/home/worker/artifacts/blocklist.diff'
               type: file
+            - name: 'public/build/remote-settings.diff'
+              path: '/home/worker/artifacts/remote-settings.diff'
+              type: file
       scopes:
          - secrets:get:project/releng/gecko/build/level-{level}/arc-phabricator-token
--- a/taskcluster/docker/periodic-updates/runme.sh
+++ b/taskcluster/docker/periodic-updates/runme.sh
@@ -32,16 +32,21 @@ then
   PARAMS="${PARAMS} --hpkp"
 fi
 
 if [ ! -z "${DO_BLOCKLIST}" ]
 then
   PARAMS="${PARAMS} --blocklist"
 fi
 
+if [ ! -z "${DO_REMOTE_SETTINGS}" ]
+then
+  PARAMS="${PARAMS} --remote-settings"
+fi
+
 export ARTIFACTS_DIR="/home/worker/artifacts"
 mkdir -p "$ARTIFACTS_DIR"
 
 # Get Arcanist API token
 
 if [ -n "${TASK_ID}" ]
 then
   curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json \
--- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
@@ -69,21 +69,29 @@ HPKP_UPDATED=false
 
 DO_BLOCKLIST=false
 BLOCKLIST_URL_AMO=''
 BLOCKLIST_URL_HG=''
 BLOCKLIST_LOCAL_AMO="blocklist_amo.xml"
 BLOCKLIST_LOCAL_HG="blocklist_hg.xml"
 BLOCKLIST_UPDATED=false
 
-ARTIFACTS_DIR="${ARTIFACTS_DIR:-'.'}"
+DO_REMOTE_SETTINGS=false
+REMOTE_SETTINGS_SERVER=''
+REMOTE_SETTINGS_INPUT="${DATADIR}/remote-settings.in"
+REMOTE_SETTINGS_OUTPUT="${DATADIR}/remote-settings.out"
+REMOTE_SETTINGS_DIR="/services/settings/dumps"
+REMOTE_SETTINGS_UPDATED=false
+
+ARTIFACTS_DIR="${ARTIFACTS_DIR:-.}"
 # Defaults
 HSTS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HSTS_DIFF_ARTIFACT:-"nsSTSPreloadList.diff"}"
 HPKP_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HPKP_DIFF_ARTIFACT:-"StaticHPKPins.h.diff"}"
 BLOCKLIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${BLOCKLIST_DIFF_ARTIFACT:-"blocklist.diff"}"
+REMOTE_SETTINGS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${REMOTE_SETTINGS_DIFF_ARTIFACT:-"remote-settings.diff"}"
 
 
 # Get the current in-tree version for a code branch.
 function get_version {
   VERSION_REPO=$1
   VERSION_FILE='version.txt'
 
   # TODO bypass temporary file
@@ -294,16 +302,54 @@ function compare_blocklist_files {
   ${DIFF} ${BLOCKLIST_LOCAL_HG} ${BLOCKLIST_LOCAL_AMO} | tee "${BLOCKLIST_DIFF_ARTIFACT}"
   if [ -s "${BLOCKLIST_DIFF_ARTIFACT}" ]
   then
     return 0
   fi
   return 1
 }
 
+function compare_remote_settings_files {
+  REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1"
+
+  # 1. List remote settings collections from server.
+  echo "INFO: fetch remote settings list from server"
+  ${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\
+      ${JQ} -r '.data[] | .bucket+"/"+.collection' |\
+      # 2. For each entry ${bucket, collection}
+      while IFS="/" read -r bucket collection; do
+
+        # 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder
+        hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
+        local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
+        mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}"
+        ${WGET} -qO "$local_location_input" "$hg_dump_url"
+        if [ $? -eq 8 ]; then
+          # We don't keep any dump for this collection, skip it.
+          # Try to clean up in case no collection in this bucket has dump.
+          rmdir "$REMOTE_SETTINGS_INPUT/${bucket}" --ignore-fail-on-non-empty
+          continue
+        fi
+
+        # 4. Download server version into REMOTE_SETTINGS_OUTPUT folder
+        remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/records"
+        local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
+        mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
+        ${WGET} -qO "$local_location_output" "$remote_records_url"
+      done
+
+  echo "INFO: diffing old/new remote settings dumps..."
+  ${DIFF} -r "${REMOTE_SETTINGS_INPUT}" "${REMOTE_SETTINGS_OUTPUT}" > "${REMOTE_SETTINGS_DIFF_ARTIFACT}"
+  if [ -s "${REMOTE_SETTINGS_DIFF_ARTIFACT}" ]
+  then
+    return 0
+  fi
+  return 1
+}
+
 function clone_build_tools {
     rm -fr "${TOOLSDIR}"
     CLONE_CMD="${HG} clone https://hg.mozilla.org/build/tools ${TOOLSDIR}"
     ${CLONE_CMD}
 }
 
 # Clones an hg repo, using hgtool preferentially.
 function clone_repo {
@@ -388,16 +434,38 @@ function commit_blocklist_files {
   fi
   if [ ${APPROVAL} == true ]; then
     COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=blocklist-update"
   fi
   echo "INFO: committing blocklist changes"
   ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
 }
 
+# Copies new remote settings dump files in place, and commits them.
+function commit_remote_settings_files {
+  cd "${BASEDIR}"
+  cp -a "${REMOTE_SETTINGS_OUTPUT}/*" "${REPODIR}${REMOTE_SETTINGS_DIR}"
+
+  COMMIT_MESSAGE="No bug, Automated remote settings update"
+  if [ -n "${TASK_ID}" ]; then
+    COMMIT_MESSAGE="${COMMIT_MESSAGE} from task ${TASK_ID}"
+  fi
+  if [ ${DONTBUILD} == true ]; then
+    COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
+  fi
+  if [ ${CLOSED_TREE} == true ]; then
+    COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
+  fi
+  if [ ${APPROVAL} == true ]; then
+    COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=remote-settings-update"
+  fi
+  echo "INFO: committing remote settings changes"
+  ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
+}
+
 # Push all pending commits to Phabricator
 function push_repo {
   cd "${REPODIR}"
   if [ ! -r "${HOME}/.arcrc" ]
   then
     return 1
   fi
   if ! ARC=$(which arc)
@@ -435,16 +503,17 @@ while [ $# -gt 0 ]; do
     -n) DRY_RUN=true ;;
     -c) CLOSED_TREE=true ;;
     -d) DONTBUILD=true ;;
     -a) APPROVAL=true ;;
     --pinset) DO_PRELOAD_PINSET=true ;;
     --hsts) DO_HSTS=true ;;
     --hpkp) DO_HPKP=true ;;
     --blocklist) DO_BLOCKLIST=true ;;
+    --remote-settings) DO_REMOTE_SETTINGS=true ;;
     -r) REPODIR="$2"; shift ;;
     --use-mozilla-central) USE_MC=true ;;
     --use-ftp-builds) USE_TC=false ;;
     -*) usage
       exit 11 ;;
     *)  break ;; # terminate while loop
   esac
   shift
@@ -453,19 +522,19 @@ done
 # Must supply a code branch to work with.
 if [ "${BRANCH}" == "" ]; then
   echo "Error: You must specify a branch with -b branchname." >&2
   usage
   exit 12
 fi
 
 # Must choose at least one update action.
-if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_BLOCKLIST" == "false" ]
+if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_BLOCKLIST" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ]
 then
-  echo "Error: you must specify at least one action from: --hsts, --hpkp, --blocklist" >&2
+  echo "Error: you must specify at least one action from: --hsts, --hpkp, --blocklist, --remote-settings" >&2
   usage
   exit 13
 fi
 
 # per-product constants
 case "${PRODUCT}" in
   thunderbird)
     APP_DIR="mail"
@@ -538,18 +607,24 @@ if [ "${DO_HPKP}" == "true" ]; then
   fi
 fi
 if [ "${DO_BLOCKLIST}" == "true" ]; then
   if compare_blocklist_files
   then
     BLOCKLIST_UPDATED=true
   fi
 fi
+if [ "${DO_REMOTE_SETTINGS}" == "true" ]; then
+  if compare_remote_settings_files
+  then
+    REMOTE_SETTINGS_UPDATED=true
+  fi
+fi
 
-if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${BLOCKLIST_UPDATED}" == "false" ]; then
+if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${BLOCKLIST_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ]; then
   echo "INFO: no updates required. Exiting."
   exit 0
 else
   if [ "${DRY_RUN}" == "true" ]; then
     echo "INFO: Updates are available, not updating hg in dry-run mode."
     exit 2
   fi
 fi
@@ -573,14 +648,20 @@ then
 fi
 
 if [ "${BLOCKLIST_UPDATED}" == "true" ]
 then
   commit_blocklist_files
   MUST_PUSH=true
 fi
 
+if [ "${REMOTE_SETTINGS_UPDATED}" == "true" ]
+then
+  commit_remote_settings_files
+  MUST_PUSH=true
+fi
+
 if [ -n "${MUST_PUSH}" ]
 then
   push_repo
 fi
 
 echo "All done"