Bug 1451040 - Download remote settings dumps regularly r?sfraser
MozReview-Commit-ID: BCCEplOq1O8
--- a/taskcluster/ci/repo-update/kind.yml
+++ b/taskcluster/ci/repo-update/kind.yml
@@ -6,33 +6,34 @@ loader: taskgraph.loader.transform:loade
transforms:
- taskgraph.transforms.task:transforms
jobs:
hsts-hpkp-blocklist:
name: periodic_file_update
- description: HSTS, HPKP, and Blocklist update
+ description: HSTS, HPKP, Blocklist and remote settings update
run-on-projects: [] # Only run via cron
treeherder:
kind: build
platform: linux64/opt
symbol: pfu
tier: 1
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
worker:
implementation: docker-worker
os: linux
docker-image: {in-tree: periodic-updates}
max-run-time: 3600 # Sometimes takes ~40 minutes
env:
DO_HSTS: "1"
DO_HPKP: "1"
DO_BLOCKLIST: "1"
+ DO_REMOTE_SETTINGS: "1"
USE_MOZILLA_CENTRAL: "1"
BRANCH: mozilla-central
PRODUCT: firefox
REVIEWERS: "mtabara, jlund"
command:
- /runme.sh
taskcluster-proxy: true
artifacts:
@@ -40,10 +41,13 @@ jobs:
path: '/home/worker/artifacts/nsSTSPreloadList.diff'
type: file
- name: 'public/build/StaticHPKPins.h.diff'
path: '/home/worker/artifacts/StaticHPKPins.h.diff'
type: file
- name: 'public/build/blocklist.diff'
path: '/home/worker/artifacts/blocklist.diff'
type: file
+ - name: 'public/build/remote-settings.diff'
+ path: '/home/worker/artifacts/remote-settings.diff'
+ type: file
scopes:
- secrets:get:project/releng/gecko/build/level-{level}/arc-phabricator-token
--- a/taskcluster/docker/periodic-updates/runme.sh
+++ b/taskcluster/docker/periodic-updates/runme.sh
@@ -32,16 +32,21 @@ then
PARAMS="${PARAMS} --hpkp"
fi
if [ ! -z "${DO_BLOCKLIST}" ]
then
PARAMS="${PARAMS} --blocklist"
fi
+if [ ! -z "${DO_REMOTE_SETTINGS}" ]
+then
+ PARAMS="${PARAMS} --remote-settings"
+fi
+
export ARTIFACTS_DIR="/home/worker/artifacts"
mkdir -p "$ARTIFACTS_DIR"
# Get Arcanist API token
if [ -n "${TASK_ID}" ]
then
curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json \
--- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
@@ -69,21 +69,29 @@ HPKP_UPDATED=false
DO_BLOCKLIST=false
BLOCKLIST_URL_AMO=''
BLOCKLIST_URL_HG=''
BLOCKLIST_LOCAL_AMO="blocklist_amo.xml"
BLOCKLIST_LOCAL_HG="blocklist_hg.xml"
BLOCKLIST_UPDATED=false
-ARTIFACTS_DIR="${ARTIFACTS_DIR:-'.'}"
+DO_REMOTE_SETTINGS=false
+REMOTE_SETTINGS_SERVER=''
+REMOTE_SETTINGS_INPUT="${DATADIR}/remote-settings.in"
+REMOTE_SETTINGS_OUTPUT="${DATADIR}/remote-settings.out"
+REMOTE_SETTINGS_DIR="/services/settings/dumps"
+REMOTE_SETTINGS_UPDATED=false
+
+ARTIFACTS_DIR="${ARTIFACTS_DIR:-.}"
# Defaults
HSTS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HSTS_DIFF_ARTIFACT:-"nsSTSPreloadList.diff"}"
HPKP_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HPKP_DIFF_ARTIFACT:-"StaticHPKPins.h.diff"}"
BLOCKLIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${BLOCKLIST_DIFF_ARTIFACT:-"blocklist.diff"}"
+REMOTE_SETTINGS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${REMOTE_SETTINGS_DIFF_ARTIFACT:-"remote-settings.diff"}"
# Get the current in-tree version for a code branch.
function get_version {
VERSION_REPO=$1
VERSION_FILE='version.txt'
# TODO bypass temporary file
@@ -294,16 +302,54 @@ function compare_blocklist_files {
${DIFF} ${BLOCKLIST_LOCAL_HG} ${BLOCKLIST_LOCAL_AMO} | tee "${BLOCKLIST_DIFF_ARTIFACT}"
if [ -s "${BLOCKLIST_DIFF_ARTIFACT}" ]
then
return 0
fi
return 1
}
+function compare_remote_settings_files {
+ REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1"
+
+ # 1. List remote settings collections from server.
+ echo "INFO: fetch remote settings list from server"
+ ${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\
+ ${JQ} -r '.data[] | .bucket+"/"+.collection' |\
+ # 2. For each entry ${bucket, collection}
+ while IFS="/" read -r bucket collection; do
+
+ # 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder
+ hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
+ local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
+ mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}"
+ ${WGET} -qO "$local_location_input" "$hg_dump_url"
+ if [ $? -eq 8 ]; then
+ # We don't keep any dump for this collection, skip it.
+ # Try to clean up in case no collection in this bucket has dump.
+ rmdir "$REMOTE_SETTINGS_INPUT/${bucket}" --ignore-fail-on-non-empty
+ continue
+ fi
+
+ # 4. Download server version into REMOTE_SETTINGS_OUTPUT folder
+ remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/records"
+ local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
+ mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
+ ${WGET} -qO "$local_location_output" "$remote_records_url"
+ done
+
+ echo "INFO: diffing old/new remote settings dumps..."
+ ${DIFF} -r "${REMOTE_SETTINGS_INPUT}" "${REMOTE_SETTINGS_OUTPUT}" > "${REMOTE_SETTINGS_DIFF_ARTIFACT}"
+ if [ -s "${REMOTE_SETTINGS_DIFF_ARTIFACT}" ]
+ then
+ return 0
+ fi
+ return 1
+}
+
function clone_build_tools {
rm -fr "${TOOLSDIR}"
CLONE_CMD="${HG} clone https://hg.mozilla.org/build/tools ${TOOLSDIR}"
${CLONE_CMD}
}
# Clones an hg repo, using hgtool preferentially.
function clone_repo {
@@ -388,16 +434,38 @@ function commit_blocklist_files {
fi
if [ ${APPROVAL} == true ]; then
COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=blocklist-update"
fi
echo "INFO: committing blocklist changes"
${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
}
+# Copies new remote settings dump files in place, and commits them.
+function commit_remote_settings_files {
+ cd "${BASEDIR}"
+ cp -a "${REMOTE_SETTINGS_OUTPUT}/*" "${REPODIR}${REMOTE_SETTINGS_DIR}"
+
+ COMMIT_MESSAGE="No bug, Automated remote settings update"
+ if [ -n "${TASK_ID}" ]; then
+ COMMIT_MESSAGE="${COMMIT_MESSAGE} from task ${TASK_ID}"
+ fi
+ if [ ${DONTBUILD} == true ]; then
+ COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
+ fi
+ if [ ${CLOSED_TREE} == true ]; then
+ COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
+ fi
+ if [ ${APPROVAL} == true ]; then
+ COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=remote-settings-update"
+ fi
+ echo "INFO: committing remote settings changes"
+ ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
+}
+
# Push all pending commits to Phabricator
function push_repo {
cd "${REPODIR}"
if [ ! -r "${HOME}/.arcrc" ]
then
return 1
fi
if ! ARC=$(which arc)
@@ -435,16 +503,17 @@ while [ $# -gt 0 ]; do
-n) DRY_RUN=true ;;
-c) CLOSED_TREE=true ;;
-d) DONTBUILD=true ;;
-a) APPROVAL=true ;;
--pinset) DO_PRELOAD_PINSET=true ;;
--hsts) DO_HSTS=true ;;
--hpkp) DO_HPKP=true ;;
--blocklist) DO_BLOCKLIST=true ;;
+ --remote-settings) DO_REMOTE_SETTINGS=true ;;
-r) REPODIR="$2"; shift ;;
--use-mozilla-central) USE_MC=true ;;
--use-ftp-builds) USE_TC=false ;;
-*) usage
exit 11 ;;
*) break ;; # terminate while loop
esac
shift
@@ -453,19 +522,19 @@ done
# Must supply a code branch to work with.
if [ "${BRANCH}" == "" ]; then
echo "Error: You must specify a branch with -b branchname." >&2
usage
exit 12
fi
# Must choose at least one update action.
-if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_BLOCKLIST" == "false" ]
+if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_BLOCKLIST" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ]
then
- echo "Error: you must specify at least one action from: --hsts, --hpkp, --blocklist" >&2
+ echo "Error: you must specify at least one action from: --hsts, --hpkp, --blocklist, --remote-settings" >&2
usage
exit 13
fi
# per-product constants
case "${PRODUCT}" in
thunderbird)
APP_DIR="mail"
@@ -538,18 +607,24 @@ if [ "${DO_HPKP}" == "true" ]; then
fi
fi
if [ "${DO_BLOCKLIST}" == "true" ]; then
if compare_blocklist_files
then
BLOCKLIST_UPDATED=true
fi
fi
+if [ "${DO_REMOTE_SETTINGS}" == "true" ]; then
+ if compare_remote_settings_files
+ then
+ REMOTE_SETTINGS_UPDATED=true
+ fi
+fi
-if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${BLOCKLIST_UPDATED}" == "false" ]; then
+if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${BLOCKLIST_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ]; then
echo "INFO: no updates required. Exiting."
exit 0
else
if [ "${DRY_RUN}" == "true" ]; then
echo "INFO: Updates are available, not updating hg in dry-run mode."
exit 2
fi
fi
@@ -573,14 +648,20 @@ then
fi
if [ "${BLOCKLIST_UPDATED}" == "true" ]
then
commit_blocklist_files
MUST_PUSH=true
fi
+if [ "${REMOTE_SETTINGS_UPDATED}" == "true" ]
+then
+ commit_remote_settings_files
+ MUST_PUSH=true
+fi
+
if [ -n "${MUST_PUSH}" ]
then
push_repo
fi
echo "All done"