nagai/fune
3
0
Fork 0
forked from mirrors/gecko-dev
Code Pull requests Activity

Bug 1905642 - Speed up the remote settings part of periodic_file_updates.sh. r=leplatrem,releng-reviewers,bhearsum a=NPOTB

Browse source
This commit is contained in:
Mark Banner 2024-07-03 08:28:16 +00:00
parent 503d25e81d
commit 7014aed982
1 changed files with 12 additions and 25 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

37
taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
View file

@ -63,7 +63,6 @@ HPKP_UPDATED=false
DO_REMOTE_SETTINGS=false
REMOTE_SETTINGS_SERVER=''
REMOTE_SETTINGS_INPUT="${DATADIR}/remote-settings.in"
REMOTE_SETTINGS_OUTPUT="${DATADIR}/remote-settings.out"
REMOTE_SETTINGS_DIR="/services/settings/dumps"
REMOTE_SETTINGS_UPDATED=false
@ -302,15 +301,10 @@ function compare_remote_settings_files {
# 2. For each entry ${bucket, collection, last_modified}
while IFS="/" read -r bucket collection last_modified; do
# 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder
hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}"
${WGET} -qO "$local_location_input" "$hg_dump_url"
if [ $? -eq 8 ]; then
# We don't keep any dump for this collection, skip it.
# Try to clean up in case no collection in this bucket has dump.
rmdir "$REMOTE_SETTINGS_INPUT/${bucket}" --ignore-fail-on-non-empty
# 3. Check to see if the collection exists in the dump directory of the repository,
# if it does not then we aren't keeping the dump, and so we skip it.
local_dump_file="${REPODIR}${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
if [ ! -r "${local_dump_file}" ]; then
continue
fi
@ -352,7 +346,7 @@ function compare_remote_settings_files {
done
echo "INFO: diffing old/new remote settings dumps..."
${DIFF} -r "${REMOTE_SETTINGS_INPUT}" "${REMOTE_SETTINGS_OUTPUT}" > "${REMOTE_SETTINGS_DIFF_ARTIFACT}"
${DIFF} -r "${REPODIR}${REMOTE_SETTINGS_DIR}" "${REMOTE_SETTINGS_OUTPUT}" > "${REMOTE_SETTINGS_DIFF_ARTIFACT}"
if [ -s "${REMOTE_SETTINGS_DIFF_ARTIFACT}" ]
then
return 0
@ -373,16 +367,13 @@ function update_remote_settings_attachment() {
# These paths match _readAttachmentDump in services/settings/Attachments.sys.mjs.
local path_to_attachment="${bucket}/${collection}/${attachment_id}"
local path_to_meta="${bucket}/${collection}/${attachment_id}.meta.json"
local old_meta="$REMOTE_SETTINGS_INPUT/${path_to_meta}"
local old_meta="${REPODIR}${REMOTE_SETTINGS_DIR}/${path_to_meta}"
local new_meta="$REMOTE_SETTINGS_OUTPUT/${path_to_meta}"
# Those files should have been created by compare_remote_settings_files before the function call.
local local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
local local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
# Compute the metadata based on already-downloaded records.
mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}/${collection}"
${JQ} -cj <"$local_location_input" "${jq_attachment_selector}" > "${old_meta}"
# Download the current meta data files.
mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}"
${JQ} -cj <"$local_location_output" "${jq_attachment_selector}" > "${new_meta}"
@ -394,13 +385,6 @@ function update_remote_settings_attachment() {
echo "INFO: Downloading updated remote settings dump: ${bucket}/${collection}/${attachment_id}"
# Overwrited old_meta with the actual file from the repo. The content should be equivalent,
# but can have minor differences (e.g. different line endings) if the checked in file was not
# generated by this script (e.g. manually checked in).
${WGET} -qO "${old_meta}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_meta}"
${WGET} -qO "${REMOTE_SETTINGS_INPUT}/${path_to_attachment}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_attachment}"
if [ -z "${ATTACHMENT_BASE_URL}" ] ; then
ATTACHMENT_BASE_URL=$(${WGET} -qO- "${REMOTE_SETTINGS_SERVER}" | ${JQ} -r .capabilities.attachments.base_url)
fi
@ -574,6 +558,11 @@ BROWSER_ARCHIVE="target.${PLATFORM_EXT}"
TESTS_ARCHIVE="target.common.tests.tar.gz"
preflight_cleanup
# Clone the repository here as some sections will use it for source data, and
# we'll need it later anyway.
clone_repo
if [ "${DO_HSTS}" == "true" ] || [ "${DO_HPKP}" == "true" ] || [ "${DO_PRELOAD_PINSET}" == "true" ]
then
if [ "${USE_TC}" == "true" ]; then
@ -626,8 +615,6 @@ else
fi
fi
clone_repo
COMMIT_MESSAGE="No Bug, ${BRANCH} repo-update"
if [ "${HSTS_UPDATED}" == "true" ]
then