|
|
|
@ -1,5 +1,27 @@
|
|
|
|
|
#!/bin/bash
|
|
|
|
|
|
|
|
|
|
function main() {
|
|
|
|
|
# Exit script on CTRL+C
|
|
|
|
|
trap "exit" INT
|
|
|
|
|
|
|
|
|
|
if [ $# -ne 1 ]; then
|
|
|
|
|
print_usage_and_exit
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
check_required_tools
|
|
|
|
|
|
|
|
|
|
# Sourcing the given tar_stamps-file to have the variables available
|
|
|
|
|
TAR_STAMP="$1"
|
|
|
|
|
source "$TAR_STAMP" || print_usage_and_exit
|
|
|
|
|
|
|
|
|
|
set_internal_variables
|
|
|
|
|
|
|
|
|
|
check_what_changed
|
|
|
|
|
download_upstream_source_tarballs
|
|
|
|
|
|
|
|
|
|
create_locales_tarballs
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function print_usage_and_exit() {
|
|
|
|
|
echo "Usage: create-tar.sh tar_stamps"
|
|
|
|
|
echo ""
|
|
|
|
@ -17,56 +39,70 @@ PREV_VERSION_SUFFIX="esr"
|
|
|
|
|
#SKIP_LOCALES="" # Uncomment to skip l10n-generation
|
|
|
|
|
EOF
|
|
|
|
|
|
|
|
|
|
exit 1
|
|
|
|
|
exit 1
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if [ $# -ne 1 ]; then
|
|
|
|
|
print_usage_and_exit
|
|
|
|
|
fi
|
|
|
|
|
function check_required_tools() {
|
|
|
|
|
# check required tools
|
|
|
|
|
check_for_binary /usr/bin/hg "mercurial"
|
|
|
|
|
check_for_binary /usr/bin/jq "jq"
|
|
|
|
|
which python3 > /dev/null || exit 1
|
|
|
|
|
|
|
|
|
|
# Sourcing the given tar_stamps-file to have the variables available
|
|
|
|
|
TAR_STAMP="$1"
|
|
|
|
|
source "$TAR_STAMP" || print_usage_and_exit
|
|
|
|
|
# use parallel compression, if available
|
|
|
|
|
compression='-J'
|
|
|
|
|
pixz -h > /dev/null 2>&1
|
|
|
|
|
if (($? != 127)); then
|
|
|
|
|
compression='-Ipixz'
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Internal variables
|
|
|
|
|
BRANCH="releases/mozilla-$CHANNEL"
|
|
|
|
|
if [ "$PRODUCT" = "firefox" ]; then
|
|
|
|
|
LOCALE_FILE="firefox-$VERSION/browser/locales/l10n-changesets.json"
|
|
|
|
|
else
|
|
|
|
|
LOCALE_FILE="thunderbird-$VERSION/comm/mail/locales/l10n-changesets.json"
|
|
|
|
|
fi
|
|
|
|
|
function set_internal_variables() {
|
|
|
|
|
# Internal variables
|
|
|
|
|
BRANCH="releases/mozilla-$CHANNEL"
|
|
|
|
|
if [ "$PRODUCT" = "firefox" ]; then
|
|
|
|
|
FF_LOCALE_FILE="firefox-$VERSION/browser/locales/l10n-changesets.json"
|
|
|
|
|
else
|
|
|
|
|
FF_LOCALE_FILE="thunderbird-$VERSION/browser/locales/l10n-changesets.json"
|
|
|
|
|
TB_LOCALE_FILE="thunderbird-$VERSION/comm/mail/locales/l10n-changesets.json"
|
|
|
|
|
L10N_STRING_PATTERNS="thunderbird-$VERSION/python/l10n/tbxchannel/l10n_merge.py"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
SOURCE_TARBALL="$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz"
|
|
|
|
|
PREV_SOURCE_TARBALL="$PRODUCT-$PREV_VERSION$PREV_VERSION_SUFFIX.source.tar.xz"
|
|
|
|
|
FTP_URL="https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source"
|
|
|
|
|
FTP_CANDIDATES_BASE_URL="https://ftp.mozilla.org/pub/$PRODUCT/candidates"
|
|
|
|
|
# Make first letter of PRODCUT upper case
|
|
|
|
|
PRODUCT_CAP="${PRODUCT^}"
|
|
|
|
|
LOCALES_URL="https://product-details.mozilla.org/1.0/l10n/$PRODUCT_CAP"
|
|
|
|
|
PRODUCT_URL="https://product-details.mozilla.org/1.0/$PRODUCT.json"
|
|
|
|
|
# Exit script on CTRL+C
|
|
|
|
|
trap "exit" INT
|
|
|
|
|
SOURCE_TARBALL="$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz"
|
|
|
|
|
PREV_SOURCE_TARBALL="$PRODUCT-$PREV_VERSION$PREV_VERSION_SUFFIX.source.tar.xz"
|
|
|
|
|
FTP_URL="https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source"
|
|
|
|
|
FTP_CANDIDATES_BASE_URL="https://ftp.mozilla.org/pub/%s/candidates"
|
|
|
|
|
LOCALES_URL="https://product-details.mozilla.org/1.0/l10n"
|
|
|
|
|
PRODUCT_URL="https://product-details.mozilla.org/1.0"
|
|
|
|
|
ALREADY_EXTRACTED_LOCALES_FILE=0
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function get_ftp_candidates_url() {
|
|
|
|
|
VERSION_WITH_SUFFIX="$1"
|
|
|
|
|
echo "$FTP_CANDIDATES_BASE_URL/$VERSION_WITH_SUFFIX-candidates"
|
|
|
|
|
local CURR_PRODUCT="$1"
|
|
|
|
|
local VERSION_WITH_SUFFIX="$2"
|
|
|
|
|
printf "$FTP_CANDIDATES_BASE_URL/$VERSION_WITH_SUFFIX-candidates" "$CURR_PRODUCT"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function check_tarball_source () {
|
|
|
|
|
TARBALL=$1
|
|
|
|
|
# Print out what is going to be done:
|
|
|
|
|
if [ -e $TARBALL ]; then
|
|
|
|
|
if [ -e "$TARBALL" ]; then
|
|
|
|
|
echo "Reuse existing file"
|
|
|
|
|
elif wget --spider $FTP_URL/$TARBALL 2> /dev/null; then
|
|
|
|
|
elif wget --spider "$FTP_URL/$TARBALL" 2> /dev/null; then
|
|
|
|
|
echo "Download file"
|
|
|
|
|
else
|
|
|
|
|
echo "Mercurial checkout"
|
|
|
|
|
else
|
|
|
|
|
local CANDIDATE_TARBALL_LOCATION=""
|
|
|
|
|
CANDIDATE_TARBALL_LOCATION="$(printf "%s/%s/source/%s" "$(get_ftp_candidates_url "$PRODUCT" "$VERSION$VERSION_SUFFIX")" "$BUILD_ID" "$TARBALL" )"
|
|
|
|
|
if wget --spider "$CANDIDATE_TARBALL_LOCATION" 2> /dev/null; then
|
|
|
|
|
echo "Download UNRELEASED candidate"
|
|
|
|
|
else
|
|
|
|
|
echo "Mercurial checkout"
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ask_cont_abort_question() {
|
|
|
|
|
while true; do
|
|
|
|
|
read -p "$1 [(c)ontinue/(a)bort] " ca
|
|
|
|
|
read -r -p "$1 [(c)ontinue/(a)bort] " ca
|
|
|
|
|
case $ca in
|
|
|
|
|
[Cc]* ) return 0 ;;
|
|
|
|
|
[Aa]* ) return 1 ;;
|
|
|
|
@ -76,16 +112,16 @@ function ask_cont_abort_question() {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function check_for_binary() {
|
|
|
|
|
if ! test -x $1; then
|
|
|
|
|
if ! test -x "$1"; then
|
|
|
|
|
echo "$1 is missing: execute zypper in $2"
|
|
|
|
|
exit 5
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function get_source_stamp() {
|
|
|
|
|
BUILD_ID="$1"
|
|
|
|
|
FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url $VERSION$VERSION_SUFFIX)
|
|
|
|
|
FTP_CANDIDATES_JSON_SUFFIX="${BUILD_ID}/linux-x86_64/en-US/$PRODUCT-$VERSION$VERSION_SUFFIX.json"
|
|
|
|
|
CURR_BUILD_ID="$1"
|
|
|
|
|
FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url "$PRODUCT" "$VERSION$VERSION_SUFFIX")
|
|
|
|
|
FTP_CANDIDATES_JSON_SUFFIX="${CURR_BUILD_ID}/linux-x86_64/en-US/$PRODUCT-$VERSION$VERSION_SUFFIX.json"
|
|
|
|
|
BUILD_JSON=$(curl --silent --fail "$FTP_CANDIDATES_BASE_URL/$FTP_CANDIDATES_JSON_SUFFIX") || return 1;
|
|
|
|
|
REV=$(echo "$BUILD_JSON" | jq .moz_source_stamp)
|
|
|
|
|
SOURCE_REPO=$(echo "$BUILD_JSON" | jq .moz_source_repo)
|
|
|
|
@ -104,39 +140,43 @@ function get_source_stamp() {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function get_build_number() {
|
|
|
|
|
LAST_FOUND=""
|
|
|
|
|
VERSION_WITH_SUFFIX="$1"
|
|
|
|
|
|
|
|
|
|
BUILD_ID=$(curl --silent "$PRODUCT_URL" | jq -e '.["releases"] | .["'$PRODUCT-$VERSION_WITH_SUFFIX'"] | .["build_number"]')
|
|
|
|
|
local LAST_FOUND=""
|
|
|
|
|
local CURR_PRODUCT="$1"
|
|
|
|
|
local VERSION_WITH_SUFFIX="$2"
|
|
|
|
|
local CURR_BUILD_ID=""
|
|
|
|
|
local CURR_FTP_BASE_URL=""
|
|
|
|
|
CURR_BUILD_ID=$(curl --silent "$PRODUCT_URL/$CURR_PRODUCT.json" | jq -e '.["releases"] | .["'$CURR_PRODUCT-$VERSION_WITH_SUFFIX'"] | .["build_number"]')
|
|
|
|
|
|
|
|
|
|
# Slow fall-back
|
|
|
|
|
if [ $? -ne 0 ]; then
|
|
|
|
|
echo "Build number not found in product URL, falling back to slow FTP-parsing." 1>&2
|
|
|
|
|
FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url $VERSION_WITH_SUFFIX)
|
|
|
|
|
CURR_FTP_BASE_URL=$(get_ftp_candidates_url "$CURR_PRODUCT" "$VERSION_WITH_SUFFIX")
|
|
|
|
|
# Unfortunately, locales-files are not associated to releases, but to builds.
|
|
|
|
|
# And since we don't know which build was the final build, we grep them all from
|
|
|
|
|
# the candidates-page, sort them and take the last one which should be the oldest
|
|
|
|
|
# Error only if not even the first one exists
|
|
|
|
|
LAST_FOUND=$(curl --silent --fail "$FTP_CANDIDATES_BASE_URL/" | grep -o "build[0-9]*/" | sort | uniq | tail -n 1 | cut -d "/" -f 1)
|
|
|
|
|
LAST_FOUND=$(curl --silent --fail "$CURR_FTP_BASE_URL/" | grep -o "build[0-9]*/" | sort | uniq | tail -n 1 | cut -d "/" -f 1)
|
|
|
|
|
else
|
|
|
|
|
LAST_FOUND="build$BUILD_ID"
|
|
|
|
|
LAST_FOUND="build$CURR_BUILD_ID"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ "$LAST_FOUND" != "" ]; then
|
|
|
|
|
echo "$LAST_FOUND"
|
|
|
|
|
return 0
|
|
|
|
|
else
|
|
|
|
|
echo "Error: Could not find build-number for Firefox $VERSION_WITH_SUFFIX !" 1>&2
|
|
|
|
|
echo "Error: Could not find build-number for $CURR_PRODUCT $VERSION_WITH_SUFFIX !" 1>&2
|
|
|
|
|
return 1
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function locales_get() {
|
|
|
|
|
TMP_VERSION="$1"
|
|
|
|
|
BUILD_ID="$2"
|
|
|
|
|
URL_TO_CHECK="${LOCALES_URL}-${TMP_VERSION}"
|
|
|
|
|
FINAL_URL="${URL_TO_CHECK}-${BUILD_ID}.json"
|
|
|
|
|
CURR_PRODUCT="$1"
|
|
|
|
|
TMP_VERSION="$2"
|
|
|
|
|
CURR_BUILD_ID="$3"
|
|
|
|
|
# Make first letter of CURR_PRODUCT upper case
|
|
|
|
|
CURR_PRODUCT_CAP="${CURR_PRODUCT^}"
|
|
|
|
|
URL_TO_CHECK="${LOCALES_URL}/${CURR_PRODUCT_CAP}-${TMP_VERSION}"
|
|
|
|
|
FINAL_URL="${URL_TO_CHECK}-${CURR_BUILD_ID}.json"
|
|
|
|
|
if wget --quiet --spider "$FINAL_URL"; then
|
|
|
|
|
echo "$FINAL_URL"
|
|
|
|
|
return 0
|
|
|
|
@ -148,39 +188,49 @@ function locales_get() {
|
|
|
|
|
|
|
|
|
|
function locales_parse_file() {
|
|
|
|
|
FILE="$1"
|
|
|
|
|
cat "$FILE" | python -c "import json; import sys; \
|
|
|
|
|
python3 -c "import json; import sys; \
|
|
|
|
|
print('\n'.join(['{} {}'.format(key, value['revision']) \
|
|
|
|
|
for key, value in sorted(json.load(sys.stdin).items())]));"
|
|
|
|
|
for key, value in sorted(json.load(sys.stdin).items())]));" < "$FILE"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function locales_parse_url() {
|
|
|
|
|
URL="$1"
|
|
|
|
|
curl -s "$URL" | python -c "import json; import sys; \
|
|
|
|
|
curl -s "$URL" | python3 -c "import json; import sys; \
|
|
|
|
|
print('\n'.join(['{} {}'.format(key, value['changeset']) \
|
|
|
|
|
for key, value in sorted(json.load(sys.stdin)['locales'].items())]));"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function extract_locales_file() {
|
|
|
|
|
# still need to extract the locale information from the archive
|
|
|
|
|
echo "extract locale changesets"
|
|
|
|
|
tar -xf $SOURCE_TARBALL $LOCALE_FILE
|
|
|
|
|
if [ $ALREADY_EXTRACTED_LOCALES_FILE -ne 1 ]; then
|
|
|
|
|
# still need to extract the locale information from the archive
|
|
|
|
|
echo "extract locale changesets"
|
|
|
|
|
if [ "$PRODUCT" = "thunderbird" ]; then
|
|
|
|
|
tar -xf "$SOURCE_TARBALL" "$FF_LOCALE_FILE" "$TB_LOCALE_FILE" "$L10N_STRING_PATTERNS"
|
|
|
|
|
else
|
|
|
|
|
tar -xf "$SOURCE_TARBALL" "$FF_LOCALE_FILE"
|
|
|
|
|
fi
|
|
|
|
|
ALREADY_EXTRACTED_LOCALES_FILE=1
|
|
|
|
|
else
|
|
|
|
|
echo "Skipping locale changeset extraction, as it was already done."
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function locales_unchanged() {
|
|
|
|
|
BUILD_ID="$1"
|
|
|
|
|
PREV_BUILD_ID=$(get_build_number "$PREV_VERSION$PREV_VERSION_SUFFIX")
|
|
|
|
|
CURR_PRODUCT="$1"
|
|
|
|
|
CURR_BUILD_ID="$2"
|
|
|
|
|
PREV_BUILD_ID=$(get_build_number "$CURR_PRODUCT" "$PREV_VERSION$PREV_VERSION_SUFFIX")
|
|
|
|
|
# If no json-file for one of the versions can be found, we say "they changed"
|
|
|
|
|
prev_url=$(locales_get "$PREV_VERSION$PREV_VERSION_SUFFIX" "$PREV_BUILD_ID") || return 1
|
|
|
|
|
prev_url=$(locales_get "$CURR_PRODUCT" "$PREV_VERSION$PREV_VERSION_SUFFIX" "$PREV_BUILD_ID") || return 1
|
|
|
|
|
prev_content=$(locales_parse_url "$prev_url") || exit 1
|
|
|
|
|
|
|
|
|
|
curr_url=$(locales_get "$VERSION$VERSION_SUFFIX" "$BUILD_ID")
|
|
|
|
|
curr_url=$(locales_get "$CURR_PRODUCT" "$VERSION$VERSION_SUFFIX" "$CURR_BUILD_ID")
|
|
|
|
|
if [ $? -ne 0 ]; then
|
|
|
|
|
# We did not find a locales file upstream on the servers
|
|
|
|
|
if [ -e $SOURCE_TARBALL ]; then
|
|
|
|
|
if [ -e "$SOURCE_TARBALL" ]; then
|
|
|
|
|
# We can find out what the locales are, by extracting the json-file from the tar-ball
|
|
|
|
|
# instead of getting it from the server
|
|
|
|
|
extract_locales_file || return 1
|
|
|
|
|
curr_content=$(locales_parse_file "$LOCALE_FILE") || exit 1
|
|
|
|
|
curr_content=$(locales_parse_file "$FF_LOCALE_FILE") || exit 1
|
|
|
|
|
else
|
|
|
|
|
# We can't know what the locales are in the current version
|
|
|
|
|
return 1
|
|
|
|
@ -192,62 +242,116 @@ function locales_unchanged() {
|
|
|
|
|
diff -y --suppress-common-lines -d <(echo "$prev_content") <(echo "$curr_content")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# check required tools
|
|
|
|
|
check_for_binary /usr/bin/hg "mercurial"
|
|
|
|
|
check_for_binary /usr/bin/jq "jq"
|
|
|
|
|
which python > /dev/null || exit 1
|
|
|
|
|
function get_locales_directories() {
|
|
|
|
|
pattern="$1"
|
|
|
|
|
|
|
|
|
|
# use parallel compression, if available
|
|
|
|
|
compression='-J'
|
|
|
|
|
pixz -h > /dev/null 2>&1
|
|
|
|
|
if (($? != 127)); then
|
|
|
|
|
compression='-Ipixz'
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Get ID
|
|
|
|
|
BUILD_ID=$(get_build_number "$VERSION$VERSION_SUFFIX")
|
|
|
|
|
|
|
|
|
|
if [ -z ${SKIP_LOCALES+x} ]; then
|
|
|
|
|
if [ "$PREV_VERSION" != "" ] && locales_unchanged "$BUILD_ID"; then
|
|
|
|
|
printf "%-40s: Did not change. Skipping.\n" "locales"
|
|
|
|
|
LOCALES_CHANGED=0
|
|
|
|
|
# This file contains a list of directories, upstream uses to build locales
|
|
|
|
|
# If it is there, use it. If not, default to all FF + 3 TB-dirs.
|
|
|
|
|
if [ -e "$L10N_STRING_PATTERNS" ]; then
|
|
|
|
|
python3 -c "import os; import sys; \
|
|
|
|
|
sys.path.append(os.path.dirname(\"$L10N_STRING_PATTERNS\")); \
|
|
|
|
|
from l10n_merge import $pattern; \
|
|
|
|
|
print(\" \".join([p.strip('*') for p in $pattern]));"
|
|
|
|
|
else
|
|
|
|
|
printf "%-40s: Need to download.\n" "locales"
|
|
|
|
|
if [ "$pattern" = "GECKO_STRINGS_PATTERNS" ]; then
|
|
|
|
|
# Default of Firefox: Take all
|
|
|
|
|
echo "{lang}/"
|
|
|
|
|
else
|
|
|
|
|
# Default of Thunderbird: Take those 3 dirs
|
|
|
|
|
echo "{lang}/calendar/" "{lang}/chat/" "{lang}/mail/"
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function create_and_copy_locales() {
|
|
|
|
|
locale="$1"
|
|
|
|
|
source_base="$2"
|
|
|
|
|
source_template="$3"
|
|
|
|
|
final_dest="$4"
|
|
|
|
|
|
|
|
|
|
# Replace {lang} with the actual language-basedir
|
|
|
|
|
for template in $source_template; do
|
|
|
|
|
locale_source=$(echo "$template" | sed "s|{lang}|./$source_base/$locale|g")
|
|
|
|
|
locale_dest=$(echo "$template" | sed "s|{lang}|./$final_dest/$locale|g")
|
|
|
|
|
|
|
|
|
|
# Create intermediary folders
|
|
|
|
|
for destdir in $locale_dest; do
|
|
|
|
|
mkdir -p "$destdir"
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
# Copy over FF-files
|
|
|
|
|
cp -r "$locale_source"/* "$locale_dest"
|
|
|
|
|
done
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function check_what_changed() {
|
|
|
|
|
# Get ID
|
|
|
|
|
BUILD_ID=$(get_build_number "$PRODUCT" "$VERSION$VERSION_SUFFIX")
|
|
|
|
|
|
|
|
|
|
if [ -z ${SKIP_LOCALES+x} ]; then
|
|
|
|
|
LOCALES_CHANGED=1
|
|
|
|
|
if [ "$PREV_VERSION" != "" ]; then
|
|
|
|
|
# If we have a previous version, check either FF or (TB and FF)
|
|
|
|
|
if [ "$PRODUCT" = "firefox" ]; then
|
|
|
|
|
locales_unchanged "$PRODUCT" "$BUILD_ID"
|
|
|
|
|
else
|
|
|
|
|
FF_BUILD_ID=$(get_build_number "firefox" "$VERSION$VERSION_SUFFIX")
|
|
|
|
|
locales_unchanged "$PRODUCT" "$BUILD_ID" && locales_unchanged "firefox" "$FF_BUILD_ID"
|
|
|
|
|
fi
|
|
|
|
|
LOCALES_CHANGED=$?
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ $LOCALES_CHANGED -eq 1 ]; then
|
|
|
|
|
printf "%-40s: Need to download.\n" "locales"
|
|
|
|
|
else
|
|
|
|
|
printf "%-40s: Did not change. Skipping.\n" "locales"
|
|
|
|
|
fi
|
|
|
|
|
else
|
|
|
|
|
printf "%-40s: User forced skip (SKIP_LOCALES set)\n" "locales"
|
|
|
|
|
fi
|
|
|
|
|
else
|
|
|
|
|
printf "%-40s: User forced skip (SKIP_LOCALES set)\n" "locales"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Check what is going to be done and ask for consent
|
|
|
|
|
for ff in $SOURCE_TARBALL $SOURCE_TARBALL.asc; do
|
|
|
|
|
printf "%-40s: %s\n" $ff "$(check_tarball_source $ff)"
|
|
|
|
|
done
|
|
|
|
|
# Check what is going to be done and ask for consent
|
|
|
|
|
for ff in $SOURCE_TARBALL $SOURCE_TARBALL.asc; do
|
|
|
|
|
printf "%-40s: %s\n" "$ff" "$(check_tarball_source $ff)"
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
$(ask_cont_abort_question "Is this ok?") || exit 0
|
|
|
|
|
ask_cont_abort_question "Is this ok?" || exit 0
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Try to download tar-ball from officiall mozilla-mirror
|
|
|
|
|
if [ ! -e $SOURCE_TARBALL ]; then
|
|
|
|
|
wget https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source/$SOURCE_TARBALL
|
|
|
|
|
fi
|
|
|
|
|
# including signature
|
|
|
|
|
if [ ! -e $SOURCE_TARBALL.asc ]; then
|
|
|
|
|
wget https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source/$SOURCE_TARBALL.asc
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# we might have an upstream archive already and can skip the checkout
|
|
|
|
|
if [ -e $SOURCE_TARBALL ]; then
|
|
|
|
|
if [ -z ${SKIP_LOCALES+x} ] && [ $LOCALES_CHANGED -ne 0 ]; then
|
|
|
|
|
extract_locales_file
|
|
|
|
|
function download_release_or_candidate_file() {
|
|
|
|
|
local upstream_file="$1"
|
|
|
|
|
if [ -e "$upstream_file" ]; then
|
|
|
|
|
return;
|
|
|
|
|
fi
|
|
|
|
|
get_source_stamp "$BUILD_ID"
|
|
|
|
|
else
|
|
|
|
|
# We are working on a version that is not yet published on the mozilla mirror
|
|
|
|
|
# so we have to actually check out the repo
|
|
|
|
|
|
|
|
|
|
# mozilla
|
|
|
|
|
if [ -d $PRODUCT-$VERSION ]; then
|
|
|
|
|
pushd $PRODUCT-$VERSION || exit 1
|
|
|
|
|
if ! wget --quiet --show-progress --progress=bar "$FTP_URL/$upstream_file"; then
|
|
|
|
|
local CANDIDATE_TARBALL_LOCATION=""
|
|
|
|
|
CANDIDATE_TARBALL_LOCATION="$(printf "%s/%s/source/%s" "$(get_ftp_candidates_url "$PRODUCT" "$VERSION$VERSION_SUFFIX")" "$BUILD_ID" "$upstream_file" )"
|
|
|
|
|
wget --quiet --show-progress --progress=bar "$CANDIDATE_TARBALL_LOCATION"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function download_upstream_source_tarballs() {
|
|
|
|
|
# Try to download tar-ball from officiall mozilla-mirror
|
|
|
|
|
download_release_or_candidate_file "$SOURCE_TARBALL"
|
|
|
|
|
download_release_or_candidate_file "$SOURCE_TARBALL.asc"
|
|
|
|
|
|
|
|
|
|
# we might have an upstream archive already and can skip the checkout
|
|
|
|
|
if [ -e "$SOURCE_TARBALL" ]; then
|
|
|
|
|
if [ -z ${SKIP_LOCALES+x} ] && [ $LOCALES_CHANGED -ne 0 ]; then
|
|
|
|
|
extract_locales_file
|
|
|
|
|
fi
|
|
|
|
|
get_source_stamp "$BUILD_ID"
|
|
|
|
|
else
|
|
|
|
|
# We are working on a version that is not yet published on the mozilla mirror
|
|
|
|
|
# so we have to actually check out the repo
|
|
|
|
|
clone_and_repackage_sources
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function clone_and_repackage_sources() {
|
|
|
|
|
if [ -d "$PRODUCT-$VERSION" ]; then
|
|
|
|
|
pushd "$PRODUCT-$VERSION" || exit 1
|
|
|
|
|
_repourl=$(hg paths)
|
|
|
|
|
case "$_repourl" in
|
|
|
|
|
*$BRANCH*)
|
|
|
|
@ -258,18 +362,18 @@ else
|
|
|
|
|
* )
|
|
|
|
|
echo "removing obsolete tree"
|
|
|
|
|
popd || exit 1
|
|
|
|
|
rm -rf $PRODUCT-$VERSION
|
|
|
|
|
rm -rf "$PRODUCT-$VERSION"
|
|
|
|
|
;;
|
|
|
|
|
esac
|
|
|
|
|
fi
|
|
|
|
|
if [ ! -d $PRODUCT-$VERSION ]; then
|
|
|
|
|
if [ ! -d "$PRODUCT-$VERSION" ]; then
|
|
|
|
|
echo "cloning new $BRANCH..."
|
|
|
|
|
hg clone https://hg.mozilla.org/$BRANCH $PRODUCT-$VERSION
|
|
|
|
|
hg clone "https://hg.mozilla.org/$BRANCH $PRODUCT-$VERSION"
|
|
|
|
|
if [ "$PRODUCT" = "thunderbird" ]; then
|
|
|
|
|
hg clone https://hg.mozilla.org/releases/comm-$CHANNEL $PRODUCT-$VERSION/comm
|
|
|
|
|
hg clone "https://hg.mozilla.org/releases/comm-$CHANNEL" "$PRODUCT-$VERSION/comm"
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
pushd $PRODUCT-$VERSION || exit 1
|
|
|
|
|
pushd "$PRODUCT-$VERSION" || exit 1
|
|
|
|
|
|
|
|
|
|
# parse out the Firefox-release tag for this Thunderbird-checkout
|
|
|
|
|
if [ "$PRODUCT" = "thunderbird" ]; then
|
|
|
|
@ -279,8 +383,8 @@ else
|
|
|
|
|
FF_RELEASE_TAG="$RELEASE_TAG"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
hg update --check $FF_RELEASE_TAG
|
|
|
|
|
[ "$FF_RELEASE_TAG" == "default" ] || hg update -r $FF_RELEASE_TAG
|
|
|
|
|
hg update --check "$FF_RELEASE_TAG"
|
|
|
|
|
[ "$FF_RELEASE_TAG" == "default" ] || hg update -r "$FF_RELEASE_TAG"
|
|
|
|
|
# get repo and source stamp
|
|
|
|
|
REV=$(hg -R . parent --template="{node|short}\n")
|
|
|
|
|
SOURCE_REPO=$(hg showconfig paths.default 2>/dev/null | head -n1 | sed -e "s/^ssh:/https:/")
|
|
|
|
@ -288,9 +392,9 @@ else
|
|
|
|
|
|
|
|
|
|
if [ "$PRODUCT" = "thunderbird" ]; then
|
|
|
|
|
pushd comm || exit 1
|
|
|
|
|
hg update --check $RELEASE_TAG
|
|
|
|
|
hg update --check "$RELEASE_TAG"
|
|
|
|
|
popd || exit 1
|
|
|
|
|
rm -rf thunderbird-${VERSION}/{,comm/}other-licenses/7zstub
|
|
|
|
|
rm -rf thunderbird-"${VERSION}"/{,comm/}other-licenses/7zstub
|
|
|
|
|
fi
|
|
|
|
|
popd || exit 1
|
|
|
|
|
|
|
|
|
@ -308,56 +412,107 @@ else
|
|
|
|
|
echo "RELEASE_TIMESTAMP=$TIMESTAMP" >> "$TAR_STAMP"
|
|
|
|
|
|
|
|
|
|
echo "creating archive..."
|
|
|
|
|
tar $compression -cf $PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg --exclude=CVS $PRODUCT-$VERSION
|
|
|
|
|
fi
|
|
|
|
|
tar "$compression" -cf "$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz" --exclude=.hgtags --exclude=.hgignore --exclude=.hg --exclude=CVS "$PRODUCT-$VERSION"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if [ ! -z ${SKIP_LOCALES+x} ]; then
|
|
|
|
|
echo "Skipping locales-creation."
|
|
|
|
|
exit 0
|
|
|
|
|
fi
|
|
|
|
|
function create_locales_tarballs() {
|
|
|
|
|
if [ ! -z ${SKIP_LOCALES+x} ]; then
|
|
|
|
|
echo "Skipping locales-creation."
|
|
|
|
|
exit 0
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ "$LOCALES_CHANGED" -ne 0 ]; then
|
|
|
|
|
clone_and_repackage_locales
|
|
|
|
|
elif [ -f "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" ]; then
|
|
|
|
|
# Locales did not change, but the old tar-ball is in this directory
|
|
|
|
|
# Simply rename it:
|
|
|
|
|
echo "Moving l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz to l10n-$VERSION$VERSION_SUFFIX.tar.xz"
|
|
|
|
|
mv "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" "l10n-$VERSION$VERSION_SUFFIX.tar.xz"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if [ $LOCALES_CHANGED -ne 0 ]; then
|
|
|
|
|
function clone_and_repackage_locales() {
|
|
|
|
|
# l10n
|
|
|
|
|
echo "fetching locales..."
|
|
|
|
|
test ! -d l10n && mkdir l10n
|
|
|
|
|
jq -r 'to_entries[]| "\(.key) \(.value|.revision)"' $LOCALE_FILE | \
|
|
|
|
|
while read locale changeset ; do
|
|
|
|
|
FINAL_L10N_BASE="l10n"
|
|
|
|
|
FF_L10N_BASE="l10n" # Only change this in TB-builds to a separate dir
|
|
|
|
|
TB_L10N_BASE="l10n_tb"
|
|
|
|
|
|
|
|
|
|
# If we are doing Thunderbird, we'll have to checkout both TB and FF l10n-repos
|
|
|
|
|
# Thunderbird has one single mono-repo, FF has one for each language
|
|
|
|
|
if [ "$PRODUCT" = "thunderbird" ]; then
|
|
|
|
|
echo "Fetching Thunderbird locales..."
|
|
|
|
|
if [ -d "$TB_L10N_BASE/.hg" ]; then
|
|
|
|
|
pushd "$TB_L10N_BASE/" || exit 1
|
|
|
|
|
hg pull || exit 1
|
|
|
|
|
popd || exit 1
|
|
|
|
|
else
|
|
|
|
|
hg clone "https://hg.mozilla.org/projects/comm-l10n/" "$TB_L10N_BASE/" || exit 1
|
|
|
|
|
fi
|
|
|
|
|
# Just using the first entry here, as all languages have the same changeset
|
|
|
|
|
tb_changeset=$(jq -r 'to_entries[0]| "\(.key) \(.value|.revision)"' "$TB_LOCALE_FILE" | cut -d " " -f 2)
|
|
|
|
|
[ "$RELEASE_TAG" == "default" ] || hg -R "$TB_L10N_BASE/" up -C -r "$tb_changeset" || exit 1
|
|
|
|
|
FF_L10N_BASE="l10n_ff"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
test ! -d $FF_L10N_BASE && mkdir $FF_L10N_BASE
|
|
|
|
|
# No-op, if we are building FF:
|
|
|
|
|
test ! -d $FINAL_L10N_BASE && mkdir $FINAL_L10N_BASE
|
|
|
|
|
|
|
|
|
|
# This is only relevant for Thunderbird-builds
|
|
|
|
|
# Here, the relevant directories we need to copy from FF and from TB
|
|
|
|
|
# are specified in a python-file in the tarball
|
|
|
|
|
# Is of form '{lang}/Foo/bar/ {lang}/Baz/bar/ ..'
|
|
|
|
|
ff_locale_template=$(get_locales_directories "GECKO_STRINGS_PATTERNS")
|
|
|
|
|
tb_locale_template=$(get_locales_directories "COMM_STRINGS_PATTERNS")
|
|
|
|
|
|
|
|
|
|
echo "Fetching Browser locales..."
|
|
|
|
|
jq -r 'to_entries[]| "\(.key) \(.value|.revision)"' "$FF_LOCALE_FILE" | \
|
|
|
|
|
while read -r locale changeset ; do
|
|
|
|
|
case $locale in
|
|
|
|
|
ja-JP-mac|en-US)
|
|
|
|
|
;;
|
|
|
|
|
*)
|
|
|
|
|
echo "reading changeset information for $locale"
|
|
|
|
|
echo "fetching $locale changeset $changeset ..."
|
|
|
|
|
if [ -d "l10n/$locale/.hg" ]; then
|
|
|
|
|
pushd "l10n/$locale" || exit 1
|
|
|
|
|
hg pull
|
|
|
|
|
if [ -d "$FF_L10N_BASE/$locale/.hg" ]; then
|
|
|
|
|
pushd "$FF_L10N_BASE/$locale" || exit 1
|
|
|
|
|
hg pull || exit 1
|
|
|
|
|
popd || exit 1
|
|
|
|
|
else
|
|
|
|
|
hg clone "https://hg.mozilla.org/l10n-central/$locale" "l10n/$locale"
|
|
|
|
|
hg clone "https://hg.mozilla.org/l10n-central/$locale" "$FF_L10N_BASE/$locale" || exit 1
|
|
|
|
|
fi
|
|
|
|
|
[ "$RELEASE_TAG" == "default" ] || hg -R "$FF_L10N_BASE/$locale" up -C -r "$changeset" || exit 1
|
|
|
|
|
|
|
|
|
|
# If we are doing TB, we have to merge both l10n-repos
|
|
|
|
|
if [ "$PRODUCT" = "thunderbird" ] && test -d "$TB_L10N_BASE/$locale/" ; then
|
|
|
|
|
create_and_copy_locales "$locale" "$FF_L10N_BASE" "$ff_locale_template" "$FINAL_L10N_BASE"
|
|
|
|
|
create_and_copy_locales "$locale" "$TB_L10N_BASE" "$tb_locale_template" "$FINAL_L10N_BASE"
|
|
|
|
|
fi
|
|
|
|
|
[ "$RELEASE_TAG" == "default" ] || hg -R "l10n/$locale" up -C -r "$changeset"
|
|
|
|
|
;;
|
|
|
|
|
esac
|
|
|
|
|
done
|
|
|
|
|
echo "creating l10n archive..."
|
|
|
|
|
if [ "$PRODUCT" = "thunderbird" ]; then
|
|
|
|
|
TB_TAR_FLAGS="--exclude=browser --exclude=suite"
|
|
|
|
|
TB_TAR_FLAGS="--exclude=suite"
|
|
|
|
|
fi
|
|
|
|
|
tar $compression -cf l10n-$VERSION$VERSION_SUFFIX.tar.xz \
|
|
|
|
|
tar "$compression" -cf "l10n-$VERSION$VERSION_SUFFIX.tar.xz" \
|
|
|
|
|
--exclude=.hgtags --exclude=.hgignore --exclude=.hg \
|
|
|
|
|
$TB_TAR_FLAGS \
|
|
|
|
|
l10n
|
|
|
|
|
elif [ -f "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" ]; then
|
|
|
|
|
# Locales did not change, but the old tar-ball is in this directory
|
|
|
|
|
# Simply rename it:
|
|
|
|
|
echo "Moving l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz to l10n-$VERSION$VERSION_SUFFIX.tar.xz"
|
|
|
|
|
mv "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" "l10n-$VERSION$VERSION_SUFFIX.tar.xz"
|
|
|
|
|
fi
|
|
|
|
|
"$TB_TAR_FLAGS" \
|
|
|
|
|
"$FINAL_L10N_BASE"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if [ -e $PREV_SOURCE_TARBALL ]; then
|
|
|
|
|
echo ""
|
|
|
|
|
echo "Deleting old sources tarball $PREV_SOURCE_TARBALL"
|
|
|
|
|
$(ask_cont_abort_question "Is this ok?") || exit 0
|
|
|
|
|
rm "$PREV_SOURCE_TARBALL"
|
|
|
|
|
rm "$PREV_SOURCE_TARBALL.asc"
|
|
|
|
|
fi
|
|
|
|
|
function clean_up_old_tarballs() {
|
|
|
|
|
if [ -e "$PREV_SOURCE_TARBALL" ]; then
|
|
|
|
|
echo ""
|
|
|
|
|
echo "Deleting old sources tarball $PREV_SOURCE_TARBALL"
|
|
|
|
|
ask_cont_abort_question "Is this ok?" || exit 0
|
|
|
|
|
rm "$PREV_SOURCE_TARBALL"
|
|
|
|
|
rm "$PREV_SOURCE_TARBALL.asc"
|
|
|
|
|
# if old and new lang-tarball are there, delete the old one
|
|
|
|
|
if [ -f "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" ] && [ -f "l10n-$VERSION$VERSION_SUFFIX.tar.xz" ]; then
|
|
|
|
|
rm "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz"
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
main "$@"
|
|
|
|
|