1#!/bin/bash 2# This Source Code Form is subject to the terms of the Mozilla Public 3# License, v. 2.0. If a copy of the MPL was not distributed with this 4# file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 6# 7# This tool generates incremental update packages for the update system. 8# Author: Darin Fisher 9# 10 11. $(dirname "$0")/common.sh 12 13# ----------------------------------------------------------------------------- 14 15print_usage() { 16 notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR" 17 notice "" 18 notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE." 19 notice "" 20 notice "Options:" 21 notice " -h show this help text" 22 notice " -f clobber this file in the installation" 23 notice " Must be a path to a file to clobber in the partial update." 24 notice " -q be less verbose" 25 notice "" 26} 27 28check_for_forced_update() { 29 force_list="$1" 30 forced_file_chk="$2" 31 32 local f 33 34 if [ "$forced_file_chk" = "precomplete" ]; then 35 ## "true" *giggle* 36 return 0; 37 fi 38 39 if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then 40 ## "true" *giggle* 41 return 0; 42 fi 43 44 if [ "$forced_file_chk" = "removed-files" ]; then 45 ## "true" *giggle* 46 return 0; 47 fi 48 49 if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then 50 ## "true" *giggle* 51 return 0; 52 fi 53 54 # notarization ticket 55 if [ "$forced_file_chk" = "Contents/CodeResources" ]; then 56 ## "true" *giggle* 57 return 0; 58 fi 59 60 if [ "${forced_file_chk##*.}" = "chk" ]; then 61 ## "true" *giggle* 62 return 0; 63 fi 64 65 for f in $force_list; do 66 #echo comparing $forced_file_chk to $f 67 if [ "$forced_file_chk" = "$f" ]; then 68 ## "true" *giggle* 69 return 0; 70 fi 71 done 72 ## 'false'... because this is bash. Oh yay! 73 return 1; 74} 75 76if [ $# = 0 ]; then 77 print_usage 78 exit 1 79fi 80 81requested_forced_updates='Contents/MacOS/firefox' 82 83while getopts "hqf:" flag 84do 85 case "$flag" in 86 h) print_usage; exit 0 87 ;; 88 q) QUIET=1 89 ;; 90 f) requested_forced_updates="$requested_forced_updates $OPTARG" 91 ;; 92 ?) print_usage; exit 1 93 ;; 94 esac 95done 96 97# ----------------------------------------------------------------------------- 98 99mar_command="$MAR -V ${MOZ_PRODUCT_VERSION:?} -H ${MAR_CHANNEL_ID:?}" 100 101let arg_start=$OPTIND-1 102shift $arg_start 103 104archive="$1" 105olddir="$2" 106newdir="$3" 107# Prevent the workdir from being inside the targetdir so it isn't included in 108# the update mar. 109if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then 110 # Remove the / 111 newdir=$(echo "$newdir" | sed -e 's:\/$::') 112fi 113workdir="$(mktemp -d)" 114updatemanifestv3="$workdir/updatev3.manifest" 115archivefiles="updatev3.manifest" 116 117mkdir -p "$workdir" 118 119# Generate a list of all files in the target directory. 120pushd "$olddir" 121if test $? -ne 0 ; then 122 exit 1 123fi 124 125list_files oldfiles 126list_dirs olddirs 127 128popd 129 130pushd "$newdir" 131if test $? -ne 0 ; then 132 exit 1 133fi 134 135if [ ! -f "precomplete" ]; then 136 if [ ! -f "Contents/Resources/precomplete" ]; then 137 notice "precomplete file is missing!" 138 exit 1 139 fi 140fi 141 142list_dirs newdirs 143list_files newfiles 144 145popd 146 147# Add the type of update to the beginning of the update manifests. 148notice "" 149notice "Adding type instruction to update manifests" 150> $updatemanifestv3 151notice " type partial" 152echo "type \"partial\"" >> $updatemanifestv3 153 154notice "" 155notice "Adding file patch and add instructions to update manifests" 156 157num_oldfiles=${#oldfiles[*]} 158remove_array= 159num_removes=0 160 161for ((i=0; $i<$num_oldfiles; i=$i+1)); do 162 f="${oldfiles[$i]}" 163 164 # If this file exists in the new directory as well, then check if it differs. 165 if [ -f "$newdir/$f" ]; then 166 167 if check_for_add_if_not_update "$f"; then 168 # The full workdir may not exist yet, so create it if necessary. 169 mkdir -p `dirname "$workdir/$f"` 170 $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" 171 copy_perm "$newdir/$f" "$workdir/$f" 172 make_add_if_not_instruction "$f" "$updatemanifestv3" 173 archivefiles="$archivefiles \"$f\"" 174 continue 1 175 fi 176 177 if check_for_forced_update "$requested_forced_updates" "$f"; then 178 # The full workdir may not exist yet, so create it if necessary. 179 mkdir -p `dirname "$workdir/$f"` 180 $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" 181 copy_perm "$newdir/$f" "$workdir/$f" 182 make_add_instruction "$f" "$updatemanifestv3" 1 183 archivefiles="$archivefiles \"$f\"" 184 continue 1 185 fi 186 187 if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then 188 # Compute both the compressed binary diff and the compressed file, and 189 # compare the sizes. Then choose the smaller of the two to package. 190 dir=$(dirname "$workdir/$f") 191 mkdir -p "$dir" 192 verbose_notice "diffing \"$f\"" 193 # MBSDIFF_HOOK represents the communication interface with funsize and, 194 # if enabled, caches the intermediate patches for future use and 195 # compute avoidance 196 # 197 # An example of MBSDIFF_HOOK env variable could look like this: 198 # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user" 199 # where myscript.sh has the following usage: 200 # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \ 201 # PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL 202 # 203 # Note: patches are bzipped or xz stashed in funsize to gain more speed 204 205 # if service is not enabled then default to old behavior 206 if [ -z "$MBSDIFF_HOOK" ]; then 207 $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" 208 $XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch" 209 else 210 # if service enabled then check patch existence for retrieval 211 if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then 212 verbose_notice "file \"$f\" found in funsize, diffing skipped" 213 else 214 # if not found already - compute it and cache it for future use 215 $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" 216 $XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch" 217 $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz" 218 fi 219 fi 220 $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" 221 copy_perm "$newdir/$f" "$workdir/$f" 222 patchfile="$workdir/$f.patch.xz" 223 patchsize=$(get_file_size "$patchfile") 224 fullsize=$(get_file_size "$workdir/$f") 225 226 if [ $patchsize -lt $fullsize ]; then 227 make_patch_instruction "$f" "$updatemanifestv3" 228 mv -f "$patchfile" "$workdir/$f.patch" 229 rm -f "$workdir/$f" 230 archivefiles="$archivefiles \"$f.patch\"" 231 else 232 make_add_instruction "$f" "$updatemanifestv3" 233 rm -f "$patchfile" 234 archivefiles="$archivefiles \"$f\"" 235 fi 236 fi 237 else 238 # remove instructions are added after add / patch instructions for 239 # consistency with make_incremental_updates.py 240 remove_array[$num_removes]=$f 241 (( num_removes++ )) 242 fi 243done 244 245# Newly added files 246notice "" 247notice "Adding file add instructions to update manifests" 248num_newfiles=${#newfiles[*]} 249 250for ((i=0; $i<$num_newfiles; i=$i+1)); do 251 f="${newfiles[$i]}" 252 253 # If we've already tested this file, then skip it 254 for ((j=0; $j<$num_oldfiles; j=$j+1)); do 255 if [ "$f" = "${oldfiles[j]}" ]; then 256 continue 2 257 fi 258 done 259 260 dir=$(dirname "$workdir/$f") 261 mkdir -p "$dir" 262 263 $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" 264 copy_perm "$newdir/$f" "$workdir/$f" 265 266 if check_for_add_if_not_update "$f"; then 267 make_add_if_not_instruction "$f" "$updatemanifestv3" 268 else 269 make_add_instruction "$f" "$updatemanifestv3" 270 fi 271 272 273 archivefiles="$archivefiles \"$f\"" 274done 275 276notice "" 277notice "Adding file remove instructions to update manifests" 278for ((i=0; $i<$num_removes; i=$i+1)); do 279 f="${remove_array[$i]}" 280 verbose_notice " remove \"$f\"" 281 echo "remove \"$f\"" >> $updatemanifestv3 282done 283 284# Add remove instructions for any dead files. 285notice "" 286notice "Adding file and directory remove instructions from file 'removed-files'" 287append_remove_instructions "$newdir" "$updatemanifestv3" 288 289notice "" 290notice "Adding directory remove instructions for directories that no longer exist" 291num_olddirs=${#olddirs[*]} 292 293for ((i=0; $i<$num_olddirs; i=$i+1)); do 294 f="${olddirs[$i]}" 295 # If this dir doesn't exist in the new directory remove it. 296 if [ ! -d "$newdir/$f" ]; then 297 verbose_notice " rmdir $f/" 298 echo "rmdir \"$f/\"" >> $updatemanifestv3 299 fi 300done 301 302$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" 303 304mar_command="$mar_command -C \"$workdir\" -c output.mar" 305eval "$mar_command $archivefiles" 306mv -f "$workdir/output.mar" "$archive" 307 308# cleanup 309rm -fr "$workdir" 310 311notice "" 312notice "Finished" 313notice "" 314