1#!/bin/bash 2# This Source Code Form is subject to the terms of the Mozilla Public 3# License, v. 2.0. If a copy of the MPL was not distributed with this 4# file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 6# 7# This tool generates incremental update packages for the update system. 8# Author: Darin Fisher 9# 10 11. $(dirname "$0")/common.sh 12 13# ----------------------------------------------------------------------------- 14 15print_usage() { 16 notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR" 17 notice "" 18 notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE." 19 notice "" 20 notice "Options:" 21 notice " -h show this help text" 22 notice " -f clobber this file in the installation" 23 notice " Must be a path to a file to clobber in the partial update." 24 notice "" 25} 26 27check_for_forced_update() { 28 force_list="$1" 29 forced_file_chk="$2" 30 31 local f 32 33 if [ "$forced_file_chk" = "precomplete" ]; then 34 ## "true" *giggle* 35 return 0; 36 fi 37 38 if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then 39 ## "true" *giggle* 40 return 0; 41 fi 42 43 if [ "$forced_file_chk" = "removed-files" ]; then 44 ## "true" *giggle* 45 return 0; 46 fi 47 48 if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then 49 ## "true" *giggle* 50 return 0; 51 fi 52 53 if [ "$forced_file_chk" = "chrome.manifest" ]; then 54 ## "true" *giggle* 55 return 0; 56 fi 57 58 if [ "$forced_file_chk" = "Contents/Resources/chrome.manifest" ]; then 59 ## "true" *giggle* 60 return 0; 61 fi 62 63 if [ "${forced_file_chk##*.}" = "chk" ]; then 64 ## "true" *giggle* 65 return 0; 66 fi 67 68 for f in $force_list; do 69 #echo comparing $forced_file_chk to $f 70 if [ "$forced_file_chk" = "$f" ]; then 71 ## "true" *giggle* 72 return 0; 73 fi 74 done 75 ## 'false'... because this is bash. Oh yay! 76 return 1; 77} 78 79if [ $# = 0 ]; then 80 print_usage 81 exit 1 82fi 83 84requested_forced_updates='Contents/MacOS/firefox' 85 86while getopts "hf:" flag 87do 88 case "$flag" in 89 h) print_usage; exit 0 90 ;; 91 f) requested_forced_updates="$requested_forced_updates $OPTARG" 92 ;; 93 ?) print_usage; exit 1 94 ;; 95 esac 96done 97 98# ----------------------------------------------------------------------------- 99 100let arg_start=$OPTIND-1 101shift $arg_start 102 103archive="$1" 104olddir="$2" 105newdir="$3" 106# Prevent the workdir from being inside the targetdir so it isn't included in 107# the update mar. 108if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then 109 # Remove the / 110 newdir=$(echo "$newdir" | sed -e 's:\/$::') 111fi 112workdir="$newdir.work" 113updatemanifestv2="$workdir/updatev2.manifest" 114updatemanifestv3="$workdir/updatev3.manifest" 115archivefiles="updatev2.manifest updatev3.manifest" 116 117mkdir -p "$workdir" 118 119# Generate a list of all files in the target directory. 120pushd "$olddir" 121if test $? -ne 0 ; then 122 exit 1 123fi 124 125list_files oldfiles 126list_dirs olddirs 127 128popd 129 130pushd "$newdir" 131if test $? -ne 0 ; then 132 exit 1 133fi 134 135if [ ! -f "precomplete" ]; then 136 if [ ! -f "Contents/Resources/precomplete" ]; then 137 notice "precomplete file is missing!" 138 exit 1 139 fi 140fi 141 142list_dirs newdirs 143list_files newfiles 144 145popd 146 147# Add the type of update to the beginning of the update manifests. 148notice "" 149notice "Adding type instruction to update manifests" 150> $updatemanifestv2 151> $updatemanifestv3 152notice " type partial" 153echo "type \"partial\"" >> $updatemanifestv2 154echo "type \"partial\"" >> $updatemanifestv3 155 156notice "" 157notice "Adding file patch and add instructions to update manifests" 158 159num_oldfiles=${#oldfiles[*]} 160remove_array= 161num_removes=0 162 163for ((i=0; $i<$num_oldfiles; i=$i+1)); do 164 f="${oldfiles[$i]}" 165 166 # If this file exists in the new directory as well, then check if it differs. 167 if [ -f "$newdir/$f" ]; then 168 169 if check_for_add_if_not_update "$f"; then 170 # The full workdir may not exist yet, so create it if necessary. 171 mkdir -p `dirname "$workdir/$f"` 172 $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" 173 copy_perm "$newdir/$f" "$workdir/$f" 174 make_add_if_not_instruction "$f" "$updatemanifestv3" 175 archivefiles="$archivefiles \"$f\"" 176 continue 1 177 fi 178 179 if check_for_forced_update "$requested_forced_updates" "$f"; then 180 # The full workdir may not exist yet, so create it if necessary. 181 mkdir -p `dirname "$workdir/$f"` 182 $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" 183 copy_perm "$newdir/$f" "$workdir/$f" 184 make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1 185 archivefiles="$archivefiles \"$f\"" 186 continue 1 187 fi 188 189 if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then 190 # Compute both the compressed binary diff and the compressed file, and 191 # compare the sizes. Then choose the smaller of the two to package. 192 dir=$(dirname "$workdir/$f") 193 mkdir -p "$dir" 194 notice "diffing \"$f\"" 195 # MBSDIFF_HOOK represents the communication interface with funsize and, 196 # if enabled, caches the intermediate patches for future use and 197 # compute avoidance 198 # 199 # An example of MBSDIFF_HOOK env variable could look like this: 200 # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user" 201 # where myscript.sh has the following usage: 202 # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \ 203 # PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL 204 # 205 # Note: patches are bzipped stashed in funsize to gain more speed 206 207 # if service is not enabled then default to old behavior 208 if [ -z "$MBSDIFF_HOOK" ]; then 209 $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" 210 $BZIP2 -z9 "$workdir/$f.patch" 211 else 212 # if service enabled then check patch existence for retrieval 213 if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then 214 notice "file \"$f\" found in funsize, diffing skipped" 215 else 216 # if not found already - compute it and cache it for future use 217 $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" 218 $BZIP2 -z9 "$workdir/$f.patch" 219 $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2" 220 fi 221 fi 222 $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" 223 copy_perm "$newdir/$f" "$workdir/$f" 224 patchfile="$workdir/$f.patch.bz2" 225 patchsize=$(get_file_size "$patchfile") 226 fullsize=$(get_file_size "$workdir/$f") 227 228 if [ $patchsize -lt $fullsize ]; then 229 make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 230 mv -f "$patchfile" "$workdir/$f.patch" 231 rm -f "$workdir/$f" 232 archivefiles="$archivefiles \"$f.patch\"" 233 else 234 make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 235 rm -f "$patchfile" 236 archivefiles="$archivefiles \"$f\"" 237 fi 238 fi 239 else 240 # remove instructions are added after add / patch instructions for 241 # consistency with make_incremental_updates.py 242 remove_array[$num_removes]=$f 243 (( num_removes++ )) 244 fi 245done 246 247# Newly added files 248notice "" 249notice "Adding file add instructions to update manifests" 250num_newfiles=${#newfiles[*]} 251 252for ((i=0; $i<$num_newfiles; i=$i+1)); do 253 f="${newfiles[$i]}" 254 255 # If we've already tested this file, then skip it 256 for ((j=0; $j<$num_oldfiles; j=$j+1)); do 257 if [ "$f" = "${oldfiles[j]}" ]; then 258 continue 2 259 fi 260 done 261 262 dir=$(dirname "$workdir/$f") 263 mkdir -p "$dir" 264 265 $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" 266 copy_perm "$newdir/$f" "$workdir/$f" 267 268 if check_for_add_if_not_update "$f"; then 269 make_add_if_not_instruction "$f" "$updatemanifestv3" 270 else 271 make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 272 fi 273 274 275 archivefiles="$archivefiles \"$f\"" 276done 277 278notice "" 279notice "Adding file remove instructions to update manifests" 280for ((i=0; $i<$num_removes; i=$i+1)); do 281 f="${remove_array[$i]}" 282 notice " remove \"$f\"" 283 echo "remove \"$f\"" >> $updatemanifestv2 284 echo "remove \"$f\"" >> $updatemanifestv3 285done 286 287# Add remove instructions for any dead files. 288notice "" 289notice "Adding file and directory remove instructions from file 'removed-files'" 290append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3" 291 292notice "" 293notice "Adding directory remove instructions for directories that no longer exist" 294num_olddirs=${#olddirs[*]} 295 296for ((i=0; $i<$num_olddirs; i=$i+1)); do 297 f="${olddirs[$i]}" 298 # If this dir doesn't exist in the new directory remove it. 299 if [ ! -d "$newdir/$f" ]; then 300 notice " rmdir $f/" 301 echo "rmdir \"$f/\"" >> $updatemanifestv2 302 echo "rmdir \"$f/\"" >> $updatemanifestv3 303 fi 304done 305 306$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" 307$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" 308 309mar_command="$MAR" 310if [[ -n $MOZ_PRODUCT_VERSION ]] 311then 312 mar_command="$mar_command -V $MOZ_PRODUCT_VERSION" 313fi 314if [[ -n $MOZ_CHANNEL_ID ]] 315then 316 mar_command="$mar_command -H $MOZ_CHANNEL_ID" 317fi 318mar_command="$mar_command -C \"$workdir\" -c output.mar" 319eval "$mar_command $archivefiles" 320mv -f "$workdir/output.mar" "$archive" 321 322# cleanup 323rm -fr "$workdir" 324 325notice "" 326notice "Finished" 327notice "" 328