1#!/bin/bash
2# This Source Code Form is subject to the terms of the Mozilla Public
3# License, v. 2.0. If a copy of the MPL was not distributed with this
4# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
6#
7# This tool generates incremental update packages for the update system.
8# Author: Darin Fisher
9#
10
11. $(dirname "$0")/common.sh
12
13# -----------------------------------------------------------------------------
14
15print_usage() {
16  notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR"
17  notice ""
18  notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE."
19  notice ""
20  notice "Options:"
21  notice "  -h  show this help text"
22  notice "  -f  clobber this file in the installation"
23  notice "      Must be a path to a file to clobber in the partial update."
24  notice ""
25}
26
27check_for_forced_update() {
28  force_list="$1"
29  forced_file_chk="$2"
30
31  local f
32
33  if [ "$forced_file_chk" = "precomplete" ]; then
34    ## "true" *giggle*
35    return 0;
36  fi
37
38  if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then
39    ## "true" *giggle*
40    return 0;
41  fi
42
43  if [ "$forced_file_chk" = "removed-files" ]; then
44    ## "true" *giggle*
45    return 0;
46  fi
47
48  if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then
49    ## "true" *giggle*
50    return 0;
51  fi
52
53  if [ "$forced_file_chk" = "chrome.manifest" ]; then
54    ## "true" *giggle*
55    return 0;
56  fi
57
58  if [ "$forced_file_chk" = "Contents/Resources/chrome.manifest" ]; then
59    ## "true" *giggle*
60    return 0;
61  fi
62
63  if [ "${forced_file_chk##*.}" = "chk" ]; then
64    ## "true" *giggle*
65    return 0;
66  fi
67
68  for f in $force_list; do
69    #echo comparing $forced_file_chk to $f
70    if [ "$forced_file_chk" = "$f" ]; then
71      ## "true" *giggle*
72      return 0;
73    fi
74  done
75  ## 'false'... because this is bash. Oh yay!
76  return 1;
77}
78
79if [ $# = 0 ]; then
80  print_usage
81  exit 1
82fi
83
84requested_forced_updates='Contents/MacOS/firefox'
85
86while getopts "hf:" flag
87do
88   case "$flag" in
89      h) print_usage; exit 0
90      ;;
91      f) requested_forced_updates="$requested_forced_updates $OPTARG"
92      ;;
93      ?) print_usage; exit 1
94      ;;
95   esac
96done
97
98# -----------------------------------------------------------------------------
99
100let arg_start=$OPTIND-1
101shift $arg_start
102
103archive="$1"
104olddir="$2"
105newdir="$3"
106# Prevent the workdir from being inside the targetdir so it isn't included in
107# the update mar.
108if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
109  # Remove the /
110  newdir=$(echo "$newdir" | sed -e 's:\/$::')
111fi
112workdir="$newdir.work"
113updatemanifestv2="$workdir/updatev2.manifest"
114updatemanifestv3="$workdir/updatev3.manifest"
115archivefiles="updatev2.manifest updatev3.manifest"
116
117mkdir -p "$workdir"
118
119# Generate a list of all files in the target directory.
120pushd "$olddir"
121if test $? -ne 0 ; then
122  exit 1
123fi
124
125list_files oldfiles
126list_dirs olddirs
127
128popd
129
130pushd "$newdir"
131if test $? -ne 0 ; then
132  exit 1
133fi
134
135if [ ! -f "precomplete" ]; then
136  if [ ! -f "Contents/Resources/precomplete" ]; then
137    notice "precomplete file is missing!"
138    exit 1
139  fi
140fi
141
142list_dirs newdirs
143list_files newfiles
144
145popd
146
147# Add the type of update to the beginning of the update manifests.
148notice ""
149notice "Adding type instruction to update manifests"
150> $updatemanifestv2
151> $updatemanifestv3
152notice "       type partial"
153echo "type \"partial\"" >> $updatemanifestv2
154echo "type \"partial\"" >> $updatemanifestv3
155
156notice ""
157notice "Adding file patch and add instructions to update manifests"
158
159num_oldfiles=${#oldfiles[*]}
160remove_array=
161num_removes=0
162
163for ((i=0; $i<$num_oldfiles; i=$i+1)); do
164  f="${oldfiles[$i]}"
165
166  # If this file exists in the new directory as well, then check if it differs.
167  if [ -f "$newdir/$f" ]; then
168
169    if check_for_add_if_not_update "$f"; then
170      # The full workdir may not exist yet, so create it if necessary.
171      mkdir -p `dirname "$workdir/$f"`
172      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
173      copy_perm "$newdir/$f" "$workdir/$f"
174      make_add_if_not_instruction "$f" "$updatemanifestv3"
175      archivefiles="$archivefiles \"$f\""
176      continue 1
177    fi
178
179    if check_for_forced_update "$requested_forced_updates" "$f"; then
180      # The full workdir may not exist yet, so create it if necessary.
181      mkdir -p `dirname "$workdir/$f"`
182      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
183      copy_perm "$newdir/$f" "$workdir/$f"
184      make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
185      archivefiles="$archivefiles \"$f\""
186      continue 1
187    fi
188
189    if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
190      # Compute both the compressed binary diff and the compressed file, and
191      # compare the sizes.  Then choose the smaller of the two to package.
192      dir=$(dirname "$workdir/$f")
193      mkdir -p "$dir"
194      notice "diffing \"$f\""
195      # MBSDIFF_HOOK represents the communication interface with funsize and,
196      # if enabled, caches the intermediate patches for future use and
197      # compute avoidance
198      #
199      # An example of MBSDIFF_HOOK env variable could look like this:
200      # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user"
201      # where myscript.sh has the following usage:
202      # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
203      #   PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
204      #
205      # Note: patches are bzipped stashed in funsize to gain more speed
206
207      # if service is not enabled then default to old behavior
208      if [ -z "$MBSDIFF_HOOK" ]; then
209        $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
210        $BZIP2 -z9 "$workdir/$f.patch"
211      else
212        # if service enabled then check patch existence for retrieval
213        if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
214          notice "file \"$f\" found in funsize, diffing skipped"
215        else
216          # if not found already - compute it and cache it for future use
217          $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
218          $BZIP2 -z9 "$workdir/$f.patch"
219          $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
220        fi
221      fi
222      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
223      copy_perm "$newdir/$f" "$workdir/$f"
224      patchfile="$workdir/$f.patch.bz2"
225      patchsize=$(get_file_size "$patchfile")
226      fullsize=$(get_file_size "$workdir/$f")
227
228      if [ $patchsize -lt $fullsize ]; then
229        make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
230        mv -f "$patchfile" "$workdir/$f.patch"
231        rm -f "$workdir/$f"
232        archivefiles="$archivefiles \"$f.patch\""
233      else
234        make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
235        rm -f "$patchfile"
236        archivefiles="$archivefiles \"$f\""
237      fi
238    fi
239  else
240    # remove instructions are added after add / patch instructions for
241    # consistency with make_incremental_updates.py
242    remove_array[$num_removes]=$f
243    (( num_removes++ ))
244  fi
245done
246
247# Newly added files
248notice ""
249notice "Adding file add instructions to update manifests"
250num_newfiles=${#newfiles[*]}
251
252for ((i=0; $i<$num_newfiles; i=$i+1)); do
253  f="${newfiles[$i]}"
254
255  # If we've already tested this file, then skip it
256  for ((j=0; $j<$num_oldfiles; j=$j+1)); do
257    if [ "$f" = "${oldfiles[j]}" ]; then
258      continue 2
259    fi
260  done
261
262  dir=$(dirname "$workdir/$f")
263  mkdir -p "$dir"
264
265  $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
266  copy_perm "$newdir/$f" "$workdir/$f"
267
268  if check_for_add_if_not_update "$f"; then
269    make_add_if_not_instruction "$f" "$updatemanifestv3"
270  else
271    make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
272  fi
273
274
275  archivefiles="$archivefiles \"$f\""
276done
277
278notice ""
279notice "Adding file remove instructions to update manifests"
280for ((i=0; $i<$num_removes; i=$i+1)); do
281  f="${remove_array[$i]}"
282  notice "     remove \"$f\""
283  echo "remove \"$f\"" >> $updatemanifestv2
284  echo "remove \"$f\"" >> $updatemanifestv3
285done
286
287# Add remove instructions for any dead files.
288notice ""
289notice "Adding file and directory remove instructions from file 'removed-files'"
290append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
291
292notice ""
293notice "Adding directory remove instructions for directories that no longer exist"
294num_olddirs=${#olddirs[*]}
295
296for ((i=0; $i<$num_olddirs; i=$i+1)); do
297  f="${olddirs[$i]}"
298  # If this dir doesn't exist in the new directory remove it.
299  if [ ! -d "$newdir/$f" ]; then
300    notice "      rmdir $f/"
301    echo "rmdir \"$f/\"" >> $updatemanifestv2
302    echo "rmdir \"$f/\"" >> $updatemanifestv3
303  fi
304done
305
306$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
307$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
308
309mar_command="$MAR"
310if [[ -n $MOZ_PRODUCT_VERSION ]]
311then
312  mar_command="$mar_command -V $MOZ_PRODUCT_VERSION"
313fi
314if [[ -n $MOZ_CHANNEL_ID ]]
315then
316  mar_command="$mar_command -H $MOZ_CHANNEL_ID"
317fi
318mar_command="$mar_command -C \"$workdir\" -c output.mar"
319eval "$mar_command $archivefiles"
320mv -f "$workdir/output.mar" "$archive"
321
322# cleanup
323rm -fr "$workdir"
324
325notice ""
326notice "Finished"
327notice ""
328