1#!/usr/bin/env bash
2
3# Bail out when commands fail.
4set -e
5
6# The URLs where data is stored for VTK.
7readonly urlbases="https://www.vtk.org/files/ExternalData/ALGO/HASH"
8
9# Move to the top of the VTK tree.
10readonly output_base="$( pwd )"
11cd "${BASH_SOURCE%/*}/../.."
12
13info () {
14    echo >&2 "$@"
15}
16
17die () {
18    echo >&2 "$@"
19    exit 1
20}
21
22check_pipeline () {
23    echo "${PIPESTATUS[@]}" | grep -q -v "[1-9]"
24}
25
26usage () {
27    die "$0: [(--tgz|--txz|--zip)...]" \
28        "[--verbose] [-v <version>] [<tag>|<commit>]"
29}
30
31# Check for a tool to get SHA512 sums from.
32if type -p sha512sum >/dev/null; then
33    readonly sha512tool="sha512sum"
34    readonly sha512regex="s/ .*//"
35elif type -p cmake >/dev/null; then
36    readonly sha512tool="cmake -E sha512sum"
37    readonly sha512regex="s/ .*//"
38else
39    die "No 'sha512sum' tool found."
40fi
41
42compute_SHA512 () {
43    local file="$1"
44    readonly file
45    shift
46
47    $sha512tool "$file" | sed -e "$sha512regex"
48}
49
50validate () {
51    local algo="$1"
52    readonly algo
53    shift
54
55    local file="$1"
56    readonly file
57    shift
58
59    local expected="$1"
60    readonly expected
61    shift
62
63    local actual="$( "compute_$algo" "$file" )"
64    readonly actual
65
66    if ! [ "$actual" = "$expected" ]; then
67        die "Object $expected is corrupt (got $actual): $file"
68    fi
69}
70
71download () {
72    local algo="$1"
73    readonly algo
74    shift
75
76    local hash="$1"
77    readonly hash
78    shift
79
80    local path="$1"
81    readonly path
82    shift
83
84    local temppath="$path.tmp$$"
85    readonly temppath
86
87    mkdir -p "$( dirname "$path" )"
88
89    for urlbase in $urlbases; do
90        url="$( echo "$urlbase" | sed -e "s/ALGO/$algo/;s/HASH/$hash/" )"
91
92        if wget --no-verbose "$url" -O "$temppath" >&2; then
93            mv "$temppath" "$path"
94            return
95        else
96            rm -f "$temppath"
97        fi
98    done
99
100    die "failed to download data for object '$hash': '$path'"
101}
102
103find_data_objects () {
104    local revision="$1"
105    readonly revision
106    shift
107    local largedata="$1"
108    readonly largedata
109    shift
110
111    # Find all .sha512 files in the tree.
112    git ls-tree --full-tree -r "$revision" | \
113        grep '\.sha512$' | \
114        while read mode type obj path; do
115            case "$path" in
116                *.sha512) algo="SHA512" ;;
117                *)
118                    die "unknown ExternalData content link: $path"
119                    ;;
120            esac
121            # Include large data iff we are making the large data archive
122            # Include the normal data iff we are making the normal data archive
123            if git check-attr vtk-is-large-data -- $path | grep -q -v -e " set$"; then
124                if test "$largedata" != "VTK_USE_LARGE_DATA"; then
125                    # Build the path to the object.
126                    echo "$algo,$( git cat-file blob $obj ),$path"
127                fi
128            elif test "$largedata" = "VTK_USE_LARGE_DATA"; then
129                echo "$algo,$( git cat-file blob $obj ),$path"
130            fi
131        done | \
132            sort | \
133            uniq
134
135    check_pipeline
136}
137
138index_data_objects () {
139    local algo
140    local hash
141    local path
142    local file
143    local obj
144    local realpath
145    local userealpath="$1"
146    readonly userealpath
147    shift
148
149    while IFS=, read algo hash realpath; do
150        # Final path in the source tarball.
151        path=".ExternalData/$algo/$hash"
152
153        # Download it if it doesn't exist.
154        if ! [ -f "$path" ]; then
155            download "$algo" "$hash" "$path"
156        fi
157        file="$path"
158
159        # Validate the file (catches 404 pages and the like).
160        validate "$algo" "$file" "$hash"
161        obj="$( git hash-object -t blob -w "$file" )"
162        case "$userealpath" in
163          "inplace")
164              echo "100644 blob $obj	${realpath%.sha512}"
165              ;;
166          "extdata")
167              echo "100644 blob $obj	$path"
168              ;;
169        esac
170    done | \
171        git update-index --index-info
172
173    check_pipeline
174}
175
176# Puts test-data objects into an index file.
177load_testdata_objects () {
178    find_data_objects "$@" | \
179        index_data_objects "extdata"
180}
181
182# Puts VTK-data objects into an index file.
183load_data_objects () {
184    find_data_objects "$@" | \
185        index_data_objects "inplace"
186}
187
188# Loads existing data files into an index file.
189load_data_files () {
190    local revision="$1"
191    readonly revision
192    shift
193
194    git ls-tree -r "$revision" -- ".ExternalData" | \
195        git update-index --index-info
196
197    check_pipeline
198}
199
200read_all_submodules () {
201    # `git submodule foreach` clears GIT_INDEX_FILE from then environment
202    # inside its command.
203    local git_index="$GIT_INDEX_FILE"
204    export git_index
205
206    git submodule foreach --recursive --quiet '
207        gitdir="$( git rev-parse --git-dir )"
208        cd "$toplevel"
209        GIT_INDEX_FILE="$git_index"
210        export GIT_INDEX_FILE
211        git add .gitmodules 2>/dev/null
212        git rm --cached "$displaypath" >&2
213        GIT_ALTERNATE_OBJECT_DIRECTORIES="$gitdir/objects" git read-tree -i --prefix="$sm_path/" "$sha1"
214        echo "$gitdir/objects"
215    ' | \
216        tr '\n' ':'
217}
218
219read_submodules_into_index () {
220    local object_dirs="$( git rev-parse --git-dir )/objects"
221    local new_object_dirs
222
223    while git ls-files -s | grep -q -e '^160000'; do
224        new_object_dirs="$( read_all_submodules )"
225        object_dirs="$object_dirs:$new_object_dirs"
226    done
227
228    object_dirs="$( echo "$object_dirs" | sed -e 's/:$//;s/^://' )"
229    readonly object_dirs
230
231    GIT_ALTERNATE_OBJECT_DIRECTORIES="$object_dirs"
232    export GIT_ALTERNATE_OBJECT_DIRECTORIES
233}
234
235# Creates an archive of a git tree object.
236git_archive () {
237    local archive_format="$1"
238    readonly archive_format
239    shift
240
241    local revision="$1"
242    readonly revision
243    shift
244
245    local destination="$1"
246    readonly destination
247    shift
248
249    local prefix
250    if [ "$#" -gt 0 ]; then
251        prefix="$1"
252        shift
253    else
254        prefix="$destination"
255    fi
256    readonly prefix
257
258    local ext
259    local format
260    local compress
261    case "$archive_format" in
262        tgz)
263            ext=".tar.gz"
264            format="tar"
265            compress="gzip --best"
266            ;;
267        txz)
268            ext=".tar.xz"
269            format="tar"
270            compress="xz --best"
271            ;;
272        zip)
273            ext=".zip"
274            format="zip"
275            compress="cat"
276            ;;
277        *)
278            die "unknown archive format: $format"
279            ;;
280    esac
281
282    local output="$output_base/$destination$ext"
283    readonly output
284
285    local temppath="$output.tmp$$"
286    readonly temppath
287
288    git -c core.autocrlf=false archive $verbose "--format=$format" "--prefix=$prefix/" "$revision" | \
289        $compress > "$temppath"
290    mv "$temppath" "$output"
291
292    info "Wrote $output"
293}
294
295#------------------------------------------------------------------------------
296
297formats=
298commit=
299verbose=
300version=
301
302while [ "$#" -gt 0 ]; do
303    case "$1" in
304        --tgz)
305            formats="$formats tgz"
306            ;;
307        --txz)
308            formats="$formats txz"
309            ;;
310        --zip)
311            formats="$formats zip"
312            ;;
313        --verbose)
314            verbose="-v"
315            ;;
316        -v)
317            shift
318            version="$1"
319            ;;
320        --)
321            shift
322            break
323            ;;
324        -*)
325            usage
326            ;;
327        *)
328            if [ -z "$commit" ]; then
329                commit="$1"
330            else
331                usage
332            fi
333            ;;
334    esac
335    shift
336done
337
338[ "$#" -eq 0 ] || \
339    usage
340[ -z "$commit" ] && \
341    commit="HEAD"
342[ -z "$formats" ] && \
343    formats="tgz"
344
345readonly formats
346readonly verbose
347readonly commit
348
349git rev-parse --verify -q "$commit" >/dev/null || \
350    die "'$commit' is not a valid commit"
351if [ -z "$version" ]; then
352    readonly desc="$( git describe "$commit" )"
353    if ! [ "${desc:0:1}" = "v" ]; then
354        die "'git describe $commit' is '$desc'; use -v <version>"
355    fi
356    version="${desc#v}"
357    echo "$commit is version $version"
358fi
359
360readonly version
361
362GIT_INDEX_FILE="$output_base/tmp-$$-index" && \
363    trap "rm -f '$GIT_INDEX_FILE'" EXIT
364export GIT_INDEX_FILE
365
366result=0
367
368info "Loading source tree from $commit..."
369rm -f "$GIT_INDEX_FILE"
370git read-tree -m -i "$commit"
371git rm -rf -q --cached ".ExternalData"
372git submodule sync --recursive
373git submodule update --init --recursive
374read_submodules_into_index
375tree="$( git write-tree )"
376
377info "Generating source archive(s)..."
378for format in $formats; do
379    git_archive "$format" "$tree" "VTK-$version" || \
380        result=1
381done
382
383info "Loading normal data for $commit..."
384rm -f "$GIT_INDEX_FILE"
385load_testdata_objects "$commit" ""
386load_data_files "$commit"
387tree="$( git write-tree )"
388
389info "Generating normal data archive(s)..."
390for format in $formats; do
391    git_archive "$format" "$tree" "VTKData-$version" "VTK-$version" || \
392        result=1
393done
394
395info "Loading normal data tree from $commit..."
396rm -f "$GIT_INDEX_FILE"
397load_data_objects "$commit" ""
398load_data_files "$commit"
399tree="$( git write-tree )"
400
401info "Generating data archive(s)..."
402for format in $formats; do
403    git_archive "$format" "$tree" "VTKDataFiles-$version" "VTK-$version" || \
404        result=1
405done
406
407info "Loading large data for $commit..."
408rm -f "$GIT_INDEX_FILE"
409load_testdata_objects "$commit" VTK_USE_LARGE_DATA
410load_data_files "$commit"
411tree="$( git write-tree )"
412
413info "Generating large data archive(s)..."
414for format in $formats; do
415    git_archive "$format" "$tree" "VTKLargeData-$version" "VTK-$version" || \
416        result=1
417done
418
419info "Loading large data tree from $commit..."
420rm -f "$GIT_INDEX_FILE"
421load_data_objects "$commit" VTK_USE_LARGE_DATA
422load_data_files "$commit"
423tree="$( git write-tree )"
424
425info "Generating data archive(s)..."
426for format in $formats; do
427    git_archive "$format" "$tree" "VTKLargeDataFiles-$version" "VTK-$version" || \
428        result=1
429done
430
431exit "$result"
432