1#!/usr/bin/env bash 2 3# Bail out when commands fail. 4set -e 5 6# The URLs where data is stored for VTK. 7readonly urlbases="https://www.vtk.org/files/ExternalData/ALGO/HASH" 8 9# Move to the top of the VTK tree. 10readonly output_base="$( pwd )" 11cd "${BASH_SOURCE%/*}/../.." 12 13info () { 14 echo >&2 "$@" 15} 16 17die () { 18 echo >&2 "$@" 19 exit 1 20} 21 22check_pipeline () { 23 echo "${PIPESTATUS[@]}" | grep -q -v "[1-9]" 24} 25 26usage () { 27 die "$0: [(--tgz|--txz|--zip)...]" \ 28 "[--verbose] [-v <version>] [<tag>|<commit>]" 29} 30 31# Check for a tool to get SHA512 sums from. 32if type -p sha512sum >/dev/null; then 33 readonly sha512tool="sha512sum" 34 readonly sha512regex="s/ .*//" 35elif type -p cmake >/dev/null; then 36 readonly sha512tool="cmake -E sha512sum" 37 readonly sha512regex="s/ .*//" 38else 39 die "No 'sha512sum' tool found." 40fi 41 42compute_SHA512 () { 43 local file="$1" 44 readonly file 45 shift 46 47 $sha512tool "$file" | sed -e "$sha512regex" 48} 49 50validate () { 51 local algo="$1" 52 readonly algo 53 shift 54 55 local file="$1" 56 readonly file 57 shift 58 59 local expected="$1" 60 readonly expected 61 shift 62 63 local actual="$( "compute_$algo" "$file" )" 64 readonly actual 65 66 if ! [ "$actual" = "$expected" ]; then 67 die "Object $expected is corrupt (got $actual): $file" 68 fi 69} 70 71download () { 72 local algo="$1" 73 readonly algo 74 shift 75 76 local hash="$1" 77 readonly hash 78 shift 79 80 local path="$1" 81 readonly path 82 shift 83 84 local temppath="$path.tmp$$" 85 readonly temppath 86 87 mkdir -p "$( dirname "$path" )" 88 89 for urlbase in $urlbases; do 90 url="$( echo "$urlbase" | sed -e "s/ALGO/$algo/;s/HASH/$hash/" )" 91 92 if wget --no-verbose "$url" -O "$temppath" >&2; then 93 mv "$temppath" "$path" 94 return 95 else 96 rm -f "$temppath" 97 fi 98 done 99 100 die "failed to download data for object '$hash': '$path'" 101} 102 103find_data_objects () { 104 local revision="$1" 105 readonly revision 106 shift 107 local largedata="$1" 108 readonly largedata 109 shift 110 111 # Find all .sha512 files in the tree. 112 git ls-tree --full-tree -r "$revision" | \ 113 grep '\.sha512$' | \ 114 while read mode type obj path; do 115 case "$path" in 116 *.sha512) algo="SHA512" ;; 117 *) 118 die "unknown ExternalData content link: $path" 119 ;; 120 esac 121 # Include large data iff we are making the large data archive 122 # Include the normal data iff we are making the normal data archive 123 if git check-attr vtk-is-large-data -- $path | grep -q -v -e " set$"; then 124 if test "$largedata" != "VTK_USE_LARGE_DATA"; then 125 # Build the path to the object. 126 echo "$algo,$( git cat-file blob $obj ),$path" 127 fi 128 elif test "$largedata" = "VTK_USE_LARGE_DATA"; then 129 echo "$algo,$( git cat-file blob $obj ),$path" 130 fi 131 done | \ 132 sort | \ 133 uniq 134 135 check_pipeline 136} 137 138index_data_objects () { 139 local algo 140 local hash 141 local path 142 local file 143 local obj 144 local realpath 145 local userealpath="$1" 146 readonly userealpath 147 shift 148 149 while IFS=, read algo hash realpath; do 150 # Final path in the source tarball. 151 path=".ExternalData/$algo/$hash" 152 153 # Download it if it doesn't exist. 154 if ! [ -f "$path" ]; then 155 download "$algo" "$hash" "$path" 156 fi 157 file="$path" 158 159 # Validate the file (catches 404 pages and the like). 160 validate "$algo" "$file" "$hash" 161 obj="$( git hash-object -t blob -w "$file" )" 162 case "$userealpath" in 163 "inplace") 164 echo "100644 blob $obj ${realpath%.sha512}" 165 ;; 166 "extdata") 167 echo "100644 blob $obj $path" 168 ;; 169 esac 170 done | \ 171 git update-index --index-info 172 173 check_pipeline 174} 175 176# Puts test-data objects into an index file. 177load_testdata_objects () { 178 find_data_objects "$@" | \ 179 index_data_objects "extdata" 180} 181 182# Puts VTK-data objects into an index file. 183load_data_objects () { 184 find_data_objects "$@" | \ 185 index_data_objects "inplace" 186} 187 188# Loads existing data files into an index file. 189load_data_files () { 190 local revision="$1" 191 readonly revision 192 shift 193 194 git ls-tree -r "$revision" -- ".ExternalData" | \ 195 git update-index --index-info 196 197 check_pipeline 198} 199 200read_all_submodules () { 201 # `git submodule foreach` clears GIT_INDEX_FILE from then environment 202 # inside its command. 203 local git_index="$GIT_INDEX_FILE" 204 export git_index 205 206 git submodule foreach --recursive --quiet ' 207 gitdir="$( git rev-parse --git-dir )" 208 cd "$toplevel" 209 GIT_INDEX_FILE="$git_index" 210 export GIT_INDEX_FILE 211 git add .gitmodules 2>/dev/null 212 git rm --cached "$displaypath" >&2 213 GIT_ALTERNATE_OBJECT_DIRECTORIES="$gitdir/objects" git read-tree -i --prefix="$sm_path/" "$sha1" 214 echo "$gitdir/objects" 215 ' | \ 216 tr '\n' ':' 217} 218 219read_submodules_into_index () { 220 local object_dirs="$( git rev-parse --git-dir )/objects" 221 local new_object_dirs 222 223 while git ls-files -s | grep -q -e '^160000'; do 224 new_object_dirs="$( read_all_submodules )" 225 object_dirs="$object_dirs:$new_object_dirs" 226 done 227 228 object_dirs="$( echo "$object_dirs" | sed -e 's/:$//;s/^://' )" 229 readonly object_dirs 230 231 GIT_ALTERNATE_OBJECT_DIRECTORIES="$object_dirs" 232 export GIT_ALTERNATE_OBJECT_DIRECTORIES 233} 234 235# Creates an archive of a git tree object. 236git_archive () { 237 local archive_format="$1" 238 readonly archive_format 239 shift 240 241 local revision="$1" 242 readonly revision 243 shift 244 245 local destination="$1" 246 readonly destination 247 shift 248 249 local prefix 250 if [ "$#" -gt 0 ]; then 251 prefix="$1" 252 shift 253 else 254 prefix="$destination" 255 fi 256 readonly prefix 257 258 local ext 259 local format 260 local compress 261 case "$archive_format" in 262 tgz) 263 ext=".tar.gz" 264 format="tar" 265 compress="gzip --best" 266 ;; 267 txz) 268 ext=".tar.xz" 269 format="tar" 270 compress="xz --best" 271 ;; 272 zip) 273 ext=".zip" 274 format="zip" 275 compress="cat" 276 ;; 277 *) 278 die "unknown archive format: $format" 279 ;; 280 esac 281 282 local output="$output_base/$destination$ext" 283 readonly output 284 285 local temppath="$output.tmp$$" 286 readonly temppath 287 288 git -c core.autocrlf=false archive $verbose "--format=$format" "--prefix=$prefix/" "$revision" | \ 289 $compress > "$temppath" 290 mv "$temppath" "$output" 291 292 info "Wrote $output" 293} 294 295#------------------------------------------------------------------------------ 296 297formats= 298commit= 299verbose= 300version= 301 302while [ "$#" -gt 0 ]; do 303 case "$1" in 304 --tgz) 305 formats="$formats tgz" 306 ;; 307 --txz) 308 formats="$formats txz" 309 ;; 310 --zip) 311 formats="$formats zip" 312 ;; 313 --verbose) 314 verbose="-v" 315 ;; 316 -v) 317 shift 318 version="$1" 319 ;; 320 --) 321 shift 322 break 323 ;; 324 -*) 325 usage 326 ;; 327 *) 328 if [ -z "$commit" ]; then 329 commit="$1" 330 else 331 usage 332 fi 333 ;; 334 esac 335 shift 336done 337 338[ "$#" -eq 0 ] || \ 339 usage 340[ -z "$commit" ] && \ 341 commit="HEAD" 342[ -z "$formats" ] && \ 343 formats="tgz" 344 345readonly formats 346readonly verbose 347readonly commit 348 349git rev-parse --verify -q "$commit" >/dev/null || \ 350 die "'$commit' is not a valid commit" 351if [ -z "$version" ]; then 352 readonly desc="$( git describe "$commit" )" 353 if ! [ "${desc:0:1}" = "v" ]; then 354 die "'git describe $commit' is '$desc'; use -v <version>" 355 fi 356 version="${desc#v}" 357 echo "$commit is version $version" 358fi 359 360readonly version 361 362GIT_INDEX_FILE="$output_base/tmp-$$-index" && \ 363 trap "rm -f '$GIT_INDEX_FILE'" EXIT 364export GIT_INDEX_FILE 365 366result=0 367 368info "Loading source tree from $commit..." 369rm -f "$GIT_INDEX_FILE" 370git read-tree -m -i "$commit" 371git rm -rf -q --cached ".ExternalData" 372git submodule sync --recursive 373git submodule update --init --recursive 374read_submodules_into_index 375tree="$( git write-tree )" 376 377info "Generating source archive(s)..." 378for format in $formats; do 379 git_archive "$format" "$tree" "VTK-$version" || \ 380 result=1 381done 382 383info "Loading normal data for $commit..." 384rm -f "$GIT_INDEX_FILE" 385load_testdata_objects "$commit" "" 386load_data_files "$commit" 387tree="$( git write-tree )" 388 389info "Generating normal data archive(s)..." 390for format in $formats; do 391 git_archive "$format" "$tree" "VTKData-$version" "VTK-$version" || \ 392 result=1 393done 394 395info "Loading normal data tree from $commit..." 396rm -f "$GIT_INDEX_FILE" 397load_data_objects "$commit" "" 398load_data_files "$commit" 399tree="$( git write-tree )" 400 401info "Generating data archive(s)..." 402for format in $formats; do 403 git_archive "$format" "$tree" "VTKDataFiles-$version" "VTK-$version" || \ 404 result=1 405done 406 407info "Loading large data for $commit..." 408rm -f "$GIT_INDEX_FILE" 409load_testdata_objects "$commit" VTK_USE_LARGE_DATA 410load_data_files "$commit" 411tree="$( git write-tree )" 412 413info "Generating large data archive(s)..." 414for format in $formats; do 415 git_archive "$format" "$tree" "VTKLargeData-$version" "VTK-$version" || \ 416 result=1 417done 418 419info "Loading large data tree from $commit..." 420rm -f "$GIT_INDEX_FILE" 421load_data_objects "$commit" VTK_USE_LARGE_DATA 422load_data_files "$commit" 423tree="$( git write-tree )" 424 425info "Generating data archive(s)..." 426for format in $formats; do 427 git_archive "$format" "$tree" "VTKLargeDataFiles-$version" "VTK-$version" || \ 428 result=1 429done 430 431exit "$result" 432