1#!/bin/sh 2 3# NAME: 4# meta2deps.sh - extract useful info from .meta files 5# 6# SYNOPSIS: 7# meta2deps.sh SB="SB" "meta" ... 8# 9# DESCRIPTION: 10# This script looks each "meta" file and extracts the 11# information needed to deduce build and src dependencies. 12# 13# To do this, we extract the 'CWD' record as well as all the 14# syscall traces which describe 'R'ead, 'C'hdir and 'E'xec 15# syscalls. 16# 17# The typical meta file looks like:: 18#.nf 19# 20# # Meta data file "path" 21# CMD "command-line" 22# CWD "cwd" 23# TARGET "target" 24# -- command output -- 25# -- filemon acquired metadata -- 26# # buildmon version 2 27# V 2 28# E "pid" "path" 29# R "pid" "path" 30# C "pid" "cwd" 31# R "pid" "path" 32# X "pid" "status" 33#.fi 34# 35# The fact that all the syscall entry lines start with a single 36# character make these files quite easy to process using sed(1). 37# 38# To simplify the logic the 'CWD' line is made to look like a 39# normal 'C'hdir entry, and "cwd" is remembered so that it can 40# be prefixed to any "path" which is not absolute. 41# 42# If the "path" being read ends in '.srcrel' it is the content 43# of (actually the first line of) that file that we are 44# interested in. 45# 46# Any "path" which lies outside of the sandbox "SB" is generally 47# not of interest and is ignored. 48# 49# The output, is a set of absolute paths with "SB" like: 50#.nf 51# 52# $SB/obj-i386/bsd/gnu/lib/csu 53# $SB/obj-i386/bsd/gnu/lib/libgcc 54# $SB/obj-i386/bsd/include 55# $SB/obj-i386/bsd/lib/csu/i386-elf 56# $SB/obj-i386/bsd/lib/libc 57# $SB/src/bsd/include 58# $SB/src/bsd/sys/i386/include 59# $SB/src/bsd/sys/sys 60# $SB/src/pan-release/rtsock 61# $SB/src/pfe-shared/include/jnx 62#.fi 63# 64# Which can then be further processed by 'gendirdeps.mk' 65# 66# If we are passed 'DPDEPS='"dpdeps", then for each src file 67# outside of "CURDIR" we read, we output a line like: 68#.nf 69# 70# DPDEPS_$path += $RELDIR 71#.fi 72# 73# with "$path" geting turned into reldir's, so that we can end 74# up with a list of all the directories which depend on each src 75# file in another directory. This can allow for efficient yet 76# complete testing of changes. 77 78 79# RCSid: 80# $Id: meta2deps.sh,v 1.19 2022/09/09 18:37:37 sjg Exp $ 81 82# Copyright (c) 2010-2013, Juniper Networks, Inc. 83# All rights reserved. 84# 85# Redistribution and use in source and binary forms, with or without 86# modification, are permitted provided that the following conditions 87# are met: 88# 1. Redistributions of source code must retain the above copyright 89# notice, this list of conditions and the following disclaimer. 90# 2. Redistributions in binary form must reproduce the above copyright 91# notice, this list of conditions and the following disclaimer in the 92# documentation and/or other materials provided with the distribution. 93# 94# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 95# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 96# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 97# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 98# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 99# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 100# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 101# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 102# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 103# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 104# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 105 106meta2src() { 107 cat /dev/null "$@" | 108 sed -n '/^R .*\.[chyl]$/s,^..[0-9]* ,,p' | 109 sort -u 110} 111 112meta2dirs() { 113 cat /dev/null "$@" | 114 sed -n '/^R .*\/.*\.[a-z0-9][^\/]*$/s,^..[0-9]* \(.*\)/[^/]*$,\1,p' | 115 sort -u 116} 117 118add_list() { 119 sep=' ' 120 suffix= 121 while : 122 do 123 case "$1" in 124 "|") sep="$1"; shift;; 125 -s) suffix="$2"; shift 2;; 126 *) break;; 127 esac 128 done 129 name=$1 130 shift 131 eval list="\$$name" 132 for top in "$@" 133 do 134 case "$sep$list$sep" in 135 *"$sep$top$suffix$sep"*) continue;; 136 esac 137 list="${list:+$list$sep}$top$suffix" 138 done 139 eval "$name=\"$list\"" 140} 141 142# some Linux systems have deprecated egrep in favor of grep -E 143# but not everyone supports that 144case "`echo bmake | egrep 'a|b' 2>&1`" in 145bmake) ;; 146*) egrep() { grep -E "$@"; } 147esac 148 149_excludes_f() { 150 egrep -v "$EXCLUDES" 151} 152 153error() { 154 echo "ERROR: $@" >&2 155 exit 1 156} 157 158meta2deps() { 159 DPDEPS= 160 SRCTOPS=$SRCTOP 161 OBJROOTS= 162 EXCLUDES= 163 while : 164 do 165 case "$1" in 166 *=*) eval export "$1"; shift;; 167 -a) MACHINE_ARCH=$2; shift 2;; 168 -m) MACHINE=$2; shift 2;; 169 -C) CURDIR=$2; shift 2;; 170 -H) HOST_TARGET=$2; shift 2;; 171 -S) add_list SRCTOPS $2; shift 2;; 172 -O) add_list OBJROOTS $2; shift 2;; 173 -X) add_list EXCLUDES '|' $2; shift 2;; 174 -R) RELDIR=$2; shift 2;; 175 -T) TARGET_SPEC=$2; shift 2;; 176 *) break;; 177 esac 178 done 179 180 _th= _o= 181 case "$MACHINE" in 182 host) _ht=$HOST_TARGET;; 183 esac 184 185 for o in $OBJROOTS 186 do 187 case "$MACHINE,/$o/" in 188 host,*$HOST_TARGET*) ;; 189 *$MACHINE*|*${TARGET_SPEC:-$MACHINE}*) ;; 190 *) add_list _o $o; continue;; 191 esac 192 for x in $_ht $TARGET_SPEC $MACHINE 193 do 194 case "$o" in 195 "") continue;; 196 */$x/) add_list _o ${o%$x/}; o=;; 197 */$x) add_list _o ${o%$x}; o=;; 198 *$x/) add_list _o ${o%$x/}; o=;; 199 *$x) add_list _o ${o%$x}; o=;; 200 esac 201 done 202 done 203 OBJROOTS="$_o" 204 205 case "$OBJTOP" in 206 "") 207 for o in $OBJROOTS 208 do 209 OBJTOP=$o${TARGET_SPEC:-$MACHINE} 210 break 211 done 212 ;; 213 esac 214 src_re= 215 obj_re= 216 add_list '|' -s '/*' src_re $SRCTOPS 217 add_list '|' -s '*' obj_re $OBJROOTS 218 219 [ -z "$RELDIR" ] && unset DPDEPS 220 tf=/tmp/m2d$$-$USER 221 rm -f $tf.* 222 trap 'rm -f $tf.*; trap 0' 0 223 224 > $tf.dirdep 225 > $tf.qual 226 > $tf.srcdep 227 > $tf.srcrel 228 > $tf.dpdeps 229 230 seenit= 231 seensrc= 232 lpid= 233 case "$EXCLUDES" in 234 "") _excludes=cat;; 235 *) _excludes=_excludes_f;; 236 esac 237 # handle @list files 238 case "$@" in 239 *@[!.]*) 240 for f in "$@" 241 do 242 case "$f" in 243 *.meta) cat $f;; 244 @*) xargs cat < ${f#@};; 245 *) cat $f;; 246 esac 247 done 248 ;; 249 *) cat /dev/null "$@";; 250 esac 2> /dev/null | 251 sed -e 's,^CWD,C C,;/^[CREFLMVX] /!d' -e "s,',,g" | 252 $_excludes | ( version=no epids= xpids= 253 while read op pid path junk 254 do 255 : op=$op pid=$pid path=$path 256 # we track cwd and ldir (of interest) per pid 257 # CWD is bmake's cwd 258 case "$lpid,$pid" in 259 ,C) CWD=$path cwd=$path ldir=$path 260 if [ -z "$SB" ]; then 261 SB=`echo $CWD | sed 's,/obj.*,,'` 262 fi 263 SRCTOP=${SRCTOP:-$SB/src} 264 case "$verion" in 265 no) ;; # ignore 266 0) error "no filemon data";; 267 *) ;; 268 esac 269 version=0 270 continue 271 ;; 272 $pid,$pid) ;; 273 *) 274 case "$lpid" in 275 "") ;; 276 *) eval ldir_$lpid=$ldir;; 277 esac 278 eval ldir=\${ldir_$pid:-$CWD} cwd=\${cwd_$pid:-$CWD} 279 lpid=$pid 280 ;; 281 esac 282 283 : op=$op path=$path 284 case "$op,$path" in 285 V,*) version=$pid; continue;; 286 W,*srcrel|*.dirdep) continue;; 287 C,*) 288 case "$path" in 289 /*) cwd=$path;; 290 *) cwd=`cd $cwd/$path 2> /dev/null && /bin/pwd`;; 291 esac 292 # watch out for temp dirs that no longer exist 293 test -d ${cwd:-/dev/null/no/such} || cwd=$CWD 294 eval cwd_$pid=$cwd 295 continue 296 ;; 297 F,*) # $path is new pid 298 eval cwd_$path=$cwd ldir_$path=$ldir 299 continue 300 ;; 301 *) dir=${path%/*} 302 case "$op" in 303 E) # setid apps get no tracing so we won't see eXit 304 case `'ls' -l $path 2> /dev/null | sed 's, .*,,'` in 305 *s*) ;; 306 *) epids="$epids $pid";; 307 esac 308 ;; 309 X) xpids="$xpids $pid"; continue;; 310 esac 311 case "$path" in 312 $src_re|$obj_re) ;; 313 /*/stage/*) ;; 314 /*) continue;; 315 *) for path in $ldir/$path $cwd/$path 316 do 317 test -e $path && break 318 done 319 dir=${path%/*} 320 ;; 321 esac 322 ;; 323 esac 324 # avoid repeating ourselves... 325 case "$DPDEPS,$seensrc," in 326 ,*) 327 case ",$seenit," in 328 *,$dir,*) continue;; 329 esac 330 ;; 331 *,$path,*) continue;; 332 esac 333 # canonicalize if needed 334 case "/$dir/" in 335 */../*|*/./*) 336 rdir=$dir 337 dir=`cd $dir 2> /dev/null && /bin/pwd` 338 seen="$rdir,$dir" 339 ;; 340 *) seen=$dir;; 341 esac 342 case "$dir" in 343 ${CURDIR:-.}|"") continue;; 344 $src_re) 345 # avoid repeating ourselves... 346 case "$DPDEPS,$seensrc," in 347 ,*) 348 case ",$seenit," in 349 *,$dir,*) continue;; 350 esac 351 ;; 352 esac 353 ;; 354 *) 355 case ",$seenit," in 356 *,$dir,*) continue;; 357 esac 358 ;; 359 esac 360 if [ -d $path ]; then 361 case "$path" in 362 */..) ldir=${dir%/*};; 363 *) ldir=$path;; 364 esac 365 continue 366 fi 367 [ -f $path ] || continue 368 case "$dir" in 369 $CWD) continue;; # ignore 370 $src_re) 371 seenit="$seenit,$seen" 372 echo $dir >> $tf.srcdep 373 case "$DPDEPS,$reldir,$seensrc," in 374 ,*) ;; 375 *) seensrc="$seensrc,$path" 376 echo "DPDEPS_$dir/${path##*/} += $RELDIR" >> $tf.dpdeps 377 ;; 378 esac 379 continue 380 ;; 381 esac 382 # if there is a .dirdep we cannot skip 383 # just because we've seen the dir before. 384 if [ -s $path.dirdep ]; then 385 # this file contains: 386 # '# ${RELDIR}.<machine>' 387 echo $path.dirdep >> $tf.qual 388 continue 389 elif [ -s $dir.dirdep ]; then 390 echo $dir.dirdep >> $tf.qual 391 seenit="$seenit,$seen" 392 continue 393 fi 394 seenit="$seenit,$seen" 395 case "$dir" in 396 $obj_re) 397 echo $dir;; 398 esac 399 done > $tf.dirdep 400 : version=$version 401 case "$version" in 402 0) error "no filemon data";; 403 esac 404 for p in $epids 405 do 406 : p=$p 407 case " $xpids " in 408 *" $p "*) ;; 409 *) error "missing eXit for pid $p";; 410 esac 411 done ) || exit 1 412 _nl=echo 413 for f in $tf.dirdep $tf.qual $tf.srcdep 414 do 415 [ -s $f ] || continue 416 case $f in 417 *qual) # a list of .dirdep files 418 # we can prefix everything with $OBJTOP to 419 # tell gendirdeps.mk that these are 420 # DIRDEP entries, since they are already 421 # qualified with .<machine> as needed. 422 # We strip .$MACHINE though 423 xargs cat < $f | sort -u | 424 sed "s,^# ,,;s,^,$OBJTOP/,;s,\.${TARGET_SPEC:-$MACHINE}\$,,;s,\.$MACHINE\$,," 425 ;; 426 *) sort -u $f;; 427 esac 428 _nl=: 429 done 430 if [ -s $tf.dpdeps ]; then 431 case "$DPDEPS" in 432 */*) ;; 433 *) echo > $DPDEPS;; # the echo is needed! 434 esac 435 sort -u $tf.dpdeps | 436 sed "s,${SRCTOP}/,,;s,${SB_BACKING_SB:-$SB}/src/,," >> $DPDEPS 437 fi 438 # ensure we produce _something_ else egrep -v gets upset 439 $_nl 440} 441 442case /$0 in 443*/meta2dep*) meta2deps "$@";; 444*/meta2dirs*) meta2dirs "$@";; 445*/meta2src*) meta2src "$@";; 446esac 447