xref: /freebsd/share/mk/meta2deps.sh (revision b0b1dbdd) 
1#!/bin/sh
2
3# NAME:
4#	meta2deps.sh - extract useful info from .meta files
5#
6# SYNOPSIS:
7#	meta2deps.sh SB="SB" "meta" ...
8#
9# DESCRIPTION:
10#	This script looks each "meta" file and extracts the
11#	information needed to deduce build and src dependencies.
12#
13#	To do this, we extract the 'CWD' record as well as all the
14#	syscall traces which describe 'R'ead, 'C'hdir and 'E'xec
15#	syscalls.
16#
17#	The typical meta file looks like::
18#.nf
19#
20#	# Meta data file "path"
21#	CMD "command-line"
22#	CWD "cwd"
23#	TARGET "target"
24#	-- command output --
25#	-- filemon acquired metadata --
26#	# buildmon version 2
27#	V 2
28#	E "pid" "path"
29#	R "pid" "path"
30#	C "pid" "cwd"
31#	R "pid" "path"
32#	X "pid" "status"
33#.fi
34#
35#	The fact that all the syscall entry lines start with a single
36#	character make these files quite easy to process using sed(1).
37#
38#	To simplify the logic the 'CWD' line is made to look like a
39#	normal 'C'hdir entry, and "cwd" is remembered so that it can
40#	be prefixed to any "path" which is not absolute.
41#
42#	If the "path" being read ends in '.srcrel' it is the content
43#	of (actually the first line of) that file that we are
44#	interested in.
45#
46#	Any "path" which lies outside of the sandbox "SB" is generally
47#	not of interest and is ignored.
48#
49#	The output, is a set of absolute paths with "SB" like:
50#.nf
51#
52#	$SB/obj-i386/bsd/gnu/lib/csu
53#	$SB/obj-i386/bsd/gnu/lib/libgcc
54#	$SB/obj-i386/bsd/include
55#	$SB/obj-i386/bsd/lib/csu/i386
56#	$SB/obj-i386/bsd/lib/libc
57#	$SB/src/bsd/include
58#	$SB/src/bsd/sys/i386/include
59#	$SB/src/bsd/sys/sys
60#	$SB/src/pan-release/rtsock
61#	$SB/src/pfe-shared/include/jnx
62#.fi
63#
64#	Which can then be further processed by 'gendirdeps.mk'
65#
66#	If we are passed 'DPDEPS='"dpdeps", then for each src file
67#	outside of "CURDIR" we read, we output a line like:
68#.nf
69#
70#	DPDEPS_$path += $RELDIR
71#.fi
72#
73#	with "$path" geting turned into reldir's, so that we can end
74#	up with a list of all the directories which depend on each src
75#	file in another directory.  This can allow for efficient yet
76#	complete testing of changes.
77
78
79# RCSid:
80#	$FreeBSD$
81#	$Id: meta2deps.sh,v 1.12 2016/12/13 20:44:16 sjg Exp $
82
83# Copyright (c) 2010-2013, Juniper Networks, Inc.
84# All rights reserved.
85#
86# Redistribution and use in source and binary forms, with or without
87# modification, are permitted provided that the following conditions
88# are met:
89# 1. Redistributions of source code must retain the above copyright
90#    notice, this list of conditions and the following disclaimer.
91# 2. Redistributions in binary form must reproduce the above copyright
92#    notice, this list of conditions and the following disclaimer in the
93#    documentation and/or other materials provided with the distribution.
94#
95# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
96# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
97# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
98# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
99# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
100# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
101# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
102# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
103# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
104# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
105# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
106
107meta2src() {
108    cat /dev/null "$@" |
109    sed -n '/^R .*\.[chyl]$/s,^..[0-9]* ,,p' |
110    sort -u
111}
112
113meta2dirs() {
114    cat /dev/null "$@" |
115    sed -n '/^R .*\/.*\.[a-z0-9][^\/]*$/s,^..[0-9]* \(.*\)/[^/]*$,\1,p' |
116    sort -u
117}
118
119add_list() {
120    sep=' '
121    suffix=
122    while :
123    do
124	case "$1" in
125	"|") sep="$1"; shift;;
126	-s) suffix="$2"; shift 2;;
127	*) break;;
128	esac
129    done
130    name=$1
131    shift
132    eval list="\$$name"
133    for top in "$@"
134    do
135	case "$sep$list$sep" in
136	*"$sep$top$suffix$sep"*) continue;;
137	esac
138	list="${list:+$list$sep}$top$suffix"
139    done
140    eval "$name=\"$list\""
141}
142
143_excludes_f() {
144    egrep -v "$EXCLUDES"
145}
146
147meta2deps() {
148    DPDEPS=
149    SRCTOPS=$SRCTOP
150    OBJROOTS=
151    EXCLUDES=
152    while :
153    do
154	case "$1" in
155	*=*) eval export "$1"; shift;;
156	-a) MACHINE_ARCH=$2; shift 2;;
157	-m) MACHINE=$2; shift 2;;
158	-C) CURDIR=$2; shift 2;;
159	-H) HOST_TARGET=$2; shift 2;;
160	-S) add_list SRCTOPS $2; shift 2;;
161	-O) add_list OBJROOTS $2; shift 2;;
162	-X) add_list EXCLUDES '|' $2; shift 2;;
163	-R) RELDIR=$2; shift 2;;
164	-T) TARGET_SPEC=$2; shift 2;;
165	*) break;;
166	esac
167    done
168
169    _th= _o=
170    case "$MACHINE" in
171    host) _ht=$HOST_TARGET;;
172    esac
173
174    for o in $OBJROOTS
175    do
176	case "$MACHINE,/$o/" in
177	host,*$HOST_TARGET*) ;;
178	*$MACHINE*|*${TARGET_SPEC:-$MACHINE}*) ;;
179	*) add_list _o $o; continue;;
180	esac
181	for x in $_ht $TARGET_SPEC $MACHINE
182	do
183	    case "$o" in
184	    "") continue;;
185	    */$x/) add_list _o ${o%$x/}; o=;;
186	    */$x) add_list _o ${o%$x}; o=;;
187	    *$x/) add_list _o ${o%$x/}; o=;;
188	    *$x) add_list _o ${o%$x}; o=;;
189	    esac
190	done
191    done
192    OBJROOTS="$_o"
193
194    case "$OBJTOP" in
195    "")
196	for o in $OBJROOTS
197	do
198	    OBJTOP=$o${TARGET_SPEC:-$MACHINE}
199	    break
200	done
201	;;
202    esac
203    src_re=
204    obj_re=
205    add_list '|' -s '/*' src_re $SRCTOPS
206    add_list '|' -s '*' obj_re $OBJROOTS
207
208    [ -z "$RELDIR" ] && unset DPDEPS
209    tf=/tmp/m2d$$-$USER
210    rm -f $tf.*
211    trap 'rm -f $tf.*; trap 0' 0
212
213    > $tf.dirdep
214    > $tf.qual
215    > $tf.srcdep
216    > $tf.srcrel
217    > $tf.dpdeps
218
219    seenit=
220    seensrc=
221    lpid=
222    case "$EXCLUDES" in
223    "") _excludes=cat;;
224    *) _excludes=_excludes_f;;
225    esac
226    # handle @list files
227    case "$@" in
228    *@[!.]*)
229	for f in "$@"
230	do
231	    case "$f" in
232	    *.meta) cat $f;;
233	    @*) xargs cat < ${f#@};;
234	    *) cat $f;;
235	    esac
236	done
237	;;
238    *) cat /dev/null "$@";;
239    esac 2> /dev/null |
240    sed -e 's,^CWD,C C,;/^[CREFLM] /!d' -e "s,',,g" |
241    $_excludes |
242    while read op pid path junk
243    do
244	: op=$op pid=$pid path=$path
245	# we track cwd and ldir (of interest) per pid
246	# CWD is bmake's cwd
247	case "$lpid,$pid" in
248	,C) CWD=$path cwd=$path ldir=$path
249	    if [ -z "$SB" ]; then
250		SB=`echo $CWD | sed 's,/obj.*,,'`
251	    fi
252	    SRCTOP=${SRCTOP:-$SB/src}
253	    continue
254	    ;;
255	$pid,$pid) ;;
256	*)
257	    case "$lpid" in
258	    "") ;;
259	    *) eval ldir_$lpid=$ldir;;
260	    esac
261	    eval ldir=\${ldir_$pid:-$CWD} cwd=\${cwd_$pid:-$CWD}
262	    lpid=$pid
263	    ;;
264	esac
265
266	case "$op,$path" in
267	W,*srcrel|*.dirdep) continue;;
268	C,*)
269	    case "$path" in
270	    /*) cwd=$path;;
271	    *) cwd=`cd $cwd/$path 2> /dev/null && /bin/pwd`;;
272	    esac
273	    # watch out for temp dirs that no longer exist
274	    test -d ${cwd:-/dev/null/no/such} || cwd=$CWD
275	    eval cwd_$pid=$cwd
276	    continue
277	    ;;
278	F,*) # $path is new pid
279	    eval cwd_$path=$cwd ldir_$path=$ldir
280	    continue
281	    ;;
282	*)  dir=${path%/*}
283	    case "$path" in
284	    $src_re|$obj_re) ;;
285	    /*/stage/*) ;;
286	    /*) continue;;
287	    *)	for path in $ldir/$path $cwd/$path
288		do
289			test -e $path && break
290		done
291		dir=${path%/*}
292		;;
293	    esac
294	    ;;
295	esac
296	# avoid repeating ourselves...
297	case "$DPDEPS,$seensrc," in
298	,*)
299	    case ",$seenit," in
300	    *,$dir,*) continue;;
301	    esac
302	    ;;
303	*,$path,*) continue;;
304	esac
305	# canonicalize if needed
306	case "/$dir/" in
307	*/../*|*/./*)
308	    rdir=$dir
309	    dir=`cd $dir 2> /dev/null && /bin/pwd`
310	    seen="$rdir,$dir"
311	    ;;
312	*)  seen=$dir;;
313	esac
314	case "$dir" in
315	${CURDIR:-.}|"") continue;;
316	$src_re)
317	    # avoid repeating ourselves...
318	    case "$DPDEPS,$seensrc," in
319	    ,*)
320		case ",$seenit," in
321		*,$dir,*) continue;;
322		esac
323		;;
324	    esac
325	    ;;
326	*)
327	    case ",$seenit," in
328	    *,$dir,*) continue;;
329	    esac
330	    ;;
331	esac
332	if [ -d $path ]; then
333	    case "$path" in
334	    */..) ldir=${dir%/*};;
335	    *) ldir=$path;;
336	    esac
337	    continue
338	fi
339	[ -f $path ] || continue
340	case "$dir" in
341	$CWD) continue;;		# ignore
342	$src_re)
343	    seenit="$seenit,$seen"
344	    echo $dir >> $tf.srcdep
345	    case "$DPDEPS,$reldir,$seensrc," in
346	    ,*) ;;
347	    *)	seensrc="$seensrc,$path"
348		echo "DPDEPS_$dir/${path##*/} += $RELDIR" >> $tf.dpdeps
349		;;
350	    esac
351	    continue
352	    ;;
353	esac
354	# if there is a .dirdep we cannot skip
355	# just because we've seen the dir before.
356	if [ -s $path.dirdep ]; then
357	    # this file contains:
358	    # '# ${RELDIR}.<machine>'
359	    echo $path.dirdep >> $tf.qual
360	    continue
361	elif [ -s $dir.dirdep ]; then
362	    echo $dir.dirdep >> $tf.qual
363	    seenit="$seenit,$seen"
364	    continue
365	fi
366	seenit="$seenit,$seen"
367	case "$dir" in
368	$obj_re)
369	    echo $dir;;
370	esac
371    done > $tf.dirdep
372    _nl=echo
373    for f in $tf.dirdep $tf.qual $tf.srcdep
374    do
375	[ -s $f ] || continue
376	case $f in
377	*qual) # a list of .dirdep files
378	    # we can prefix everything with $OBJTOP to
379	    # tell gendirdeps.mk that these are
380	    # DIRDEP entries, since they are already
381	    # qualified with .<machine> as needed.
382	    # We strip .$MACHINE though
383	    xargs cat < $f | sort -u |
384	    sed "s,^# ,,;s,^,$OBJTOP/,;s,\.${TARGET_SPEC:-$MACHINE}\$,,;s,\.$MACHINE\$,,"
385	    ;;
386	*)  sort -u $f;;
387	esac
388	_nl=:
389    done
390    if [ -s $tf.dpdeps ]; then
391	case "$DPDEPS" in
392	*/*) ;;
393	*) echo > $DPDEPS;;		# the echo is needed!
394	esac
395	sort -u $tf.dpdeps |
396	sed "s,${SRCTOP}/,,;s,${SB_BACKING_SB:-$SB}/src/,," >> $DPDEPS
397    fi
398    # ensure we produce _something_ else egrep -v gets upset
399    $_nl
400}
401
402case /$0 in
403*/meta2dep*) meta2deps "$@";;
404*/meta2dirs*) meta2dirs "$@";;
405*/meta2src*) meta2src "$@";;
406esac
407