1#! /bin/sh
2#
3# abxtest - simple ABX double-blind testing script
4# Copyright (C) 2000-2004 Robert Leslie
5#
6# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License as published by
8# the Free Software Foundation; either version 2 of the License, or
9# (at your option) any later version.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19#
20# $Id: abxtest,v 1.18 2004/02/23 21:34:53 rob Exp $
21#
22
23min=10
24max=20
25goal=.05
26
27version="0.15.2 (beta)"
28publishyear="2000-2004"
29author="Robert Leslie"
30
31usage() {
32    echo >&2 "Usage: $0 [-n min] [-m max] [-g goal] A-cmd B-cmd"
33    exit $1
34}
35
36banner() {
37    echo >&2  \
38	"ABX Double-Blind Test $version - Copyright (C) $publishyear $author"
39}
40
41while [ $# -gt 0 ]
42do
43    case "$1" in
44	--help)
45	    usage 0
46	    ;;
47
48	--version)
49	    banner
50	    exit 0
51	    ;;
52
53	-n)
54	    test $# -gt 1 || usage 1
55	    min="$2"
56	    shift 2
57	    ;;
58
59	-m)
60	    test $# -gt 1 || usage 1
61	    max="$2"
62	    shift 2
63	    ;;
64
65	-g)
66	    test $# -gt 1 || usage 1
67	    goal="$2"
68	    shift 2
69	    ;;
70
71	--)
72	    shift
73	    break
74	    ;;
75
76	-*)
77	    usage 1
78	    ;;
79
80	*)
81	    break
82	    ;;
83    esac
84done
85
86test $# -eq 2 || usage 1
87
88banner
89echo "minimum $min, maximum $max trials"
90echo "statistical goal to disprove null hypothesis is p <= $goal"
91
92A="$1"
93B="$2"
94
95echo "randomizing ..."
96
97tmp="/tmp/abx.$$"
98trap "rm -f $tmp" 0
99
100rand="${RANDOM_FILE:-/dev/random}"
101
102od -t o1 -N "$max" "$rand" >$tmp || exit 2
103exec 3<$tmp
104
105actual=""
106
107trial=1
108while read <&3 line
109do
110    set -- $line
111    shift
112
113    while [ $# -gt 0 ]
114    do
115	case $1 in
116	    *[0246]) x="A" ;;
117	    *[1357]) x="B" ;;
118
119	    *)
120		echo >&2 "bad output from od"
121		exit 3
122		;;
123	esac
124	shift
125
126	eval x$trial=$x
127	actual="$actual$x"
128
129	trial=`expr $trial + 1`
130    done
131done
132
133exec 3<&-
134rm -f $tmp
135
136probability() {
137    bc <<EOF
138
139    define f(x) {
140	auto i;
141
142	if (x == 0) return (1);
143
144	i = x;
145	while (--i > 1) x *= i;
146
147	return (x);
148    }
149
150    define c(n, r) {
151	return (f(n) / (f(n - r) * f(r)));
152    }
153
154    define p(r, n, p) {
155	return (c(n, r) * (p ^ r) * ((1 - p) ^ (n - r)));
156    }
157
158    define g(r, n) {
159	auto p;
160
161	while (r <= n) p += p(r++, n, 0.5);
162
163	return (p);
164    }
165
166    scale = 7;
167    g($1, $2)
168EOF
169}
170
171notdisproved() {
172    return `bc <<EOF
173    if ($1 <= $2) 1
174    if ($1 >  $2) 0
175EOF`
176}
177
178if (echo "testing\c"; echo 1,2,3) | grep c >/dev/null
179then
180    if (echo -n testing; echo 1,2,3) | sed s/-n/xn/ | grep xn >/dev/null
181    then
182	n="" c='
183'
184    else
185	n="-n" c=""
186    fi
187else
188    n="" c='\c'
189fi
190
191input=""
192votes=""
193trials=0
194correct=0
195prob=1
196
197echo
198
199trial=1
200while [ $trial -le "$min" ] || {
201      [ $trial -le "$max" ] && notdisproved $prob $goal
202}
203do
204    echo $n "trial $trial: $c"
205    X=`eval eval echo \\\\\$\\\$x$trial`
206
207    if [ -z "$input" ]
208    then
209	echo "play [a] / play [b] / play [x] / vote [A] / vote [B] / [stop]"
210	echo $n "> $c"
211	if read input
212	then
213	    continue
214	else
215	    break
216	fi
217    fi
218
219    case "$input" in
220	a|b|x)
221	    input=`echo $input | tr abx ABX`
222	    echo "playing $input ..."
223	    cmd=`eval echo \\\$$input`
224	    trap "" 2
225	    sh -c "$cmd" 1>/dev/null 2>$tmp
226	    status=$?
227	    if [ $status -eq 0 -o $status -ge 128 ]
228	    then
229		rm -f $tmp
230		trap - 2
231	    else
232		cat >&2 $tmp
233		exit $status
234	    fi
235	    input=""
236	    ;;
237
238	A|B)
239	    echo "voting for $input"
240	    eval vote$trial="$input"
241	    votes="$votes$input"
242	    if [ $input = "`eval echo \\\$x$trial`" ]
243	    then
244		correct=`expr $correct + 1`
245	    fi
246	    trials=$trial
247	    prob=`probability $correct $trials`
248	    trial=`expr $trial + 1`
249	    input="x"
250	    ;;
251
252	stop)
253	    echo "stopping"
254	    break
255	    ;;
256
257	*)
258	    echo "invalid input"
259	    input=""
260	    ;;
261    esac
262done
263
264echo
265echo "$trials trials completed"
266echo "A = $A"
267echo "B = $B"
268
269echo
270echo "  votes: $votes"
271echo " actual: $actual"
272echo $n "correct: $correct/$trials$c"
273
274if [ $trials -gt 0 ]
275then
276    echo $n " (`expr \( $correct \* 1000 / $trials + 5 \) / 10`%)$c"
277fi
278
279echo
280
281echo
282echo "probability (p) of result being the same as random guesses = $prob"
283
284if notdisproved $prob $goal
285then
286    echo "failed to disprove null hypothesis (p > $goal)"
287    exit 9
288else
289    echo "null hypothesis disproved (p <= $goal)"
290    exit 0
291fi
292