1#! /bin/sh 2# 3# abxtest - simple ABX double-blind testing script 4# Copyright (C) 2000-2004 Robert Leslie 5# 6# This program is free software; you can redistribute it and/or modify 7# it under the terms of the GNU General Public License as published by 8# the Free Software Foundation; either version 2 of the License, or 9# (at your option) any later version. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with this program; if not, write to the Free Software 18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 19# 20# $Id: abxtest,v 1.18 2004/02/23 21:34:53 rob Exp $ 21# 22 23min=10 24max=20 25goal=.05 26 27version="0.15.2 (beta)" 28publishyear="2000-2004" 29author="Robert Leslie" 30 31usage() { 32 echo >&2 "Usage: $0 [-n min] [-m max] [-g goal] A-cmd B-cmd" 33 exit $1 34} 35 36banner() { 37 echo >&2 \ 38 "ABX Double-Blind Test $version - Copyright (C) $publishyear $author" 39} 40 41while [ $# -gt 0 ] 42do 43 case "$1" in 44 --help) 45 usage 0 46 ;; 47 48 --version) 49 banner 50 exit 0 51 ;; 52 53 -n) 54 test $# -gt 1 || usage 1 55 min="$2" 56 shift 2 57 ;; 58 59 -m) 60 test $# -gt 1 || usage 1 61 max="$2" 62 shift 2 63 ;; 64 65 -g) 66 test $# -gt 1 || usage 1 67 goal="$2" 68 shift 2 69 ;; 70 71 --) 72 shift 73 break 74 ;; 75 76 -*) 77 usage 1 78 ;; 79 80 *) 81 break 82 ;; 83 esac 84done 85 86test $# -eq 2 || usage 1 87 88banner 89echo "minimum $min, maximum $max trials" 90echo "statistical goal to disprove null hypothesis is p <= $goal" 91 92A="$1" 93B="$2" 94 95echo "randomizing ..." 96 97tmp="/tmp/abx.$$" 98trap "rm -f $tmp" 0 99 100rand="${RANDOM_FILE:-/dev/random}" 101 102od -t o1 -N "$max" "$rand" >$tmp || exit 2 103exec 3<$tmp 104 105actual="" 106 107trial=1 108while read <&3 line 109do 110 set -- $line 111 shift 112 113 while [ $# -gt 0 ] 114 do 115 case $1 in 116 *[0246]) x="A" ;; 117 *[1357]) x="B" ;; 118 119 *) 120 echo >&2 "bad output from od" 121 exit 3 122 ;; 123 esac 124 shift 125 126 eval x$trial=$x 127 actual="$actual$x" 128 129 trial=`expr $trial + 1` 130 done 131done 132 133exec 3<&- 134rm -f $tmp 135 136probability() { 137 bc <<EOF 138 139 define f(x) { 140 auto i; 141 142 if (x == 0) return (1); 143 144 i = x; 145 while (--i > 1) x *= i; 146 147 return (x); 148 } 149 150 define c(n, r) { 151 return (f(n) / (f(n - r) * f(r))); 152 } 153 154 define p(r, n, p) { 155 return (c(n, r) * (p ^ r) * ((1 - p) ^ (n - r))); 156 } 157 158 define g(r, n) { 159 auto p; 160 161 while (r <= n) p += p(r++, n, 0.5); 162 163 return (p); 164 } 165 166 scale = 7; 167 g($1, $2) 168EOF 169} 170 171notdisproved() { 172 return `bc <<EOF 173 if ($1 <= $2) 1 174 if ($1 > $2) 0 175EOF` 176} 177 178if (echo "testing\c"; echo 1,2,3) | grep c >/dev/null 179then 180 if (echo -n testing; echo 1,2,3) | sed s/-n/xn/ | grep xn >/dev/null 181 then 182 n="" c=' 183' 184 else 185 n="-n" c="" 186 fi 187else 188 n="" c='\c' 189fi 190 191input="" 192votes="" 193trials=0 194correct=0 195prob=1 196 197echo 198 199trial=1 200while [ $trial -le "$min" ] || { 201 [ $trial -le "$max" ] && notdisproved $prob $goal 202} 203do 204 echo $n "trial $trial: $c" 205 X=`eval eval echo \\\\\$\\\$x$trial` 206 207 if [ -z "$input" ] 208 then 209 echo "play [a] / play [b] / play [x] / vote [A] / vote [B] / [stop]" 210 echo $n "> $c" 211 if read input 212 then 213 continue 214 else 215 break 216 fi 217 fi 218 219 case "$input" in 220 a|b|x) 221 input=`echo $input | tr abx ABX` 222 echo "playing $input ..." 223 cmd=`eval echo \\\$$input` 224 trap "" 2 225 sh -c "$cmd" 1>/dev/null 2>$tmp 226 status=$? 227 if [ $status -eq 0 -o $status -ge 128 ] 228 then 229 rm -f $tmp 230 trap - 2 231 else 232 cat >&2 $tmp 233 exit $status 234 fi 235 input="" 236 ;; 237 238 A|B) 239 echo "voting for $input" 240 eval vote$trial="$input" 241 votes="$votes$input" 242 if [ $input = "`eval echo \\\$x$trial`" ] 243 then 244 correct=`expr $correct + 1` 245 fi 246 trials=$trial 247 prob=`probability $correct $trials` 248 trial=`expr $trial + 1` 249 input="x" 250 ;; 251 252 stop) 253 echo "stopping" 254 break 255 ;; 256 257 *) 258 echo "invalid input" 259 input="" 260 ;; 261 esac 262done 263 264echo 265echo "$trials trials completed" 266echo "A = $A" 267echo "B = $B" 268 269echo 270echo " votes: $votes" 271echo " actual: $actual" 272echo $n "correct: $correct/$trials$c" 273 274if [ $trials -gt 0 ] 275then 276 echo $n " (`expr \( $correct \* 1000 / $trials + 5 \) / 10`%)$c" 277fi 278 279echo 280 281echo 282echo "probability (p) of result being the same as random guesses = $prob" 283 284if notdisproved $prob $goal 285then 286 echo "failed to disprove null hypothesis (p > $goal)" 287 exit 9 288else 289 echo "null hypothesis disproved (p <= $goal)" 290 exit 0 291fi 292