1# Copyright 2006-2008 The FLWOR Foundation.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14#!/bin/bash
15
16
17################################################################################
18#                                                                              #
19#                                   rbkt usage                                 #
20#                                                                              #
21################################################################################
22
23
24function usage ()
25{
26  echo "rbkt.sh [-b bucketName [-q queryName]] [-d displayFormat] [-h]"
27  echo "or"
28  echo "rbkt.sh [-b bucketName1 ... bucketNameN] [-d displayFormat] [-h]"
29  echo
30  echo "Run a single query, or all the queries inside a directory (bucket), or all"
31  echo "the queries in a list of buckets, or all the queries that can be found"
32  echo "anywhere inside the Queries dir."
33  echo
34  echo "Options : "
35  echo
36  echo "-b: Specifies the query bucket to run. <bucketName> is used to construct"
37  echo "    the full pathnames for the directories that contain a group of queries,"
38  echo "    their expected results, and their actual results. For example, if"
39  echo "    bucketName is foo, rbkt will run queries in .../test/rbkt/Queries/foo"
40  echo
41  echo "-q: The query to run. The query string is in file <queryName>.xq inside"
42  echo "    the .../test/rbkt/Queries/<bucketName> directory."
43  echo
44  echo "-d: Specifies the display format for the query results. Currently, this"
45  echo "    may be one of xml or show. The default is xml"
46  echo
47  echo "-h : help"
48}
49
50
51################################################################################
52#                                                                              #
53#                                 Test Directories                             #
54#                                                                              #
55################################################################################
56
57#
58# Function to check the existence of the required test dirs and to create
59# the results dir.
60#
61function buildDirEnv
62{
63  #echo; echo "buildDirEnv: Checking/Building Environment Directories"
64
65  if [ ! -d ${zorbaRepos} ]; then
66     echo "ERROR 5 buildDirEnv: zorba root directory ${sorbaRepos} does not exist"
67     return 5
68  fi
69
70  if [ ! -d ${testRootDir} ]; then
71     echo "ERROR 10 buildDirEnv: test root directory ${testRootDir} does not exist"
72     return 10
73  fi
74
75  if [ ! -d $scriptsDir ]; then
76     echo "ERROR 14 buildDirEnv: test scripts directory $scriptsDir does not exist"
77     return 14
78  fi
79
80  if [ ! -d ${queriesDir} ]; then
81     echo "ERROR 12 buildDirEnv: queries directory ${queriesDir} does not exist"
82     return 12
83  fi
84  if [ ! -d ${expQueryResultsDir} ]; then
85     echo "ERROR 13 buildDirEnv: expected results directory ${expQueryResultsDir} does not exist"
86     return 13
87  fi
88
89  mkdir -p ${queryResultsDir}
90
91  if [ ! -d ${docsDir} ]; then
92     echo "ERROR 12 buildDirEnv: source docs directory ${docsDir} does not exist"
93     return 12
94  fi
95  if [ ! -d ${expLoaderResultsDir} ]; then
96     echo "ERROR 13 buildDirEnv: expected docs directory ${expLoaderResultsDir} does not exist"
97     return 13
98  fi
99
100  mkdir -p ${loaderResultsDir}
101
102  return 0
103}
104
105
106################################################################################
107#                                                                              #
108#                                 Running Queries                              #
109#                                                                              #
110################################################################################
111
112
113#
114# params:
115#
116# $1 queryFile
117# $2 querySpecFile
118# $3 displayFormat
119#
120function run_query
121{
122  local queryFile="$1"
123  local querySpecFile="$2"
124  local displayFormat=$3
125
126  local error=0
127
128  local EXE=${zorbaExecDir}/apitest
129
130  if [ ! -e "${EXE}" ]; then
131    echo "ERROR 1 run_query: binary executable ${EXE} not found"
132    exit 17
133  fi
134
135  cd ${queryFile%/*}
136
137  #echo "queryFile = ${queryFile}"
138  #echo "querySpecFile = ${querySpecFile}"
139
140  if test -f $querySpecFile; then
141      XARGS=`egrep ^Args: ${querySpecFile} | cut -f2- -d:`
142      ERRS=`egrep ^Error: ${querySpecFile} | cut -f2- -d:`
143  else
144      XARGS=""
145      ERRS=""
146  fi
147
148  #echo "XARGS = ${XARGS}"
149
150  SEDARG=`echo ${testRootDir} | sed -r 's/\//\\\\\//g'`
151
152  echo "s/\$RBKT_SRC_DIR/$SEDARG/g" > sedargs
153
154  XARGS=`echo ${XARGS} | sed -f sedargs`
155
156  rm sedargs
157
158  if [ $displayFormat == "xml" ]; then
159    #echo "${EXE} ${XARGS} -r -o ${queryFile}.res ${queryFile} 2> ${queryFile}.err"
160    ${EXE} ${XARGS} -r -o "${queryFile}.res" "${queryFile}" 2> ${queryFile}.err
161  else
162    ${EXE} -o "${queryFile}.res" "${queryFile}" 2> "${queryFile}.err"
163  fi
164  error=$?
165  if [ ${error} != 0 ]; then
166    echo "ERROR 2 run_query: ${EXE} $1 failed with error code ${error}"
167    # To allow more queries to be run by the caller, do not propagate error
168  fi
169
170  cd - >/dev/null
171
172  return 0
173}
174
175
176#
177# params
178#
179# $1 bucketName
180# $2 queryName
181# $3 displayFormat
182#
183function run_query_in_bucket
184{
185  local bucketName="$1"
186  local queryName="$2"
187  local displayFormat=$3
188
189  local error=0
190
191  local inputDir="${queriesDir}/${bucketName}"
192  local queryFile="${inputDir}/${queryName}.xq"
193  local querySpec="${inputDir}/${queryName}.spec"
194
195  local expResultsDir="${expQueryResultsDir}/${bucketName}"
196  local fmtExt=""
197  if [ $displayFormat == "xml" ]; then
198    fmtExt="xml"
199  else
200    fmtExt="show"
201  fi
202  local expResultFile="${expResultsDir}/${queryName}.${fmtExt}.res"
203
204  local resultsDir="${queryResultsDir}/${bucketName}"
205  local resultFile="${resultsDir}/${queryName}.${fmtExt}.res"
206  local errorsFile="${resultsDir}/${queryName}.${fmtExt}.err"
207  local diffFile="${resultsDir}/${queryName}.${fmtExt}.diff"
208
209  if [ ! -e "${queryFile}" ]; then
210    echo "ERROR 1 run_query_in_bucket: query file ${queryFile} does not exist"
211    return 17
212  fi
213
214  #
215  # Create results directory, if it doesn't exist already.
216  # If it exists, then clean it up.
217  #
218  mkdir -p "${resultsDir}"
219  if [ $? != 0 ]; then echo "ERROR 2 run_query_in_bucket: mkdir -p failed"; exit 19; fi
220  rm -f "${resultDir}"/${queryName}.*
221
222  #
223  # Run the query
224  #
225  run_query "${queryFile}" "${querySpec}" ${displayFormat}
226  error=$?
227  if [ ${error} != 0 ]; then
228    echo "ERROR 3 run_query_in_bucket: run_query failed with error code ${error}"
229    return ${error}
230  fi
231
232  #
233  # apitest creates the result file in the same dir as the query file.
234  # So, we must move the result file to the result dir.
235  #
236  # If no result file was generated, then we create an empty one.
237  #
238  if [ -s "${inputDir}/${queryName}.xq.res.err" ]; then
239      cat "${inputDir}/${queryName}.xq.res.err" | head -1 | cut -f1 -d':' > "${inputDir}/${queryName}.xq.res"
240  fi
241
242  mkdir -p `dirname ${resultFile}`
243  if [ -e "${inputDir}/${queryName}.xq.res" ]; then
244    mv "${inputDir}/${queryName}.xq.res" "${resultFile}"
245    if [ ! -f "${resultFile}" ]; then
246      echo "ERROR 12 run_query_in_bucket: could not create output file"; exit 19;
247    fi
248  else
249    touch "${resultFile}"
250    if [ $? != 0 ]; then
251      echo "ERROR 13 run_query_in_bucket: touch failed"; exit 19;
252    fi
253  fi
254
255  if [ -e "${inputDir}/${queryName}.xq.err" ]; then
256    mv "${inputDir}/${queryName}.xq.err" ${errorsFile}
257  fi
258
259  #
260  # Do the diffs
261  #
262  if [ -e "${expResultFile}" ]; then
263    cat "${expResultFile}" | diff -I '^Duration.*:' "${resultFile}" - > "${diffFile}"
264    #cat "${expResultFile}" | diff -I '^Duration.*:' "${resultFile}" - > "${diffFile}"
265  else
266    echo "Missing expected results for ${queryName}"
267    cp "${queryFile}" "${diffFile}"
268    if [ $? != 0 ]; then echo "ERROR 21 run_query_in_bucket: cp failed"; exit 19; fi
269  fi
270
271  if [ -s "${diffFile}" ]; then
272    echo "FAILURE: ${bucketName} / ${queryName}"
273    echo
274    echo >> rbkt_summary.txt
275    echo "FAILURE : -bucket ${bucketName} -query ${queryName}" >> rbkt_summary.txt
276    echo >> rbkt_summary.txt
277    let failedQueries=failedQueries+1
278  else
279    echo "SUCCESS: ${bucketName} / ${queryName}"
280    echo
281  fi
282
283  let totalQueries=totalQueries+1
284
285  return 0
286}
287
288
289#
290# params:
291#
292# $1 bucketName
293# $2 displayFormat
294#
295function run_bucket()
296{
297  local bucketName="$1"
298  local displayFormat=$2
299  local error=0
300  local queryList=""
301  local q=""
302  local queryName=""
303
304  if [ ! -e "${queriesDir}/${bucketName}" ]; then
305    echo "ERROR 1 run_bucket: bucket ${queriesDir}/${bucketName} does not exist"
306    return 17
307  fi
308
309  queryList=`mktemp`
310  (cd "${queriesDir}/${bucketName}"; find . -name '*.xq' | cut -f2- -d/ ) > $queryList
311
312  # Uses fd 3 for reading queries.
313  # Nasty, but remember there can be many queries, and redirection
314  # applies for the entire while loop.
315  while read -u3 q; do
316    queryName=`echo $q | sed -e 's/.xq$//g'`
317    run_query_in_bucket "${bucketName}" "${queryName}" ${displayFormat}
318    error=$?
319    if [ ${error} != 0 ]; then
320      echo "ERROR 3 run_bucket: run_bucket failed with error code ${error}"
321      return ${error}
322    fi
323  done 3< $queryList
324
325  rm -f $queryList
326
327  return 0
328}
329
330
331################################################################################
332#                                                                              #
333#                                 Loading Docs                                 #
334#                                                                              #
335################################################################################
336
337
338#
339# $1 - docFile  : Full pathname of the file that contains the document text
340#
341function load_doc
342{
343  local error=0
344
345  local docFile="$1"
346
347  local EXE=${zorbaExecDir}/test_store
348
349  if [ ! -e "${EXE}" ]; then
350    echo "ERROR 1 load_doc: binary executable ${EXE} not found"
351    exit 17
352  fi
353
354  ${EXE} "${docFile}"
355  error=$?
356  if [ ${error} != 0 ]; then
357    echo "ERROR 2 load_doc: ${EXE} $1 failed with error code ${error}"
358    # To allow more docs to be loaded by the caller, do not propagate error
359  fi
360
361  return 0
362}
363
364
365#
366# $1 bucketName   : See load_bucket() function
367# $2 docName      : The doc name.
368#
369function load_doc_in_bucket()
370{
371  local error=0
372
373  local bucketName="$1"
374  local docName="$2"
375
376  local inputDir="${docsDir}/${bucketName}"
377  local docFile="${inputDir}/${docName}.xml"
378
379  local expResultsDir="${expLoaderResultsDir}/${bucketName}"
380  local expResultFile="${expResultsDir}/${docName}.xml"
381
382  local resultsDir="${loaderResultsDir}/${bucketName}"
383  local resultFile="${resultsDir}/${docName}.xml"
384  local diffFile="${resultsDir}/${docName}.diff"
385
386  if [ ! -e "${docFile}" ]; then
387    echo "ERROR 1 load_doc_in_bucket: doc file ${docFile} does not exist"
388    return 17
389  fi
390
391  #
392  # Create results directory, if it doesn't exist already.
393  # If it exists, then clean it up.
394  #
395  mkdir -p "${resultsDir}"
396  if [ $? != 0 ]; then echo "ERROR 2 load_doc_in_bucket: mkdir -p failed"; exit 19; fi
397  rm -f "${resultsDir}"/${docName}.*
398
399  #
400  # Load the doc
401  #
402  load_doc "${docFile}"
403  error=$?
404  if [ ${error} != 0 ]; then
405    echo "ERROR 3 load_doc_in_bucket: load_doc failed with error code ${error}"
406    return ${error}
407  fi
408
409  #
410  # test_store creates the result file in the same dir as the doc file.
411  # So, we must move the result file to the result dir.
412  #
413  # If no result file was generated, then we create an empty one.
414  #
415  if [ -e "${inputDir}/${docName}.xml.res" ]; then
416    mv "${inputDir}/${docName}.xml.res" "${resultFile}"
417    if [ $? != 0 ]; then echo "ERROR 12 load_doc_in_bucket: mv failed"; exit 19; fi
418  else
419    touch "${resultFile}"
420    if [ $? != 0 ]; then echo "ERROR 13 load_doc_in_bucket: touch failed"; exit 19; fi
421  fi
422
423  #
424  # Do the diffs
425  #
426  if [ -e "${expResultFile}" ]; then
427    diff "${resultFile}" "${expResultFile}" > "${diffFile}"
428  else
429    echo "Missing expected results for ${docName}"
430    cp "${docFile}" "${diffFile}"
431    if [ $? != 0 ]; then echo "ERROR 21 load_doc_in_bucket: cp failed"; exit 19; fi
432  fi
433
434  if [ -s "${diffFile}" ]; then
435    echo
436    echo "FAILURE : -bucket ${bucketName} -doc ${docName}"
437    echo
438    let failedDocs=failedDocs+1
439  else
440    echo
441    echo "SUCCESS : -bucket ${bucketName} -doc ${docName}"
442    echo
443  fi
444
445  let totalDocs=totalDocs+1
446
447  return 0
448}
449
450
451#
452# params:
453#
454# $1 bucketName
455#
456function load_bucket()
457{
458  local error=0
459  local docList=""
460  local d=""
461  local docName=""
462  local bucketName="$1"
463
464  if [ ! -e "${docsDir}/${bucketName}" ]; then
465    echo "ERROR 1 load_bucket: bucket ${docsDir}/${bucketName} does not exist"
466    return 17
467  fi
468
469  cd "${docsDir}/${bucketName}"
470  docList=`ls *.xml`
471  cd - >/dev/null
472
473  for d in $docList
474  do
475    echo ${d} | sed -e s/".xml"/""/g  > tmp_docFile
476    for docName in `cat tmp_docFile`
477    do
478      load_doc_in_bucket "${bucketName}" "${docName}"
479      error=$?
480      if [ ${error} != 0 ]; then
481        echo "ERROR 3 load_bucket: load_bucket failed with error code ${error}"
482        return ${error}
483      fi
484    done
485    rm -f tmp_docFile
486  done
487
488  return 0
489}
490