1#!/usr/bin/awk -f
2#* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
3#*                                                                           *
4#*                  This file is part of the program and library             *
5#*         SCIP --- Solving Constraint Integer Programs                      *
6#*                                                                           *
7#*                                                                           *
8#*    Copyright (C) 2002-2021 Konrad-Zuse-Zentrum                            *
9#*                            fuer Informationstechnik Berlin                *
10#*                                                                           *
11#*  SCIP is distributed under the terms of the ZIB Academic License.         *
12#*                                                                           *
13#*  You should have received a copy of the ZIB Academic License              *
14#*  along with SCIP; see the file COPYING. If not email to scip@zib.de.      *
15#*                                                                           *
16#* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
17#
18#@file    cmpres.awk
19#@brief   SCIP Check Comparison Report Generator
20#@author  Tobias Achterberg
21#@author  Robert Waniek
22#@author  Marc Pfetsch
23#@author  Timo Berthold
24#
25function abs(x)
26{
27   return x < 0 ? -x : x;
28}
29
30function min(x,y)
31{
32   return (x) < (y) ? (x) : (y);
33}
34
35function max(x,y)
36{
37   return (x) > (y) ? (x) : (y);
38}
39
40function max3(x,y,z)
41{
42   return (x) >= (y) ? max(x,z) : max(y,z);
43}
44
45function ceil(x)
46{
47   return (x == int(x) ? x : (x < 0 ? int(x) : int(x+1)));
48}
49
50function floor(x)
51{
52   return (x == int(x) ? x : (x < 0 ? int(x-1) : int(x)));
53}
54
55function fracceil(x,f)
56{
57   return ceil(x/f)*f;
58}
59
60function fracfloor(x,f)
61{
62   return floor(x/f)*f;
63}
64
65function sign(x)
66{
67   return (x >= 0 ? 1.0 : -1.0);
68}
69
70function mod(x,m)
71{
72   return (x - m*floor(x/m));
73}
74
75function printhline(nsolver,short, printsoltimes, printconfs)
76{
77   for( s = 0; s < nsolver; ++s )
78   {
79
80      if( s == 0 )
81         printf("--------------------+-+---------+--------+");
82      else
83      {
84         if( !short )
85            printf("-+---------+--------+------+------+");
86         else
87            printf("-+---------+--------+");
88      }
89      if( printsoltimes )
90      {
91         if( s == 0 )
92            printf("---------+--------+");
93         else
94            printf("------+------+");
95      }
96      if( printconfs )
97      {
98         if( s == 0 )
99            printf("--------+-------+");
100         else
101            printf("--------+-------+");
102      }
103   }
104   printf("-------------\n");
105}
106
107function isfaster(t,reft,tol)
108{
109   return (t < 1.0/tol*reft && t <= reft - 0.2);
110}
111
112function isslower(t,reft,tol)
113{
114   return isfaster(reft, t, tol);
115}
116
117function texcompstr(val,refval, x,s,t)
118{
119   x = floor(100*(val/refval-1.0)+0.5);
120   s = "";
121   t = "";
122   if( x < 0 )
123   {
124      if( x <= -texcolorlimit )
125      {
126         s = "\\textcolor{red}{\\raisebox{0.25ex}{\\tiny $-$}";
127         t = "}";
128      }
129      else
130         s = "\\raisebox{0.25ex}{\\tiny $-$}";
131   }
132   else if( x > 0 )
133   {
134      if( x >= +texcolorlimit )
135      {
136         s = "\\textcolor{blue}{\\raisebox{0.25ex}{\\tiny $+$}";
137         t = "}";
138      }
139      else
140         s = "\\raisebox{0.25ex}{\\tiny $+$}";
141   }
142
143   return sprintf("%s%d%s", s, abs(x), t);
144}
145
146function texstring(s, ts)
147{
148   ts = s;
149   gsub(/_/, "\\_", ts);
150
151   return ts;
152}
153
154function texint(x, ts,r)
155{
156   ts = "";
157   x = floor(x);
158   while( x != 0 )
159   {
160      r = mod(x, 1000);
161      x = floor(x/1000);
162      if( ts != "" )
163         ts = "\\," ts;
164
165      if ( x > 0 )
166         ts = sprintf("%03d", r) "" ts;
167      else
168         ts = r "" ts;
169   }
170
171   return ts;
172}
173
174function texsolvername(s, sname)
175{
176   sname = solvername[s];
177   if( setname[sname] != "" )
178      sname = setname[sname];
179   else
180   {
181      sub(/.*:/, "", sname);
182      sub(/.*_/, "", sname);
183      if( length(sname) > 12 )
184         sname = substr(sname, length(sname)-11, 12);
185   }
186
187   return sname;
188}
189
190# McNemar statistical test
191#
192# input: two arrays of Boolean values whose difference should be tested for statistical significance and the length of
193#        both arrays
194#
195# output: the chi_squared value, (which needs to be transformed to the desired p-value, see also function chi_to_p
196function mcnemar(ref_array, solver_array, problistlen)
197{
198   chi_squared = 0.0;
199   b = 0;
200   c = 0;
201
202   # count the number of entries for which both arrays differ,
203   # separately for both possible differences (TRUE/FALSE and FALSE/TRUE)
204   for( i = 0; i < problistlen; ++i )
205   {
206      if( ref_array[i] && !solver_array[i] )
207         b++;
208      else if( !ref_array[i] && solver_array[i] )
209         c++;
210   }
211
212   # textbook McNemar formula, the square of the differences of both counters divided by their sum is supposed to be
213   # chi-square distributed for a random experiment
214   if( b + c > 0 )
215      chi_squared = (b-c)*(b-c)/(1.0*(b+c));
216
217   return chi_squared;
218}
219
220# check significance of chi-square distribution with degree 1 using quantiles
221function chi_to_p(chi)
222{
223   if ( chi < 3.841 )
224      printf(", 0.05 < p           X");
225   else if ( 3.841 <= chi && chi < 5.024 )  # quantile for 1 - 0.05 = 0.95
226      printf(", p ~ (0.025, 0.05]  !");
227   else if ( 5.024 <= chi && chi < 6.635 )  # quantile for 1 - 0.025 = 0.975
228      printf(", p ~ (0.01, 0.025] !!");
229   else if ( 6.635 <= chi && chi < 7.879 )  # quantile for 1 - 0.01 = 0.99
230      printf(", p ~ (0.005, 0.01]!!!");
231   else if ( 7.879 <= chi && chi < 12.116 ) # quantile for 1 - 0.005 = 0.995
232      printf(", p ~(0.0005,0.005]!!!");
233   else # quantile for 1 - 0.0005 = 0.9995
234      printf(", p <= 0.0005      !!!");
235}
236
237# swaps position i and j of array a
238function swap(a, i, j) {
239   t = a[i]; a[i] = a[j]; a[j] = t;
240}
241
242
243# quicksort algorithm that sorts by the absolute values in non-increasing order
244function qsort(a, left, right)
245{
246   # stop recursion
247   if (left >= right)
248      return;
249
250   # use a random pivot element
251   swap(a, left, left+int((right-left+1)*rand()));
252   last = left;
253
254   for( i = left+1; i <= right; i++ )
255   {
256      if( abs(a[i]) < abs(a[left]) )
257         swap(a, ++last, i);
258   }
259
260   # swap back pivot, recursive calls
261   swap(a, left, last);
262   qsort(a, left, last-1);
263   qsort(a, last+1, right);
264}
265
266# copy time array
267function parse_time(ref_array,solver_array,time,o,printorder,probidx,problistlen)
268{
269   s = printorder[o];
270   p0 = printorder[0];
271   n = 0;
272
273   for( i = 0; i < problistlen; i++ )
274   {
275      p = problist[i];
276      if(probidx[p,p0] != "" && probidx[p,s] != "")
277      {
278         ref_array[n] = time[p0,probidx[p,p0]];
279         solver_array[n] = time[s,probidx[p,s]];
280         n++;
281      }
282   }
283}
284
285# copy node array
286function parse_nodes(ref_array,solver_array,nodes,o,probidx,problistlen,status,infinity)
287{
288   s = printorder[o];
289   p0 = printorder[0];
290   n = 0;
291
292   for( i = 0; i < problistlen; i++ )
293   {
294      p = problist[i];
295      if(probidx[p,p0] != "" && probidx[p,s] != "")
296      {
297         if( status[p0,probidx[p,p0]] == "timeout" || status[p0,probidx[p,p0]] == "memlimit" )
298            ref_array[n] = infinity;
299         else
300            ref_array[n] = nodes[p0,probidx[p,p0]];
301
302         if( status[s,probidx[p,s]] == "timeout" || status[s,probidx[p,s]] == "memlimit" )
303            solver_array[n] = infinity;
304         else
305            solver_array[n] = nodes[s,probidx[p,s]];
306
307         n++;
308      }
309   }
310}
311
312# filter results for Wilcoxon test, only keep data for instances for which both, the relative difference and the
313# absolute difference are larger than the given thresholds rel_epsilon and abs_delta
314function filter(ref_array, solver_array, problistlen, rel_epsilon, abs_delta)
315{
316   n = 0;
317
318   for( i = 0; i < problistlen; i++ )
319   {
320      diff = abs(solver_array[i] - ref_array[i]);
321      if( diff > abs_delta && diff / max3(abs(solver_array[i]), abs(ref_array[i]), 1.0) > rel_epsilon )
322      {
323         ref_array[n] = ref_array[i];
324         solver_array[n] = solver_array[i];
325         n++;
326      }
327   }
328
329   return n;
330}
331
332# computes the improvement/degradation factors (always >= 1) and marks them by opposite signs
333# (negative sign if solver_array[i] is faster than ref_array[i])
334function factorize(ref_array, solver_array, n, maxval)
335{
336   for( i = 0; i < n; ++i )
337   {
338      if( ref_array[i] >= maxval && solver_array[i] < maxval )
339         ref_array[i] = -1.0 * maxval;
340      else if( ref_array[i] < maxval && solver_array[i] >= maxval )
341         ref_array[i] = 1.0 * maxval;
342      else if( ref_array[i] == 0.0 && solver_array[i] == 0.0 )
343         ref_array[i] = 0.0;
344      else if( ref_array[i] == 0.0 )
345         ref_array[i] = 1.0 * maxval;
346      else if( solver_array[i] == 0.0 )
347         ref_array[i] = -1.0 * maxval;
348      else if( solver_array[i] / ref_array[i]  < 1.0 )
349         ref_array[i] = -1.0 * ref_array[i] / solver_array[i];
350      else
351         ref_array[i] = solver_array[i] / ref_array[i];
352
353      solver_array[i] = 0.0;
354   }
355}
356
357
358# Wilcoxon signed rank test
359#
360# input: two arrays of real values whose difference should be tested for statistical significance, the length of
361#        both arrays, and the employed timelimit (to rank instances for which one solver hit the timelimit equally)
362#
363# output: the z value, (which needs to be transformed to the desired p-value, see also function z_to_p
364#
365# note: to get meaningful results, some form of filtering should be applied to remove nearly-identical results from the
366#       test set, see also the filter() function
367function wilcoxon(ref_array, solver_array, problistlen, timelimit)
368{
369   w_minus = 0;
370   w_plus = 0;
371
372   # avoid degenerate case
373   if ( problistlen == 0 )
374      return 0.0;
375
376   # calculate difference
377   for( i = 0; i < problistlen; i++ )
378   {
379      differences[i] = ref_array[i] - solver_array[i];
380      if( (ref_array[i] >= timelimit) != (solver_array[i] >= timelimit) )
381         differences[i] = sign(differences[i]) * timelimit;
382   }
383
384   # sort differences by their absolute values
385   qsort(differences, 0, problistlen-1);
386
387   i = 0;
388
389   # calculate rank sums
390   while( i < problistlen )
391   {
392      i_start = i;
393      i_end = i;
394      i_sum = 0;
395
396      # use average in case of tied samples (identical differences) - always executed once
397      while ( i_end < problistlen && abs((abs(differences[i_start]) - abs(differences[i_end]))) < 1e-06 )
398      {
399         i_sum += i_end;
400         i_end++;
401      }
402
403      i_sum = i_sum/(i_end - i_start);
404
405      # add (average) rank values to rank sums
406      #
407      # in the default case that the value is unique, this loop and the previous loop are traversed exactly once, s.t.
408      # the value is simply added to one of the sums
409      while ( i < i_end )
410      {
411         if( differences[i] < 0 )
412            w_minus += (i_sum+1);
413         else if( differences[i] > 0 )
414            w_plus += (i_sum+1);
415         i++;
416      }
417   }
418
419   # apply Wilcoxon formula
420   w = 1.0 * min(w_minus, w_plus);
421
422   # apply correction for small number of instances
423   if ( problistlen <= 60 )
424      z = (abs(w - 0.25 * problistlen * (problistlen + 1.0)) - 0.5) / sqrt(problistlen * (problistlen+1) * (2*problistlen + 1.0)/24.0);
425   else
426      z = (w - 0.25 * problistlen * (problistlen + 1.0)) / sqrt(problistlen * (problistlen + 1.0) * (2*problistlen + 1.0)/24.0);
427
428   return z;
429}
430
431# check significance of z-value with respect to the normal distribution with mean 0 and variance 1
432function z_to_p(z)
433{
434   # check whether z lies in (1 - 0.05) quantile -> null hypothesis is accepted
435   if ( -1.960 <= z && z <= 1.960 )       # quantile for 1 - 0.05
436      printf(", 0.05 <= p          X");
437   else if ( -2.241 <= z && z <= 2.241 ) # quantile for 1 - 0.025
438      printf(", p ~ [0.025, 0.05)  !");
439   else if ( - 2.576 <= z && z <= 2.576 ) # quantile for 1 - 0.01
440      printf(", p ~ [0.01, 0.025) !!");
441   else if ( - 2.807 <= z && z <= 2.807 ) # quantile for 1 - 0.005
442      printf(", p ~ [0.005, 0.01)!!!");
443   else if ( - 3.481 <= z && z <= 3.481 ) # quantile for 1 - 0.0005
444      printf(", p ~[0.0005,0.005)!!!");
445   else # quantile for 1 - 0.0005 = 0.9995
446      printf(", p < 0.0005       !!!");
447}
448
449BEGIN {
450
451   short = 0;  #for each non reference solver, only absolute time and number of nodes are printed
452   printsoltimes = 0; # for reference solver, absolute time to first and best solution are printed, for other solvers the corresponding ratios
453                      #! please NOTE that this additional output is currently only available for SCIP .res-files created with the evalcheck.sh script and
454                      #  the flag printsoltimes = 1 set in check.awk. If other solvers are involved, leave this flag set to 0.
455   printconfs = 0;
456   printgap = 0; # if timeout, then print absolute gap at termination in time column, if gap is finite
457   printsoltimes = !short && printsoltimes; # short deactivates the detailed solution times output
458   infinity = 1e+20;
459   timegeomshift = 1.0;
460   nodegeomshift = 100.0;
461   mintime = 0.5;
462   wintolerance = 1.1;
463   markbettertime = 1.1;
464   markworsetime  = 1.1;
465   markbetternodes = 5.0;
466   markworsenodes  = 5.0;
467   onlymarked = 0;
468   onlyprocessed = 0;
469   maxscore = 10.0;
470   consistency = 1;
471   onlyfeasible = 0;
472   onlyinfeasible = 0;
473   onlyfail = 0;
474   exclude = "";
475   texfile = "";
476   texincfile = "";
477   texsummaryfile = "";
478   texsummaryheader = 0;
479   texsummaryweight = 0;
480   texsummaryshifted = 0;
481   texcolorlimit = 5;
482   textestset = "";
483   texcmpfile = "";
484   texcmpfiledir = "";
485   texcmpfilename = "";
486   diagramfile = "";
487   diagramnsteps = 5;        # number of steps at the time line
488   diagramyellowbg = 0;      # Should the background be colored in SCIP-HP-yellow ?
489   diagramgerman = 0;        # Soll die Beschriftung deutsch sein?
490   thesisnames = 0;
491   nsetnames = 0;
492   onlygroup = 0;
493   group = "default";
494
495   problistlen = 0;
496   nsolver = 0;
497   nprobs[nsolver] = 0;
498   fulltotaltime = 0.0;
499}
500/^=group=/ {
501   group = $2;
502}
503/^=setname= / {
504   if( setorder[$2] == 0 )
505   {
506      nsetnames++;
507      setorder[$2] = nsetnames;
508      setname[$2] = $3;
509      for( i = 4; i <= NF; i++ )
510         setname[$2] = setname[$2]" "$i;
511   }
512   setingroup[$2,group] = 1;
513}
514/^@02 timelimit: / {
515   timelimit[nsolver] = $3;
516}
517/^@01 / {
518   if( onlygroup == 0 || setingroup[$2,onlygroup] )
519   {
520      solvername[nsolver] = $2;
521      nsolver++;
522   }
523   nprobs[nsolver] = 0;
524}
525// {
526   statuses["ok"];
527   statuses["timeout"];
528   statuses["unknown"];
529   statuses["abort"];
530   statuses["fail"];
531   statuses["readerror"];
532   statuses["better"];
533   statuses["solved"];
534   statuses["sollimit"];
535   statuses["gaplimit"];
536   statuses["memlimit"];
537   statuses["nodelimit"];
538
539   name[nsolver,nprobs[nsolver]] = $1;
540   validline = 0;
541   # check if this is a useable line
542   if( $10 in statuses ) # BLIS, SYMPHONY
543   {
544      # collect data (line with problem size and simplex iterations)
545      type[nsolver,nprobs[nsolver]] = "?";
546      conss[nsolver,nprobs[nsolver]] = $2;
547      vars[nsolver,nprobs[nsolver]] = $3;
548      dualbound[nsolver,nprobs[nsolver]] = max(min($4, +infinity), -infinity);
549      primalbound[nsolver,nprobs[nsolver]] = max(min($5, +infinity), -infinity);
550      gap[nsolver,nprobs[nsolver]] = $6;
551      iters[nsolver,nprobs[nsolver]] = $7;
552      nodes[nsolver,nprobs[nsolver]] = max($8,1);
553      time[nsolver,nprobs[nsolver]] = fracceil(max($9,mintime),0.1);
554      status[nsolver,nprobs[nsolver]] = $10;
555      printsoltimes = 0; # additional output is only available for SCIP-.res files
556      validline = 1;
557   }
558   if( $11 in statuses ) # from NLP-trace-files
559   {
560      # collect data (line with problem type, problem size and simplex iterations)
561      type[nsolver,nprobs[nsolver]] = $2;
562      conss[nsolver,nprobs[nsolver]] = $3;
563      vars[nsolver,nprobs[nsolver]] = $4;
564      dualbound[nsolver,nprobs[nsolver]] = max(min($5, +infinity), -infinity);
565      primalbound[nsolver,nprobs[nsolver]] = max(min($6, +infinity), -infinity);
566      gap[nsolver,nprobs[nsolver]] = $7;
567      iters[nsolver,nprobs[nsolver]] = $8;
568      nodes[nsolver,nprobs[nsolver]] = max($9,1);
569      time[nsolver,nprobs[nsolver]] = fracceil(max($10,mintime),0.1);
570      status[nsolver,nprobs[nsolver]] = $11;
571      printsoltimes = 0; # additional output is only available for SCIP-.res files
572      validline = 1;
573   }
574   if( $12 in statuses ) # GUROBI, CBC
575   {
576      # collect data (line with original and presolved problem size and simplex iterations)
577      type[nsolver,nprobs[nsolver]] = "?";
578      conss[nsolver,nprobs[nsolver]] = $4;
579      vars[nsolver,nprobs[nsolver]] = $5;
580      dualbound[nsolver,nprobs[nsolver]] = max(min($6, +infinity), -infinity);
581      primalbound[nsolver,nprobs[nsolver]] = max(min($7, +infinity), -infinity);
582      gap[nsolver,nprobs[nsolver]] = $8;
583      iters[nsolver,nprobs[nsolver]] = $9;
584      nodes[nsolver,nprobs[nsolver]] = max($10,1);
585      time[nsolver,nprobs[nsolver]] = fracceil(max($11,mintime),0.1);
586      status[nsolver,nprobs[nsolver]] = $12;
587      printsoltimes = 0; # additional output is only available for SCIP-.res files
588      validline = 1;
589   }
590   if( $13 in statuses ) # GLPK, CPLEX, SCIP without columns displaying times to first and best solution
591   {
592      # collect data (line with problem type, original and presolved problem size and simplex iterations)
593      type[nsolver,nprobs[nsolver]] = $2;
594      conss[nsolver,nprobs[nsolver]] = $5;
595      vars[nsolver,nprobs[nsolver]] = $6;
596      dualbound[nsolver,nprobs[nsolver]] = max(min($7, +infinity), -infinity);
597      primalbound[nsolver,nprobs[nsolver]] = max(min($8, +infinity), -infinity);
598      gap[nsolver,nprobs[nsolver]] = $9;
599      iters[nsolver,nprobs[nsolver]] = $10;
600      nodes[nsolver,nprobs[nsolver]] = max($11,1);
601      time[nsolver,nprobs[nsolver]] = fracceil(max($12,mintime),0.1);
602      status[nsolver,nprobs[nsolver]] = $13;
603      printsoltimes = 0; # additional output is only available for SCIP-.res files
604      validline = 1;
605   }
606   if( $15 in statuses ) # SCIP with solution times to first/last
607   {
608      # collect data (line with problem type, original and presolved problem size and simplex iterations)
609      type[nsolver,nprobs[nsolver]] = $2;
610      conss[nsolver,nprobs[nsolver]] = $5;
611      vars[nsolver,nprobs[nsolver]] = $6;
612      dualbound[nsolver,nprobs[nsolver]] = max(min($7, +infinity), -infinity);
613      primalbound[nsolver,nprobs[nsolver]] = max(min($8, +infinity), -infinity);
614      gap[nsolver,nprobs[nsolver]] = $9;
615      iters[nsolver,nprobs[nsolver]] = $10;
616      nodes[nsolver,nprobs[nsolver]] = max($11,1);
617      time[nsolver,nprobs[nsolver]] = fracceil(max($12,mintime),0.1);
618      timetofirst[nsolver,nprobs[nsolver]] = fracceil(max($13,mintime),0.1);
619      timetobest[nsolver, nprobs[nsolver]] = fracceil(max($14, mintime), 0.1);
620      status[nsolver,nprobs[nsolver]] = $15;
621      validline = 1;
622   }
623   if( $19 in statuses ) # SCIP with conflict analysis
624   {
625      # collect data (line with problem type, original and presolved problem size and simplex iterations)
626      type[nsolver,nprobs[nsolver]] = $2;
627      conss[nsolver,nprobs[nsolver]] = $5;
628      vars[nsolver,nprobs[nsolver]] = $6;
629      dualbound[nsolver,nprobs[nsolver]] = max(min($7, +infinity), -infinity);
630      primalbound[nsolver,nprobs[nsolver]] = max(min($8, +infinity), -infinity);
631      gap[nsolver,nprobs[nsolver]] = $9;
632      iters[nsolver,nprobs[nsolver]] = $10;
633      nodes[nsolver,nprobs[nsolver]] = max($11,1);
634      time[nsolver,nprobs[nsolver]] = fracceil(max($12,mintime),0.1);
635      confs[nsolver,nprobs[nsolver]] = $13+$14+$15+$16+$17;
636      conftime[nsolver,nprobs[nsolver]] = max($18, 0.1);
637      status[nsolver,nprobs[nsolver]] = $19;
638      printconfs = 1;
639      validline = 1;
640   }
641
642   if( validline )
643   {
644      # postprocessing of information
645      if( status[nsolver,nprobs[nsolver]] == "better" )
646         status[nsolver,nprobs[nsolver]] = "timeout";
647      if( status[nsolver,nprobs[nsolver]] == "sollimit" || status[nsolver,nprobs[nsolver]] == "gaplimit" || status[nsolver,nprobs[nsolver]] == "solved" )
648         status[nsolver,nprobs[nsolver]] = "ok";
649
650      if( status[nsolver,nprobs[nsolver]] == "timeout" || status[nsolver,nprobs[nsolver]] == "nodelimit" ||  status[nsolver,nprobs[nsolver]] == "memlimit")
651         hitlimit[nsolver,nprobs[nsolver]] = 1;
652      else
653         hitlimit[nsolver,nprobs[nsolver]] = 0;
654      probidx[$1,nsolver] = nprobs[nsolver];
655      probcnt[$1]++;
656      nprobs[nsolver]++;
657      if( probcnt[$1] == 1 )
658      {
659         problist[problistlen] = $1;
660         problistlen++;
661      }
662   }
663}
664END {
665   if( onlygroup > 0 && nsolver == 1 && solvername[1] == "SCIP:default" )
666   {
667      printf("only SCIP:default setting found\n");
668      exit 1;
669   }
670   if( nsolver == 0 )
671   {
672      printf("no instances found in log file\n");
673      exit 1;
674   }
675
676   # tex comparison file: either directly as 'texcmpfile' or as pair 'texcmpfiledir/texcmpfilename'
677   if( texcmpfile == "" && texcmpfiledir != "" && texcmpfilename != "" )
678      texcmpfile = texcmpfiledir "/" texcmpfilename;
679
680   # process exclude string
681   n = split(exclude, a, ",");
682   for( i = 1; i <= n; i++ )
683      excluded[a[i]] = 1;
684
685   # initialize means
686   for( s = 0; s < nsolver; ++s )
687   {
688      # cat: -1 - all within time limit, 0 - all, 1 - different path, 2 - equal path, 3 - all timeout
689      for( cat = -1; cat <= 3; cat++ )
690      {
691         nevalprobs[s,cat] = 0;
692         nprocessedprobs[s,cat] = 0;
693         timetotal[s,cat] = 0.0;
694         nodetotal[s,cat] = 0.0;
695         timegeom[s,cat] = 1.0;
696         nodegeom[s,cat] = 1.0;
697         timeshiftedgeom[s,cat] = timegeomshift;
698         timetofirstgeom[s,cat] = 1.0;
699         timetofirstshiftedgeom[s,cat] = timegeomshift;
700         timetobestgeom[s,cat] = 1.0;
701         timetobestshiftedgeom[s,cat] = timegeomshift;
702         confsgeom[s,cat] = 1.0;
703         conftimegeom[s,cat] = 1.0;
704         nodeshiftedgeom[s,cat] = nodegeomshift;
705         reftimetotal[s,cat] = 0.0;
706         refnodetotal[s,cat] = 0.0;
707         reftimegeom[s,cat] = 1.0;
708         reftimetofirstgeom[s,cat] = 1.0;
709         reftimetobestgeom[s,cat] = 1.0;
710         refnodegeom[s,cat] = 1.0;
711         reftimeshiftedgeom[s,cat] = timegeomshift;
712         refnodeshiftedgeom[s,cat] = nodegeomshift;
713         reftimetofirstshiftedgeom[s,cat] = timegeomshift;
714         reftimetobestshiftedgeom[s,cat] = timegeomshift;
715
716         wins[s,cat] = 0;
717         nsolved[s,cat] = 0;
718         ntimeouts[s,cat] = 0;
719         nfails[s,cat] = 0;
720         better[s,cat] = 0;
721         worse[s,cat] = 0;
722         betterobj[s,cat] = 0;
723         worseobj[s,cat] = 0;
724         feasibles[s,cat] = 0;
725         score[s,cat] = 1.0;
726      }
727   }
728   besttimegeom = 1.0;
729   besttimetofirstgeom = 1.0;
730   besttimetobestgeom = 1.0;
731   bestnodegeom = 1.0;
732   besttimeshiftedgeom = timegeomshift;
733   besttimetofirstshiftedgeom = timegeomshift;
734   besttimetobestshiftedgeom = timegeomshift;
735   bestnodeshiftedgeom = nodegeomshift;
736   bestnsolved = 0;
737   bestntimeouts = 0;
738   bestnfails = 0;
739   bestbetter = 0;
740   bestbetterobj = 0;
741   bestfeasibles = 0;
742
743   # calculate the order in which the columns should be printed: use alphabetical order but SCIP/SoPlex < other, default < non-default
744   for( s = 0; s < nsolver; ++s )
745   {
746      sname = solvername[s];
747      for( o = 0; o < s; ++o )
748      {
749         i = printorder[o];
750         iname = solvername[i];
751         if( nsetnames > 0 )
752         {
753            # use order given by =setname= entries
754            if( setorder[sname] < setorder[iname] )
755               break;
756         }
757         else
758         {
759            # SCIP with SoPlex before all others (note that the version numbers can be 7 or 8 characters long)
760            if( substr(sname, 1, 4) == "SCIP" && (substr(sname, 12, 3) == "spx" || substr(sname, 13, 3) == "spx") &&
761		substr(iname, 1, 4) == "SCIP" && substr(iname, 12, 3) != "spx" && substr(iname, 13, 3) != "spx" )
762               break;
763
764            # put default settings before other settings for same solver
765            if( substr(sname, 1, 5) == substr(iname, 1, 5) && match(sname, "default") != 0 && match(iname, "default") == 0 )
766               break;
767
768            # sort alphabetically if solver is the same and no default settings are involved
769            if( substr(sname, 1, 5) == substr(iname, 1, 5) && (match(sname, "default") == 0) == (match(iname, "default") == 0) && sname < iname )
770               break;
771         }
772      }
773      for( j = s-1; j >= o; --j )
774         printorder[j+1] = printorder[j];
775      printorder[o] = s;
776   }
777
778   # print headers
779   for( o = 0; o < nsolver; ++o )
780   {
781      s = printorder[o];
782      sname = solvername[s];
783      if( o == 0 )
784      {
785	 if( printsoltimes && printconfs == 0 )
786	 {
787	    if ( length(sname) <= 58 )
788	       printf(" %58s |", sname);
789	    else
790	       printf(" *%57s |", substr(sname, length(sname)-58));
791	 }
792	 else if( printsoltimes == 0 && printconfs )
793	 {
794	    if ( length(sname) <= 55 )
795	       printf(" %55s |", sname);
796	    else
797	       printf(" *%54s |", substr(sname, length(sname)-54));
798	 }
799	 else
800	 {
801	    if ( length(sname) <= 39 )
802	       printf(" %39s |", sname)
803	    else
804	       printf(" *%38s |", substr(sname, length(sname)-39));
805	 }
806      }
807      else
808      {
809	 if( short )
810	 {
811	    if( length(sname) <= 19 )
812	       printf("%19s |", sname);
813	    else
814	       printf("*%16s |", substr(sname, length(sname)-17));
815	 }
816	 else if( printsoltimes && printconfs == 0 )
817	 {
818	    if( length(sname) <= 47 )
819	       printf("%47s |", sname);
820	    else
821	       printf("*%46s |", substr(sname, length(sname)-47));
822	 }
823	 else if( printsoltimes == 0 && printconfs )
824	 {
825	    if( length(sname) <= 48 )
826	       printf("%48s |", sname);
827	    else
828	       printf("*%47s |", substr(sname, length(sname)-48));
829	 }
830	 else
831	 {
832	    if( length(sname) <= 33 )
833	       printf("%31s |", sname);
834	    else
835	       printf("*%30s |", substr(sname, length(sname)-31));
836	 }
837      }
838   }
839   printf("\n");
840   printhline(nsolver,short, printsoltimes, printconfs);
841   printf("  Name              |");
842   for( s = 0; s < nsolver; ++s )
843   {
844      if( s == 0 || short )
845         printf("F|   Nodes |   Time |");
846      else
847         printf("F|   Nodes |   Time | NodQ | TimQ |");
848      if( printsoltimes )
849      {
850	 if( s == 0 )
851	    printf(" ToFirst | ToLast |");
852	 else
853	    printf(" FirQ | LasQ |");
854      }
855      if( printconfs )
856      {
857	 if( s == 0 )
858	    printf(" NConfs | ConfT |");
859	 else
860	    printf(" NConfQ | ConTQ |");
861      }
862   }
863   printf(" bounds check\n");
864   printhline(nsolver,short, printsoltimes, printconfs);
865
866   # tex comparison headers
867   if( texcmpfile != "" )
868   {
869      printf("{\\sffamily\n") > texcmpfile;
870      printf("\\scriptsize\n") > texcmpfile;
871      printf("\\setlength{\\extrarowheight}{1pt}\n") > texcmpfile;
872      printf("\\setlength{\\tabcolsep}{2pt}\n") > texcmpfile;
873      printf("\\newcommand{\\g}{\\raisebox{0.25ex}{\\tiny $>$}}\n") > texcmpfile;
874      printf("\\newcommand{\\spc}{\\hspace{2em}}\n") > texcmpfile;
875
876      # add names of solvers
877      for ( o = 0; o < nsolver; ++o )
878      {
879         s = printorder[o];
880         solverextension = solverextension "i";
881         printf("\\newcommand{\\solvername%s}{%s}\n", solverextension, solvername[s]) > texcmpfile;
882      }
883
884      printf("\\begin{tabular*}{\\columnwidth}{@{\\extracolsep{\\fill}}l") > texcmpfile;
885      for( s = 0; s < nsolver; ++s )
886         printf("@{\\spc}rr") > texcmpfile;
887      printf("@{}}\n") > texcmpfile;
888      printf("\\toprule\n") > texcmpfile;
889      solverextension = "";
890      for( o = 0; o < nsolver; ++o )
891      {
892         s = printorder[o];
893         solverextension = solverextension "i";
894         printf("& \\multicolumn{2}{@{\\spc}c%s}{\\solvername%s} ", o < nsolver-1 ? "@{\\spc}" : "", solverextension) > texcmpfile;
895      }
896      printf("\\\\\n") > texcmpfile;
897      for( o = 0; o < nsolver; ++o )
898         printf("& Nodes & Time ") > texcmpfile;
899      printf("\\\\\n") > texcmpfile;
900      printf("\\midrule\n") > texcmpfile;
901   }
902
903   # display the problem results and calculate mean values
904   for( i = 0; i < problistlen; ++i )
905   {
906      p = problist[i];
907      if( length(p) > 18 )
908         shortp = substr(p, length(p)-17, 18);
909      else
910         shortp = p;
911
912      line = sprintf("%-18s", shortp);
913      fail = 0;
914      readerror = 0;
915      unprocessed = 0;
916      mindb = +infinity;
917      maxdb = -infinity;
918      minpb = +infinity;
919      maxpb = -infinity;
920      itercomp = -1;
921      nodecomp = -1;
922      timecomp = -1;
923      timetofirstcomp = -1;
924      timetobestcomp = -1;
925      besttime = +infinity;
926      besttimetofirst = +infinity;
927      besttimetobest = +infinity;
928      bestnodes = +infinity;
929      worsttime = -infinity;
930      worstnodes = -infinity;
931      worstiters = -infinity;
932      worsttimetofirst = -infinity;
933      worsttimetobest = -infinity;
934      minconfs = +infinity;
935      maxconfs = -infinity;
936      minconftime = +infinity;
937      maxconftime = -infinity;
938      nthisunprocessed = 0;
939      nthissolved = 0;
940      nthistimeouts = 0;
941      nthisfails = 0;
942      ismini = 0;
943      ismaxi = 0;
944      mark = " ";
945      marker = " ";
946      notimeout = 1;
947
948      # check for exclusion
949      if( excluded[p] )
950         unprocessed = 1;
951
952      # find best and worst run and check whether this instance should be counted in overall statistics
953      for( s = 0; s < nsolver; ++s )
954      {
955         pidx = probidx[p,s];
956         processed = (pidx != "");
957
958         # make sure, nodes and time are non-zero for geometric means
959         nodes[s,pidx] = max(nodes[s,pidx], 1);
960         time[s,pidx] = max(time[s,pidx], mintime);
961         fulltotaltime += time[s,pidx];
962
963         # If we got a timeout although the time limit has not been reached (e.g., due to a memory limit),
964         # we assume that the run would have been continued with the same nodes/sec.
965         # Set the time to the time limit and increase the nodes accordingly.
966         # if( status[s,pidx] == "timeout" && time[s,pidx] < timelimit[s] )
967         # {
968         #    nodes[s,pidx] *= timelimit[s]/time[s,pidx];
969         #    time[s,pidx] = timelimit[s];
970         # }
971
972         # if the solver exceeded the timelimit, set status accordingly
973         if( (status[s,pidx] == "ok" || status[s,pidx] == "unknown") && timelimit[s] > 0.0 && time[s,pidx] > timelimit[s] )
974         {
975            status[s,pidx] = "timeout";
976            time[s,pidx] = timelimit[s];
977         }
978
979         # check if all solvers processed the problem
980         if( !processed )
981         {
982            marker = "?";
983            unprocessed = 1;
984         }
985
986         # check if solver ran successfully (i.e., no abort nor fail)
987         if( processed && (status[s,pidx] == "ok" || status[s,pidx] == "unknown" || status[s,pidx] == "timeout" || status[s,pidx] == "nodelimit" || status[s,pidx] == "memlimit") )
988         {
989            besttime = min(besttime, time[s,pidx]);
990            bestnodes = min(bestnodes, nodes[s,pidx]);
991            besttimetofirst = min(besttimetofirst, timetofirst[s,pidx]);
992            besttimetobest = min(besttimetobest, timetobest[s,pidx]);
993            worsttime = max(worsttime, time[s,pidx]);
994            worstnodes = max(worstnodes, nodes[s,pidx]);
995            worstiters = max(worstiters, iters[s,pidx]);
996            worsttimetofirst = max(worsttimetofirst, timetofirst[s, pidx]);
997            worsttimetobest = max(worsttimetobest, timetobest[s, pidx]);
998            minconfs = min(minconfs, confs[s, pidx]);
999            maxconfs = max(maxconfs, confs[s, pidx]);
1000            minconftime = min(minconftime, conftime[s, pidx]);
1001            maxconftime = max(maxconftime, conftime[s, pidx]);
1002         }
1003      }
1004      worsttime = max(worsttime, mintime);
1005      worsttimetofirst = max(worsttimetofirst, mintime);
1006      worsttimetobest = max(worsttimetobest, mintime);
1007      worstnodes = max(worstnodes, 1);
1008      worstiters = max(worstiters, 0);
1009
1010      # check for each solver if it has same path as reference solver -> category
1011      for( o = 0; o < nsolver; ++o )
1012      {
1013         s = printorder[o];
1014         pidx = probidx[p,s];
1015         processed = (pidx != "");
1016
1017         if( !processed )
1018         {
1019            notimeout = 0;
1020            continue;
1021         }
1022         else
1023         {
1024            if ( status[s,pidx] == "timeout" )
1025            {
1026               # If memory limit was exceeded or we hit a hard time/memory limit,
1027               # replace time and nodes by worst time and worst nodes of all runs.
1028               # Note this also takes action if the time limits of the runs are
1029               # different: in this case we set the values to the worst case.
1030               # if ( time[s,pidx] < 0.99*worsttime || nodes[s,pidx] <= 1 )
1031               # {
1032               #    iters[s,pidx] = worstiters+s; # make sure this is not treated as equal path
1033               #    nodes[s,pidx] = worstnodes;
1034               #    time[s,pidx] = worsttime;
1035               # }
1036            }
1037         }
1038
1039         if( nodecomp == -1 )
1040         {
1041            itercomp = iters[s,pidx];
1042            nodecomp = nodes[s,pidx];
1043            timecomp = time[s,pidx];
1044            timeoutcomp = hitlimit[s,pidx];
1045            timetofirstcomp = max(mintime, timetofirst[s,pidx]);
1046            timetobestcomp = max(mintime, timetobest[s,pidx]);
1047            confsoffirst = confs[s,pidx];
1048            conftimeoffirst = conftime[s,pidx];
1049         }
1050         iseqpath = (iters[s,pidx] == itercomp && nodes[s,pidx] == nodecomp);
1051         hastimeout = timeoutcomp || hitlimit[s,pidx];
1052         notimeout = notimeout && !timeoutcomp &&  !hitlimit[s,pidx];
1053
1054         # which category?
1055         if( hastimeout )
1056            category[s] = 3;
1057         else if( iseqpath )
1058            category[s] = 2;
1059         else
1060            category[s] = 1;
1061      }
1062
1063      # evaluate instance for all solvers
1064      for( o = 0; o < nsolver; ++o )
1065      {
1066         s = printorder[o];
1067         pidx = probidx[p,s];
1068         processed = (pidx != "");
1069
1070         # arrays for applying McNemar tests
1071         solfound[s,pidx] = 0;
1072         optproven[s,pidx] = 0;
1073
1074         if( processed && name[s,pidx] != p )
1075            printf("Error: solver %d, probidx %d, <%s> != <%s>\n", solvername[s], pidx, name[s,pidx], p);
1076
1077         # check if solver ran successfully (i.e., no abort nor fail)
1078         if( processed )
1079         {
1080            if( status[s,pidx] == "ok" || status[s,pidx] == "unknown" )
1081            {
1082               marker = " ";
1083               if( !unprocessed )
1084               {
1085                  if ( notimeout )
1086                     nsolved[s,-1]++;
1087                  nsolved[s,0]++;
1088                  nsolved[s,category[s]]++;
1089                  nthissolved++;
1090                  # fill array for  McNemar test "optimality proven?"
1091                  optproven[s,pidx] = 1;
1092               }
1093            }
1094            else if( hitlimit[s,pidx] )
1095            {
1096               marker = ">";
1097               notimeout = 0;
1098               if( !unprocessed )
1099               {
1100                  ntimeouts[s,0]++;
1101                  ntimeouts[s,category[s]]++;
1102                  nthistimeouts++;
1103               }
1104            }
1105            else
1106            {
1107               marker = "!";
1108               fail = 1;
1109               notimeout = 0;
1110               if( status[s,pidx] == "readerror" )
1111                  readerror = 1;
1112               if( !unprocessed )
1113               {
1114                  nfails[s,0]++;
1115                  nfails[s,category[s]]++;
1116                  nthisfails++;
1117               }
1118            }
1119         }
1120
1121         if( primalbound[s,pidx] < infinity )
1122	 {
1123            feasmark = " ";
1124            # fill the array for McNemar test "solution found?"
1125            solfound[s,pidx] = 1;
1126         }
1127         else
1128            feasmark = "#";
1129
1130         if( processed && !fail )
1131         {
1132            mindb = min(mindb, dualbound[s,pidx]);
1133            maxdb = max(maxdb, dualbound[s,pidx]);
1134            minpb = min(minpb, primalbound[s,pidx]);
1135            maxpb = max(maxpb, primalbound[s,pidx]);
1136            ismini = ismini || (primalbound[s,pidx] > dualbound[s,pidx] + 1e-06);
1137            ismaxi = ismaxi || (primalbound[s,pidx] < dualbound[s,pidx] - 1e-06);
1138         }
1139
1140         # print statistics
1141         if( !processed )
1142            line = sprintf("%s           -        -", line);
1143         else
1144         {
1145            if( printgap && hitlimit[s,pidx] && gap[s,pidx] != "--" && gap[s,pidx] != "Large" )
1146	       line = sprintf("%s %s%10d %7.2f%%", line, feasmark, nodes[s,pidx], gap[s,pidx]);
1147            else
1148	       line = sprintf("%s %s%10d %s%7.1f", line, feasmark, nodes[s,pidx], marker, time[s,pidx]);
1149            if( printsoltimes && o == 0 )
1150               line = sprintf("%s  %8.1f %8.1f", line, timetofirst[s,pidx], timetobest[s, pidx] );
1151            if( printconfs && o == 0 )
1152               line = sprintf("%s %8d %7.1f", line, confs[s,pidx], conftime[s, pidx] );
1153         }
1154         if( o > 0 && !short )
1155         {
1156            if( !processed )
1157               line = sprintf("%s      -", line);
1158            else if( nodes[s,pidx]/nodecomp > 999.99 )
1159               line = sprintf("%s  Large", line);
1160            else
1161               line = sprintf("%s %6.2f", line, nodes[s,pidx]/nodecomp);
1162            if( !processed )
1163               line = sprintf("%s      -", line);
1164            else if( time[s,pidx]/timecomp > 999.99 )
1165               line = sprintf("%s  Large", line);
1166            else
1167               line = sprintf("%s %6.2f", line, time[s,pidx]/timecomp);
1168            if( processed &&
1169                (timeoutcomp != hitlimit[s,pidx] ||
1170                 nodes[s,pidx] > markworsenodes * nodecomp ||
1171                 nodes[s,pidx] < 1.0/markbetternodes * nodecomp ||
1172                 isfaster(time[s,pidx], timecomp, markbettertime) ||
1173                 isslower(time[s,pidx], timecomp, markworsetime)) )
1174               mark = "*";
1175         }
1176         if( o > 0 && printsoltimes )
1177         {
1178            if( !processed )
1179               line = sprintf("%s      -", line);
1180            else if( timetofirst[s,pidx]/timetofirstcomp > 999.99 )
1181               line = sprintf("%s  Large", line);
1182            else
1183               line = sprintf("%s %6.2f", line, timetofirst[s,pidx]/timetofirstcomp);
1184            if( !processed )
1185               line = sprintf("%s      -", line);
1186            else if( timetobest[s,pidx]/timetobestcomp > 999.99 )
1187               line = sprintf("%s  Large", line);
1188            else
1189               line = sprintf("%s %6.2f", line, timetobest[s,pidx]/timetobestcomp);
1190         }
1191         if( o > 0 && printconfs )
1192         {
1193            if( !processed )
1194               line = sprintf("%s        -", line);
1195            else if( confs[s,pidx] == confsoffirst )
1196               line = sprintf("%s %8.2f", line, 1.0);
1197            else if( confsoffirst == 0 )
1198               line = sprintf("%s        -", line);
1199            else if( confs[s,pidx]/confsoffirst > 999.99 )
1200               line = sprintf("%s  Large", line);
1201            else
1202               line = sprintf("%s %8.2f", line, confs[s,pidx]/confsoffirst);
1203
1204            if( !processed )
1205               line = sprintf("%s        -", line);
1206            else if( conftime[s,pidx] == conftimeoffirst )
1207               line = sprintf("%s %7.2f", line, 1.0);
1208            else if( conftimeoffirst == 0 )
1209               line = sprintf("%s        -", line);
1210            else if( conftime[s,pidx]/conftimeoffirst> 999.99 )
1211               line = sprintf("%s  Large", line);
1212            else
1213               line = sprintf("%s %7.2f", line, conftime[s,pidx]/conftimeoffirst);
1214         }
1215      }
1216
1217      # update the best status information
1218      if( nthissolved > 0 )
1219	 bestnsolved++;
1220      else if( nthistimeouts > 0 )
1221	 bestntimeouts++;
1222      else if( nthisfails == nsolver - nthisunprocessed )
1223	 bestnfails++;
1224
1225      # check for inconsistency in the primal and dual bounds
1226      if( readerror )
1227      {
1228         line = sprintf("%s  readerror", line);
1229         mark = " ";
1230      }
1231      else if( fail )
1232      {
1233         line = sprintf("%s  fail", line);
1234         mark = " ";
1235      }
1236      else if( consistency &&
1237	       ((ismini && ismaxi) ||
1238		(ismini && maxdb - minpb > 1e-5 * max(max(abs(maxdb), abs(minpb)), 1.0)) ||
1239		(ismaxi && maxpb - mindb > 1e-5 * max(max(abs(maxpb), abs(mindb)), 1.0)) ||
1240		(!ismini && !ismaxi && abs(maxpb - minpb) > 1e-5 * max(abs(maxpb), 1.0))) )
1241      {
1242         line = sprintf("%s  inconsistent (maxdb=%g, minpb=%g)", line, maxdb, minpb);
1243         fail = 1;
1244         mark = " ";
1245      }
1246      else if( excluded[p] )
1247      {
1248         line = sprintf("%s  excluded", line);
1249         mark = " ";
1250      }
1251      else if( unprocessed )
1252      {
1253         line = sprintf("%s  unprocessed", line);
1254         mark = " ";
1255      }
1256      else
1257         line = sprintf("%s  ok", line);
1258
1259      # calculate number of instances for which feasible solution has been found
1260      hasfeasible = 0;
1261      if( !unprocessed )
1262      {
1263         for( s = 0; s < nsolver; ++s )
1264         {
1265            if ( notimeout )
1266               nprocessedprobs[s,-1]++;
1267            nprocessedprobs[s,0]++;
1268            nprocessedprobs[s,category[s]]++;
1269            pidx = probidx[p,s];
1270            if( primalbound[s,pidx] < infinity )
1271            {
1272               if ( notimeout )
1273                  feasibles[s,-1]++;
1274               feasibles[s,0]++;
1275               feasibles[s,category[s]]++;
1276               hasfeasible = 1;
1277            }
1278         }
1279         if( hasfeasible )
1280	    bestfeasibles++;
1281      }
1282
1283      if( (!onlymarked || mark == "*") && (!onlyprocessed || !unprocessed) &&
1284          (!onlyfeasible || hasfeasible) && (!onlyinfeasible || !hasfeasible) &&
1285          (!onlyfail || fail) )
1286      {
1287         printf("%s %s\n", mark, line);
1288
1289         # tex comparison file
1290         if( texcmpfile != "" )
1291         {
1292            printf("%s ", texstring(p)) > texcmpfile;
1293            ref = printorder[0];
1294            refnodes = nodes[ref,probidx[p,ref]];
1295            reftime = time[ref,probidx[p,ref]];
1296            refstatus = status[ref,probidx[p,ref]];
1297            refhitlimit = hitlimit[ref,probidx[p,ref]];
1298            for( o = 0; o < nsolver; ++o )
1299            {
1300               s = printorder[o];
1301               pidx = probidx[p,s];
1302
1303               if( hitlimit[s,pidx] )
1304                  timeoutmarker = "\\g";
1305               else
1306                  timeoutmarker = "  ";
1307
1308               if( nodes[s,pidx] <= 0.5*refnodes && !hitlimit[s,pidx] )
1309                  nodecolor = "red";
1310               else if( nodes[s,pidx] >= 2.0*refnodes && !refhitlimit )
1311                  nodecolor = "blue";
1312               else
1313                  nodecolor = "black";
1314
1315               if( (time[s,pidx] <= 0.5*reftime && !hitlimit[s,pidx]) || (!hitlimit[s,pidx] && refhitlimit) )
1316                  timecolor = "red";
1317               else if( (time[s,pidx] >= 2.0*reftime && !refhitlimit) || (hitlimit[s,pidx] && !refhitlimit) )
1318                  timecolor = "blue";
1319               else
1320                  timecolor = "black";
1321
1322               if( status[s,pidx] == "ok" || status[s,pidx] == "unknown" || status[s,pidx] == "timeout" || status[s,pidx] == "memlimit" || status[s,pidx] == "nodelimit" )
1323                  printf("&\\textcolor{%s}{%s %8s} &\\textcolor{%s}{%s %8.1f} ",
1324			 nodecolor, timeoutmarker, texint(nodes[s,pidx]), timecolor, timeoutmarker, time[s,pidx]) > texcmpfile;
1325               else
1326                  printf("&        --- &        --- ") > texcmpfile;
1327            }
1328            printf("\\\\\n") > texcmpfile;
1329         }
1330      }
1331
1332      # calculate totals and means for instances where no solver failed
1333      if( !fail && !unprocessed && (!onlyfeasible || hasfeasible) && (!onlyinfeasible || !hasfeasible) )
1334      {
1335         reftime = time[printorder[0],probidx[p,printorder[0]]];
1336         refhitlimit = hitlimit[printorder[0],probidx[p,printorder[0]]];
1337         refnodes = nodes[printorder[0],probidx[p,printorder[0]]];
1338         refobj = primalbound[printorder[0],probidx[p,printorder[0]]];
1339         reftimetofirst = timetofirst[printorder[0],probidx[p,printorder[0]]];
1340         reftimetobest = timetobest[printorder[0],probidx[p,printorder[0]]];
1341         hasbetter = 0;
1342         hasbetterobj = 0;
1343         for( s = 0; s < nsolver; ++s )
1344         {
1345            pidx = probidx[p,s];
1346            for( cat = 0; cat <= 3; cat = 3*cat + category[s] )
1347            {
1348               nevalprobs[s,cat]++;
1349               nep = nevalprobs[s,cat];
1350               timetotal[s,cat] += time[s,pidx];
1351               timetofirsttotal[s,cat] += timetofirst[s,pidx];
1352               timetobesttotal[s, cat] += timetobest[s, pidx];
1353               nodetotal[s,cat] += nodes[s,pidx];
1354               timegeom[s,cat] = timegeom[s,cat]^((nep-1)/nep) * time[s,pidx]^(1.0/nep);
1355               timetofirstgeom[s,cat] = timetofirstgeom[s,cat]^((nep-1)/nep) * max(timetofirst[s,pidx], mintime)^(1.0/nep);
1356               timetobestgeom[s,cat] = timetobestgeom[s,cat]^((nep-1)/nep) * max(timetobest[s,pidx])^(1.0/nep);
1357               confsgeom[s,cat] = confsgeom[s,cat]^((nep-1)/nep) * max(confs[s,pidx], 1.0)^(1.0/nep);
1358               conftimegeom[s,cat] = conftimegeom[s,cat]^((nep-1)/nep) * max(conftime[s,pidx],1.0)^(1.0/nep);
1359               nodegeom[s,cat] = nodegeom[s,cat]^((nep-1)/nep) * nodes[s,pidx]^(1.0/nep);
1360               timeshiftedgeom[s,cat] = timeshiftedgeom[s,cat]^((nep-1)/nep) * (time[s,pidx]+timegeomshift)^(1.0/nep);
1361               timetofirstshiftedgeom[s,cat] = timetofirstshiftedgeom[s,cat]^((nep-1)/nep) * max(timetofirst[s,pidx]+timegeomshift, 1.0)^(1.0/nep);
1362               timetobestshiftedgeom[s,cat] = timetobestshiftedgeom[s,cat]^((nep-1)/nep) * max(timetobest[s,pidx]+timegeomshift, 1.0)^(1.0/nep);
1363               confsshiftedgeomean[s,cat] = confsshiftedgeomean[s,cat]^((nep-1)/nep) * max(confs[s,pidx]+nodegeomshift, 1.0)^(1.0/nep);
1364               conftimeshiftedgeomean[s,cat] = conftimeshiftedgeomean[s,cat]^((nep-1)/nep) * max(conftime[s,pidx]+timegeomshift,1.0)^(1.0/nep);
1365               nodeshiftedgeom[s,cat] = nodeshiftedgeom[s,cat]^((nep-1)/nep) * (nodes[s,pidx]+nodegeomshift)^(1.0/nep);
1366               reftimetotal[s,cat] += reftime;
1367               reftimetofirsttotal[s,cat] += reftimetofirst;
1368               reftimetobesttotal[s,cat] += reftimetobest;
1369               refnodetotal[s,cat] += refnodes;
1370               reftimegeom[s,cat] = reftimegeom[s,cat]^((nep-1)/nep) * reftime^(1.0/nep);
1371               reftimetofirstgeom[s,cat] = reftimetofirstgeom[s,cat]^((nep-1)/nep) * reftimetofirst^(1.0/nep);
1372               reftimetobestgeom[s,cat] = reftimetobestgeom[s,cat]^((nep-1)/nep) * reftimetobest^(1.0/nep);
1373               refnodegeom[s,cat] = refnodegeom[s,cat]^((nep-1)/nep) * refnodes^(1.0/nep);
1374               reftimeshiftedgeom[s,cat] = reftimeshiftedgeom[s,cat]^((nep-1)/nep) * (reftime+timegeomshift)^(1.0/nep);
1375               reftimetofirstshiftedgeom[s,cat] = reftimetofirstshiftedgeom[s,cat]^((nep-1)/nep) * (reftimetofirst+timegeomshift)^(1.0/nep);
1376               reftimetobestshiftedgeom[s,cat] = reftimetobestshiftedgeom[s,cat]^((nep-1)/nep) * (reftimetobest+timegeomshift)^(1.0/nep);
1377               refnodeshiftedgeom[s,cat] = refnodeshiftedgeom[s,cat]^((nep-1)/nep) * (refnodes+nodegeomshift)^(1.0/nep);
1378               if( time[s,pidx] <= wintolerance*besttime )
1379                  wins[s,cat]++;
1380               if( !hitlimit[s,pidx] && (isfaster(time[s,pidx], reftime, wintolerance) || refhitlimit))
1381               {
1382                  better[s,cat]++;
1383                  hasbetter = 1;
1384               }
1385               else if( !refhitlimit && (isslower(time[s,pidx], reftime, wintolerance) || (hitlimit[s,pidx])))
1386                  worse[s,cat]++;
1387               pb = primalbound[s,pidx];
1388               if( (ismini && pb - refobj < -0.01 * max(max(abs(refobj), abs(pb)), 1.0)) ||
1389                   (ismaxi && pb - refobj > +0.01 * max(max(abs(refobj), abs(pb)), 1.0)) ) {
1390                  betterobj[s,cat]++;
1391                  hasbetterobj = 1;
1392               }
1393               else if( (ismini && pb - refobj > +0.01 * max(max(abs(refobj), abs(pb)), 1.0)) ||
1394                        (ismaxi && pb - refobj < -0.01 * max(max(abs(refobj), abs(pb)), 1.0)) )
1395                  worseobj[s,cat]++;
1396               thisscore = reftime/time[s,pidx];
1397               thisscore = max(thisscore, 1/maxscore);
1398               thisscore = min(thisscore, maxscore);
1399               score[s,cat] = score[s,cat]^((nep-1)/nep) * thisscore^(1.0/nep);
1400            }
1401         }
1402         s = printorder[0];
1403         besttimegeom = besttimegeom^((nevalprobs[s,0]-1)/nevalprobs[s,0]) * besttime^(1.0/nevalprobs[s,0]);
1404         bestnodegeom = bestnodegeom^((nevalprobs[s,0]-1)/nevalprobs[s,0]) * bestnodes^(1.0/nevalprobs[s,0]);
1405         besttimeshiftedgeom = besttimeshiftedgeom^((nevalprobs[s,0]-1)/nevalprobs[s,0]) * (besttime+timegeomshift)^(1.0/nevalprobs[s,0]);
1406         bestnodeshiftedgeom = bestnodeshiftedgeom^((nevalprobs[s,0]-1)/nevalprobs[s,0]) * (bestnodes+nodegeomshift)^(1.0/nevalprobs[s,0]);
1407         if( hasbetter )
1408            bestbetter++;
1409         if( hasbetterobj )
1410	    bestbetterobj++;
1411
1412         # once again for the case in which all instances have been solved to optimality
1413         if ( notimeout )
1414         {
1415            for( s = 0; s < nsolver; ++s )
1416            {
1417               pidx = probidx[p,s];
1418               cat = -1;
1419               nevalprobs[s,cat]++;
1420               nep = nevalprobs[s,cat];
1421               timetotal[s,cat] += time[s,pidx];
1422               nodetotal[s,cat] += nodes[s,pidx];
1423               timegeom[s,cat] = timegeom[s,cat]^((nep-1)/nep) * time[s,pidx]^(1.0/nep);
1424               nodegeom[s,cat] = nodegeom[s,cat]^((nep-1)/nep) * nodes[s,pidx]^(1.0/nep);
1425               timeshiftedgeom[s,cat] = timeshiftedgeom[s,cat]^((nep-1)/nep) * (time[s,pidx]+timegeomshift)^(1.0/nep);
1426               nodeshiftedgeom[s,cat] = nodeshiftedgeom[s,cat]^((nep-1)/nep) * (nodes[s,pidx]+nodegeomshift)^(1.0/nep);
1427               reftimetotal[s,cat] += reftime;
1428               refnodetotal[s,cat] += refnodes;
1429               reftimegeom[s,cat] = reftimegeom[s,cat]^((nep-1)/nep) * reftime^(1.0/nep);
1430               refnodegeom[s,cat] = refnodegeom[s,cat]^((nep-1)/nep) * refnodes^(1.0/nep);
1431               reftimeshiftedgeom[s,cat] = reftimeshiftedgeom[s,cat]^((nep-1)/nep) * (reftime+timegeomshift)^(1.0/nep);
1432               refnodeshiftedgeom[s,cat] = refnodeshiftedgeom[s,cat]^((nep-1)/nep) * (refnodes+nodegeomshift)^(1.0/nep);
1433               if( time[s,pidx] <= wintolerance*besttime )
1434                  wins[s,cat]++;
1435               if( isfaster(time[s,pidx], reftime, wintolerance) )
1436                  better[s,cat]++;
1437               else if( isslower(time[s,pidx], reftime, wintolerance) )
1438                  worse[s,cat]++;
1439               pb = primalbound[s,pidx];
1440               if( (ismini && pb - refobj < -0.01 * max(max(abs(refobj), abs(pb)), 1.0)) ||
1441                   (ismaxi && pb - refobj > +0.01 * max(max(abs(refobj), abs(pb)), 1.0)) ) {
1442                  betterobj[s,cat]++;
1443               }
1444               else if( (ismini && pb - refobj > +0.01 * max(max(abs(refobj), abs(pb)), 1.0)) ||
1445                        (ismaxi && pb - refobj < -0.01 * max(max(abs(refobj), abs(pb)), 1.0)) )
1446                  worseobj[s,cat]++;
1447               thisscore = reftime/time[s,pidx];
1448               thisscore = max(thisscore, 1/maxscore);
1449               thisscore = min(thisscore, maxscore);
1450               score[s,cat] = score[s,cat]^((nep-1)/nep) * thisscore^(1.0/nep);
1451            }
1452         }
1453      }
1454   }
1455   printhline(nsolver,short, printsoltimes, printconfs);
1456
1457   # make sure total time and nodes is not zero
1458   for( s = 0; s < nsolver; ++s )
1459   {
1460      for( cat = -1; cat <= 3; cat++ )
1461      {
1462         nodetotal[s,cat] = max(nodetotal[s,cat], 1);
1463         refnodetotal[s,cat] = max(refnodetotal[s,cat], 1);
1464         timetotal[s,cat] = max(timetotal[s,cat], mintime);
1465         reftimetotal[s,cat] = max(reftimetotal[s,cat], mintime);
1466      }
1467   }
1468
1469   # print solvers' overall statistics
1470   probnumstr = "("nevalprobs[printorder[0],0]")";
1471   printf("%-14s %5s", "total", probnumstr);
1472   for( o = 0; o < nsolver; ++o )
1473   {
1474      s = printorder[o];
1475      if( o == 0 || short )
1476      {
1477	 printf(" %11d %8d", nodetotal[s,0], timetotal[s,0]);
1478	 if( o == 0 && printsoltimes )
1479	    printf(" %9d %8d" , timetofirsttotal[s,0], timetobesttotal[s,0]);
1480	 if( o == 0 && printconfs )
1481	    printf(" %8d %7d" , 0, 0);
1482	 #printf(" %9d %8d" , timetofirsttotal[s,0], timetobesttotal[s,0]);
1483      }
1484      else
1485      {
1486	 printf(" %11d %8d              ", nodetotal[s,0], timetotal[s,0]);
1487	 if( printsoltimes )
1488	 {
1489	    referencesolvername = printorder[0];
1490	    comptimetofirst = timetofirsttotal[s,0]/(max(timetofirsttotal[referencesolvername,0], 1));
1491	    comptimetobest = timetobesttotal[s,0]/(max(timetobesttotal[referencesolvername,0], 1));
1492	    printf("%7.2f %6.2f", comptimetofirst, comptimetobest);
1493	 }
1494	 if( printconfs )
1495	 {
1496	    printf("%17s", "");
1497	 }
1498      }
1499   }
1500   printf("\n");
1501   printf("%-20s", "geom. mean");
1502   nodegeomcomp = -1;
1503   timegeomcomp = -1;
1504   nodetotalcomp = -1;
1505   timetotalcomp = -1;
1506   timetofirstcomp = -1;
1507   timetobestcomp = -1;
1508   timetofirstgeomcomp = -1;
1509   timetobestgeomcomp = -1;
1510   confsgeomcomp = -1;
1511   conftimegeomcomp = -1;
1512
1513   for( o = 0; o < nsolver; ++o )
1514   {
1515      s = printorder[o];
1516      if( o == 0 || short )
1517      {
1518	 printf(" %11d %8.1f", nodegeom[s,0], timegeom[s,0]);
1519	 if( o == 0 && printsoltimes )
1520	    printf(" %9.1f %8.1f", timetofirstgeom[s,0], timetobestgeom[s,0]);
1521	 if( o == 0 && printconfs )
1522	    printf(" %8.1f %7.1f", confsgeom[s,0], conftimegeom[s,0]);
1523
1524         if( nodegeomcomp < 0 )
1525            nodegeomcomp = nodegeom[s,0];
1526         if( timegeomcomp < 0 )
1527            timegeomcomp = timegeom[s,0];
1528         if( nodetotalcomp < 0 )
1529            nodetotalcomp = nodetotal[s,0];
1530         if( timetotalcomp < 0 )
1531            timetotalcomp = timetotal[s,0];
1532         if( timetofirstcomp < 0 )
1533	    timetofirstcomp = timetofirsttotal[s,0];
1534         if( timetobestcomp < 0 )
1535	    timetobestcomp = timetobesttotal[s,0];
1536         if( timetofirstgeomcomp < 0 )
1537	    timetofirstgeomcomp = timetofirstgeom[s,0];
1538         if( timetobestgeomcomp < 0 )
1539	    timetobestgeomcomp = timetobestgeom[s,0];
1540         if( confsgeomcomp < 0 )
1541	    confsgeomcomp = max(1.0,confsgeom[s,0]);
1542         if( conftimegeomcomp < 0 )
1543	    conftimegeomcomp = max(1.0,conftimegeom[s,0]);
1544      }
1545      else
1546      {
1547	 printf(" %11d %8.1f %6.2f %6.2f", nodegeom[s,0], timegeom[s,0], nodegeom[s,0]/nodegeomcomp, timegeom[s,0]/timegeomcomp);
1548
1549	 if( printsoltimes )
1550	    printf(" %6.2f %6.2f", timetofirstgeom[s,0]/timetofirstgeomcomp, timetobestgeom[s,0]/timetobestgeomcomp);
1551	 if( printconfs )
1552	    printf(" %8.2f %7.2f", confsgeom[s,0]/confsgeomcomp, conftimegeom[s,0]/conftimegeomcomp);
1553      }
1554   }
1555   printf("\n");
1556   printf("%-20s", "shifted geom.");
1557   nodeshiftedgeomcomp = -1;
1558   timeshiftedgeomcomp = -1;
1559   timetofirstshiftedgeomcomp = -1;
1560   timetobestshiftedgeomcomp = -1;
1561   confsshiftedgeomeancomp = -1;
1562   conftimeshiftedgeomeancomp = -1;
1563   for( o = 0; o < nsolver; ++o )
1564   {
1565      s = printorder[o];
1566      for( cat = -1; cat <= 3; cat++ )
1567      {
1568         nodeshiftedgeom[s,cat] -= nodegeomshift;
1569         timeshiftedgeom[s,cat] -= timegeomshift;
1570         timetofirstshiftedgeom[s,cat] -= timegeomshift;
1571         timetobestshiftedgeom[s,cat] -= timegeomshift;
1572         confsshiftedgeomean[s,cat] -= nodegeomshift
1573         conftimeshiftedgeomean[s,cat] -= timegeomshift;
1574         nodeshiftedgeom[s,cat] = max(nodeshiftedgeom[s,cat], 1);
1575         timeshiftedgeom[s,cat] = max(timeshiftedgeom[s,cat], mintime);
1576         timetofirstshiftedgeom[s,cat] = max(timetofirstshiftedgeom[s,cat], mintime);
1577         timetobestshiftedgeom[s,cat] = max(timetobestshiftedgeom[s,cat], mintime);
1578         confsshiftedgeomean[s,cat] = max(confsshiftedgeomean[s,cat], 1.0);
1579         conftimeshiftedgeomean[s,cat] = max(conftimeshiftedgeomean[s,cat], 1.0);
1580         refnodeshiftedgeom[s,cat] -= nodegeomshift;
1581         reftimeshiftedgeom[s,cat] -= timegeomshift;
1582         refnodeshiftedgeom[s,cat] = max(refnodeshiftedgeom[s,cat], mintime);
1583         reftimeshiftedgeom[s,cat] = max(reftimeshiftedgeom[s,cat], mintime);
1584      }
1585      if( o == 0 || short )
1586      {
1587	 printf(" %11d %8.1f", nodeshiftedgeom[s,0], timeshiftedgeom[s,0]);
1588
1589	 if( o == 0 && printsoltimes )
1590	    printf(" %9.1f %8.1f", timetofirstshiftedgeom[s,0], timetobestshiftedgeom[s,0]);
1591	 if( o == 0 && printconfs )
1592	    printf(" %8.1f %7.1f", confsshiftedgeomean[s,0], conftimeshiftedgeomean[s,0]);
1593
1594         if( nodeshiftedgeomcomp < 0 )
1595            nodeshiftedgeomcomp = nodeshiftedgeom[s,0];
1596         if( timeshiftedgeomcomp < 0 )
1597            timeshiftedgeomcomp = timeshiftedgeom[s,0];
1598         if( timetofirstshiftedgeomcomp < 0 )
1599	    timetofirstshiftedgeomcomp = timetofirstshiftedgeom[s,0];
1600         if( timetobestshiftedgeomcomp < 0 )
1601	    timetobestshiftedgeomcomp = timetobestshiftedgeom[s,0];
1602         if( confsshiftedgeomeancomp < 0 )
1603	    confsshiftedgeomeancomp = confsshiftedgeomean[s,0];
1604         if( conftimeshiftedgeomeancomp < 0 )
1605	    conftimeshiftedgeomeancomp = conftimeshiftedgeomean[s,0];
1606      }
1607      else
1608      {
1609         printf(" %11d %8.1f %6.2f %6.2f", nodeshiftedgeom[s,0], timeshiftedgeom[s,0],
1610                nodeshiftedgeom[s,0]/nodeshiftedgeomcomp, timeshiftedgeom[s,0]/timeshiftedgeomcomp);
1611
1612         if( printsoltimes )
1613            printf(" %6.2f %6.2f", timetofirstshiftedgeom[s,0]/timetofirstshiftedgeomcomp, timetobestshiftedgeom[s,0]/timetobestshiftedgeomcomp);
1614         if( printconfs )
1615            printf(" %8.2f %7.2f", confsshiftedgeomean[s,0]/confsshiftedgeomeancomp, conftimeshiftedgeomean[s,0]/conftimeshiftedgeomeancomp);
1616      }
1617   }
1618   bestnodeshiftedgeom -= nodegeomshift;
1619   besttimeshiftedgeom -= timegeomshift;
1620   bestnodeshiftedgeom = max(bestnodeshiftedgeom, 1.0);
1621   besttimeshiftedgeom = max(besttimeshiftedgeom, 1.0);
1622
1623   printf("\n");
1624   printhline(nsolver,short, printsoltimes, printconfs);
1625
1626   # compute and print result for McNemar test to "solution found?"  w.r.t. reference setting
1627   printf("%-20s ","McNemar (feas)");
1628   printf("%-18s  ","               ");
1629
1630   if( printconfs )
1631      printf("%17s", "");
1632
1633   for( o = 1; o < nsolver; ++o )
1634   {
1635      # copy two-indexed arrays to one-indexed arrays
1636      for( i = 0; i < problistlen; ++i )
1637      {
1638         s = printorder[o];
1639         ref_array[i] = solfound[printorder[0],i];
1640         solver_array[i] = solfound[s,i];
1641      }
1642
1643      # compute chi-squared value and convert to p-value
1644      chi_squared = mcnemar(ref_array, solver_array, problistlen);
1645      printf("   x2 %7.4f",chi_squared);
1646      chi_to_p(chi_squared);
1647
1648      if( printconfs )
1649	 printf("%18s", "");
1650   }
1651   printf("\n");
1652
1653   # compute and print result for McNemar test to "optimality proven?" w.r.t. reference setting
1654   printf("%-20s ","McNemar (opt)");
1655   printf("%-18s  ","               ");
1656
1657   if( printconfs )
1658      printf("%17s", "");
1659
1660   for( o = 1; o < nsolver; ++o )
1661   {
1662      # copy two-indexed arrays to one-indexed arrays
1663      for( i = 0; i < problistlen; ++i )
1664      {
1665         s = printorder[o];
1666         ref_array[i] = optproven[printorder[0],i];
1667         solver_array[i] = optproven[s,i];
1668      }
1669
1670      # compute chi-squared value and convert to p-value
1671      chi_squared = mcnemar(ref_array, solver_array, problistlen);
1672      printf("   x2 %7.4f",chi_squared);
1673      chi_to_p(chi_squared);
1674
1675      if( printconfs )
1676	 printf("%18s", "");
1677   }
1678   printf("\n");
1679
1680   # compute and print result for Wilcoxon signed rank test for time to optimality w.r.t. reference setting
1681   printf("%-20s ","Wilcoxon (time)");
1682   printf("%-18s  ","               ");
1683
1684   if( printconfs )
1685      printf("%17s", "");
1686
1687   for( o = 1; o < nsolver; ++o )
1688   {
1689      s = printorder[o];
1690
1691      parse_time(ref_array,solver_array,time,o,printorder,probidx,problistlen);
1692      n = filter(ref_array, solver_array, problistlen, 0.01, 0.01);
1693      factorize(ref_array, solver_array, n, timelimit[s])
1694
1695      z = wilcoxon(ref_array, solver_array, n, timelimit[s]);
1696      printf("   z %8.4f",z);
1697      z_to_p(z);
1698
1699      if( printconfs )
1700	 printf("%18s", "");
1701   }
1702   printf("\n");
1703
1704
1705   # compute and print result for Wilcoxon signed rank test for time to first solution w.r.t. reference setting
1706   if( printsoltimes )
1707   {
1708      printf("%-20s ","Wilcoxon (first)");
1709      printf("%-18s  ","               ");
1710
1711      if( printconfs )
1712	 printf("%17s", "");
1713
1714      for( o = 1; o < nsolver; ++o )
1715      {
1716         s = printorder[o];
1717
1718         parse_time(ref_array,solver_array,timetofirst,o,printorder,probidx,problistlen);
1719         n = filter(ref_array, solver_array, problistlen, 0.01, 0.01);
1720         factorize(ref_array, solver_array, n, timelimit[s])
1721
1722	 z = wilcoxon(ref_array, solver_array, n, timelimit[s]);
1723	 printf("   z %8.4f",z);
1724	 z_to_p(z);
1725
1726         if( printconfs )
1727            printf("%18s", "");
1728      }
1729      printf("\n");
1730   }
1731
1732   # compute and print result for Wilcoxon signed rank test for number of nodes w.r.t. reference setting
1733   printf("%-20s ","Wilcoxon (nodes)");
1734   printf("%-18s  ","               ");
1735
1736   if( printconfs )
1737      printf("%17s", "");
1738
1739   for( o = 1; o < nsolver; ++o )
1740   {
1741      parse_nodes(ref_array,solver_array,nodes,o,probidx,problistlen,status,infinity);
1742      n = filter(ref_array, solver_array, problistlen, 0.01, 0.01);
1743      factorize(ref_array, solver_array, n, infinity)
1744
1745      z = wilcoxon(ref_array, solver_array, n, infinity);
1746      printf("   z %8.4f",z);
1747      z_to_p(z);
1748
1749      if( printconfs )
1750	 printf("%18s", "");
1751   }
1752   printf("\n");
1753
1754   if( printconfs )
1755   {
1756      printf("%-20s ","Wilcoxon (confs)");
1757      printf("%-35s  ","               ");
1758      for( o = 1; o < nsolver; ++o )
1759      {
1760         s = printorder[o];
1761
1762         parse_time(ref_array,solver_array,confs,o,printorder,probidx,problistlen);
1763         n = filter(ref_array, solver_array, problistlen, 0.01, 0.01);
1764         factorize(ref_array, solver_array, n, timelimit[s])
1765
1766         z = wilcoxon(ref_array, solver_array, n, timelimit[s]);
1767         printf("   z %8.4f", z );
1768         z_to_p(z);
1769
1770         if( printconfs )
1771	    printf("%18s", "");
1772      }
1773      printf("\n");
1774   }
1775
1776   #since the rows of the quotients are not printed, print the quotients of the geometric means
1777   if( short )
1778   {
1779      printf("quot. geom. mean                         ");
1780      for( o = 0; o < nsolver; ++o )
1781      {
1782         if( o > 0 )
1783         {
1784            s = printorder[o];
1785            printf("      %6.2f   %6.2f",nodegeom[s,0]/nodegeomcomp, timegeom[s,0]/timegeomcomp);
1786         }
1787      }
1788      printf("\n");
1789      printf("quot. sh. geom. mean                     ");
1790      for( o = 0; o < nsolver; ++o )
1791      {
1792         if( o > 0 )
1793         {
1794            s = printorder[o];
1795            printf("      %6.2f   %6.2f",nodeshiftedgeom[s,0]/nodeshiftedgeomcomp, timeshiftedgeom[s,0]/timeshiftedgeomcomp);
1796         }
1797      }
1798      printf("\n");
1799      printf("percent not solved                 ");
1800      for( o = 0; o < nsolver; ++o )
1801      {
1802         s = printorder[o];
1803         printf("%6.2f",100-100*nsolved[s,0]/nprocessedprobs[s,0]);
1804         printf("               ");
1805      }
1806      printf("\n");
1807   }
1808   printhline(nsolver,short, printsoltimes, printconfs);
1809
1810   # tex comparison footer
1811   if( texcmpfile != "" )
1812   {
1813      # all problems
1814      printf("\\midrule\n") > texcmpfile;
1815      printf("geom. mean     ") > texcmpfile;
1816      for( o = 0; o < nsolver; ++o )
1817      {
1818         s = printorder[o];
1819         printf("& %8s & %8.1f", texint(nodegeom[s,0]), timegeom[s,0]) > texcmpfile;
1820      }
1821      printf("\\\\\n") > texcmpfile;
1822      printf("sh. geom. mean ") > texcmpfile;
1823      for( o = 0; o < nsolver; ++o )
1824      {
1825         s = printorder[o];
1826         printf("& %8s & %8.1f", texint(nodeshiftedgeom[s,0]), timeshiftedgeom[s,0]) > texcmpfile;
1827      }
1828      printf("\\\\\n") > texcmpfile;
1829      printf("arithm. mean   ") > texcmpfile;
1830      for( o = 0; o < nsolver; ++o )
1831      {
1832         s = printorder[o];
1833         printf("& %8s & %8.1f", texint(nodetotal[s,0]/max(1,nevalprobs[s,0])), timetotal[s,0]/max(1,nevalprobs[s,0])) > texcmpfile;
1834      }
1835      printf("\\\\\n") > texcmpfile;
1836
1837      # add statistics for problems solved to optimality
1838      printf("\\midrule\n") > texcmpfile;
1839      printf("\\multicolumn{%d}{@{}l}{all optimal}\\\\\n", 1 + 2 * nsolver) > texcmpfile;
1840      printf("geom. mean     ") > texcmpfile;
1841      for( o = 0; o < nsolver; ++o )
1842      {
1843         s = printorder[o];
1844         printf("& %8s & %8.1f", texint(nodegeom[s,-1]), timegeom[s,-1]) > texcmpfile;
1845      }
1846      printf("\\\\\n") > texcmpfile;
1847      printf("sh. geom. mean ") > texcmpfile;
1848      for( o = 0; o < nsolver; ++o )
1849      {
1850         s = printorder[o];
1851         printf("& %8s & %8.1f", texint(nodeshiftedgeom[s,-1]), timeshiftedgeom[s,-1]) > texcmpfile;
1852      }
1853      printf("\\\\\n") > texcmpfile;
1854      printf("arithm. mean   ") > texcmpfile;
1855      for( o = 0; o < nsolver; ++o )
1856      {
1857         s = printorder[o];
1858         printf("& %8s & %8.1f", texint(nodetotal[s,-1]/max(1,nevalprobs[s,-1])), timetotal[s,-1]/max(1,nevalprobs[s,-1])) > texcmpfile;
1859      }
1860      printf("\\\\\n") > texcmpfile;
1861      printf("\\bottomrule\n") > texcmpfile;
1862      printf("\\end{tabular*}\n") > texcmpfile;
1863      printf("}\n") > texcmpfile;
1864   }
1865
1866   if( !short )
1867   {
1868      for( cat = 0; cat <= 3; cat++ )
1869      {
1870#         if( nprocessedprobs[cat] == 0 )
1871#            continue;
1872
1873         header = (cat == -1 ? "optimal" : (cat == 0 ? "all" : (cat == 1 ? "diff" : (cat == 2 ? "equal" : "timeout"))));
1874         printf("\n");
1875         printf("%-7s                                            proc eval fail time solv wins bett wors bobj wobj feas    gnodes   shnodes   gnodesQ  shnodesQ   gtime  shtime  gtimeQ shtimeQ   score\n",
1876		header);
1877
1878         for( o = 0; o < nsolver; ++o )
1879         {
1880            s = printorder[o];
1881            sname = solvername[s];
1882            if( o == 0 )
1883            {
1884               nodegeomcomp = nodegeom[s,cat];
1885               timegeomcomp = timegeom[s,cat];
1886               nodeshiftedgeomcomp = nodeshiftedgeom[s,cat];
1887               timeshiftedgeomcomp = timeshiftedgeom[s,cat];
1888            }
1889            if( (o > 0 || cat == 0 || cat == -1) && nevalprobs[s,cat] > 0 )
1890            {
1891               if ( length(sname) <= 50 )
1892                  printf("%-50s %4d %4d %4d %4d %4d %4d", sname, nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
1893                         ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]);
1894               else
1895                  printf("*%-49s %4d %4d %4d %4d %4d %4d", substr(sname, length(sname)-48), nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
1896                         ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]);
1897
1898               printf(" %4d %4d", better[s,cat], worse[s,cat]);
1899               printf(" %4d %4d %4d %9d %9d %9.2f %9.2f %7.1f %7.1f %7.2f %7.2f %7.2f\n",
1900		      betterobj[s,cat], worseobj[s,cat], feasibles[s,cat],
1901		      nodegeom[s,cat], nodeshiftedgeom[s,cat], nodegeom[s,cat]/refnodegeom[s,cat],
1902		      nodeshiftedgeom[s,cat]/refnodeshiftedgeom[s,cat],
1903		      timegeom[s,cat], timeshiftedgeom[s,cat], timegeom[s,cat]/reftimegeom[s,cat],
1904		      timeshiftedgeom[s,cat]/reftimeshiftedgeom[s,cat], score[s,cat]);
1905            }
1906         }
1907         if( cat == 0 )
1908         {
1909            printf("%-50s           %4d %4d %4d %4s", "optimal auto settings", bestnfails, bestntimeouts, bestnsolved, "");
1910            printf(" %4d %4s", bestbetter, "");
1911            printf(" %4d %4s %4d %9d %9d %9.2f %9.2f %7.1f %7.1f %7.2f %7.2f %7s\n",
1912                   bestbetterobj, "", bestfeasibles,
1913                   bestnodegeom, bestnodeshiftedgeom, bestnodegeom/nodegeomcomp, bestnodeshiftedgeom/nodeshiftedgeomcomp,
1914                   besttimegeom, besttimeshiftedgeom, besttimegeom/timegeomcomp, besttimeshiftedgeom/timeshiftedgeomcomp,
1915                   "");
1916         }
1917      }
1918
1919      # output the all optimal case
1920      header = "all optimal";
1921      printf("\n");
1922      printf("%-11s                                        proc eval fail time solv wins bett wors bobj wobj feas    gnodes   shnodes   gnodesQ  shnodesQ   gtime  shtime  gtimeQ shtimeQ   score\n", header);
1923      cat = -1;
1924      for( o = 0; o < nsolver; ++o )
1925      {
1926         s = printorder[o];
1927         sname = solvername[s];
1928         if( o == 0 )
1929         {
1930            nodegeomcomp = nodegeom[s,cat];
1931            timegeomcomp = timegeom[s,cat];
1932            nodeshiftedgeomcomp = nodeshiftedgeom[s,cat];
1933            timeshiftedgeomcomp = timeshiftedgeom[s,cat];
1934         }
1935         if( (o > 0 || cat == 0 || cat == -1) && nevalprobs[s,cat] > 0 )
1936         {
1937            if ( length(sname) <= 50 )
1938               printf("%-50s %4d %4d %4d %4d %4d %4d", sname, nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
1939                      ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]);
1940            else
1941               printf("*%-49s %4d %4d %4d %4d %4d %4d", substr(sname, length(sname)-48), nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
1942                      ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]);
1943            printf(" %4d %4d", better[s,cat], worse[s,cat]);
1944            printf(" %4d %4d %4d %9d %9d %9.2f %9.2f %7.1f %7.1f %7.2f %7.2f %7.2f\n",
1945                   betterobj[s,cat], worseobj[s,cat], feasibles[s,cat],
1946                   nodegeom[s,cat], nodeshiftedgeom[s,cat], nodegeom[s,cat]/refnodegeom[s,cat],
1947                   nodeshiftedgeom[s,cat]/refnodeshiftedgeom[s,cat],
1948                   timegeom[s,cat], timeshiftedgeom[s,cat], timegeom[s,cat]/reftimegeom[s,cat],
1949                   timeshiftedgeom[s,cat]/reftimeshiftedgeom[s,cat], score[s,cat]);
1950         }
1951      }
1952      if( cat == 0 )
1953      {
1954         printf("%-50s           %4d %4d %4d %4s", "optimal auto settings", bestnfails, bestntimeouts, bestnsolved, "");
1955         printf(" %4d %4s", bestbetter, "");
1956         printf(" %4d %4s %4d %9d %9d %9.2f %9.2f %7.1f %7.1f %7.2f %7.2f %7s\n",
1957                bestbetterobj, "", bestfeasibles,
1958                bestnodegeom, bestnodeshiftedgeom, bestnodegeom/nodegeomcomp, bestnodeshiftedgeom/nodeshiftedgeomcomp,
1959                besttimegeom, besttimeshiftedgeom, besttimegeom/timegeomcomp, besttimeshiftedgeom/timeshiftedgeomcomp,
1960                "");
1961      }
1962   }
1963
1964   printf("\n");
1965   printf("total time over all settings: %.1f sec = %.1f hours = %.1f days = %.1f weeks = %.1f months\n",
1966	  fulltotaltime, fulltotaltime/3600.0, fulltotaltime/(3600.0*24), fulltotaltime/(3600.0*24*7),
1967	  fulltotaltime/(3600.0*24*30));
1968
1969   # generate tex file
1970   if( texfile != "" )
1971   {
1972      hasequalpath = 0;
1973      for( o = 0; o < nsolver; ++o )
1974      {
1975         if( nevalprobs[s,2] > 0 )
1976         {
1977            hasequalpath = 1;
1978            break;
1979         }
1980      }
1981
1982      printf("generating tex file <%s>\n", texfile);
1983      printf("{\\sffamily\n") > texfile;
1984      printf("\\scriptsize\n") > texfile;
1985      printf("\\setlength{\\extrarowheight}{1pt}\n") > texfile;
1986      printf("\\setlength{\\tabcolsep}{2pt}\n") > texfile;
1987      printf("\\newcommand{\\spc}{\\hspace{%dem}}\n", 2-hasequalpath) > texfile;
1988
1989      printf("\\begin{tabular*}{\\columnwidth}{@{\\extracolsep{\\fill}}lrrr@{\\spc}rrrrrr@{\\spc}rrrr") > texfile;
1990      if( hasequalpath )
1991         printf("@{\\spc}rrrr") > texfile;
1992      printf("@{}}\n") > texfile;
1993
1994      printf("\\toprule\n") > texfile;
1995
1996      printf("& & & & \\multicolumn{6}{c@{\\spc}}{all instances (%d)} & \\multicolumn{4}{c@{\\spc}}{different path}",
1997	     nevalprobs[printorder[0],0]) > texfile;
1998      if( hasequalpath )
1999         printf("& \\multicolumn{4}{c}{equal path}") > texfile;
2000      printf("\\\\\n") > texfile;
2001
2002      printf("setting & T & fst & slw & $\\textit{n}_\\textit{gm}$ & $\\textit{n}_\\textit{sgm}$ & $\\textit{n}_\\textit{tot}$ & $\\textit{t}_\\textit{gm}$ & $\\textit{t}_\\textit{sgm}$ & $\\textit{t}_\\textit{tot}$ & \\# & $\\textit{t}_\\textit{gm}$ & $\\textit{t}_\\textit{sgm}$ & $\\textit{t}_\\textit{tot}$") > texfile;
2003      if( hasequalpath )
2004         printf("& \\# & $\\textit{t}_\\textit{gm}$ & $\\textit{t}_\\textit{sgm}$ & $\\textit{t}_\\textit{tot}$") > texfile;
2005      printf("\\\\\n") > texfile;
2006
2007      printf("\\midrule\n") > texfile;
2008
2009      for( o = 0; o < nsolver; ++o )
2010      {
2011         s = printorder[o];
2012         printf("%-45s & %4d & %3d & %3d", texsolvername(s), ntimeouts[s,0],  better[s,0], worse[s,0]) > texfile;
2013         printf(" & %5s & %5s & %5s & %5s & %5s & %5s",
2014		texcompstr(nodegeom[s,0], refnodegeom[s,0]),
2015		texcompstr(nodeshiftedgeom[s,0], refnodeshiftedgeom[s,0]),
2016		texcompstr(nodetotal[s,0], refnodetotal[s,0]),
2017		texcompstr(timegeom[s,0], reftimegeom[s,0]),
2018		texcompstr(timeshiftedgeom[s,0], reftimeshiftedgeom[s,0]),
2019		texcompstr(timetotal[s,0], reftimetotal[s,0])) > texfile;
2020         if( nevalprobs[s,1] > 0 )
2021         {
2022            printf(" & %2d & %5s & %5s & %5s",
2023		   nevalprobs[s,1],
2024		   texcompstr(timegeom[s,1], reftimegeom[s,1]),
2025		   texcompstr(timeshiftedgeom[s,1], reftimeshiftedgeom[s,1]),
2026		   texcompstr(timetotal[s,1], reftimetotal[s,1])) > texfile;
2027         }
2028         else
2029            printf(" &  0 &     --- &     --- &     ---") > texfile;
2030         if( hasequalpath )
2031         {
2032            if( nevalprobs[s,2] > 0 )
2033            {
2034               printf(" & %2d & %5s & %5s & %5s",
2035		      nevalprobs[s,2],
2036		      texcompstr(timegeom[s,2], reftimegeom[s,2]),
2037		      texcompstr(timeshiftedgeom[s,2], reftimeshiftedgeom[s,2]),
2038		      texcompstr(timetotal[s,2], reftimetotal[s,2])) > texfile;
2039            }
2040            else
2041               printf(" &  0 &     --- &     --- &     ---") > texfile;
2042         }
2043         printf(" \\\\\n") > texfile;
2044      }
2045
2046      printf("\\bottomrule\n") > texfile;
2047      printf("\\end{tabular*}\n") > texfile;
2048      printf("}\n") > texfile;
2049
2050      # extend tex include file
2051      if( texincfile != "" )
2052      {
2053         n = split(texfile, a, "/");
2054         texpath = "";
2055         for( i = 1; i < n; i++ )
2056            texpath = texpath a[i] "/";
2057         texbase = a[n];
2058         sub(/\.tex/, "", texbase);
2059         n = split(texbase, a, "_");
2060         texsetname = a[2];
2061         texgroupname = a[3];
2062         textestname = a[4];
2063
2064         printf("\n") >> texincfile;
2065         printf("\\begin{table}[hp]\n") > texincfile;
2066         printf("\\input{Tables/mip/auto/%s}\n", texbase) > texincfile;
2067         printf("\\smalltabcaption{\\label{table_%s_%s_%s}\n", texsetname, texgroupname, textestname) > texincfile;
2068         printf("Evaluation of \\setting%s%s on test set \\testset{%s}.}\n", texsetname, texgroupname, textestname) > texincfile;
2069         printf("\\end{table}\n") > texincfile;
2070         printf("\n") > texincfile;
2071      }
2072   }
2073
2074   # generate (or extend) tex summary file
2075   if( texsummaryfile != "" )
2076   {
2077      n = split(texsummaryfile, a, "/");
2078      texsummarypath = "";
2079      for( i = 1; i < n; i++ )
2080         texsummarypath = texsummarypath a[i] "/";
2081      texsummarybase = a[n];
2082      sub(/\.tex/, "", texsummarybase);
2083      texsummaryfiletime = texsummarypath texsummarybase "_time.tex";
2084      texsummaryfilenodes = texsummarypath texsummarybase "_nodes.tex";
2085      if( texsummaryheader > 0 )
2086      {
2087         printf("{\\sffamily\n") > texsummaryfile;
2088         printf("\\scriptsize\n") > texsummaryfile;
2089         printf("\\setlength{\\extrarowheight}{1pt}\n") > texsummaryfile;
2090         printf("\\setlength{\\tabcolsep}{2pt}\n") > texsummaryfile;
2091         printf("\\newcommand{\\spc}{\\hspace{1em}}\n") > texsummaryfile;
2092         for( si = 0; si <= 2; si++ )
2093         {
2094            printf("\\ifthenelse{\\summaryinfo = %d}{\n", si) > texsummaryfile;
2095            printf("\\begin{tabular*}{\\columnwidth}{@{}ll@{\\extracolsep{\\fill}}") > texsummaryfile;
2096            for( o = 1; o < nsolver; o++ )
2097               printf("r") > texsummaryfile;
2098            printf("@{}}\n") > texsummaryfile;
2099            printf("\\toprule\n") > texsummaryfile;
2100            printf("& test set") > texsummaryfile;
2101            if( nsolver >= 9 )
2102            {
2103               for( o = 1; o < nsolver; o++ )
2104                  printf(" & %s", texsolvername(printorder[o])) > texsummaryfile;
2105            }
2106            else
2107            {
2108               for( o = 1; o < nsolver; o++ )
2109                  printf(" & \\makebox[0em][r]{%s}", texsolvername(printorder[o])) > texsummaryfile;
2110            }
2111            printf(" \\\\\n") > texsummaryfile;
2112            if( si == 0 || si == 1 )
2113            {
2114               printf("\\midrule\n") > texsummaryfile;
2115               printf("\\input{Tables/mip/auto/%s_time}\n", texsummarybase) > texsummaryfile;
2116            }
2117            if( si == 0 || si == 2 )
2118            {
2119               printf("\\midrule\n") > texsummaryfile;
2120               printf("\\input{Tables/mip/auto/%s_nodes}\n", texsummarybase) > texsummaryfile;
2121            }
2122            printf("\\bottomrule\n") >> texsummaryfile;
2123            printf("\\end{tabular*}\n") >> texsummaryfile;
2124            printf("}{}\n") >> texsummaryfile;
2125         }
2126         printf("}\n") > texsummaryfile;
2127         printf("\\raisebox{-%.1fex}[0em][0em]{\\rotatebox{90}{\\makebox[3em]{time}}}",
2128		1.5*(texsummaryheader+1)) > texsummaryfiletime;
2129         printf("\\raisebox{-%.1fex}[0em][0em]{\\rotatebox{90}{\\makebox[3em]{nodes}}}",
2130		1.5*(texsummaryheader+1)) > texsummaryfilenodes;
2131      }
2132      printf("& \\testset{%s}", textestset) >> texsummaryfiletime;
2133      for( o = 1; o < nsolver; o++ )
2134      {
2135         s = printorder[o];
2136         if( texsummaryshifted )
2137            printf(" & %s", texcompstr(timeshiftedgeom[s,0], reftimeshiftedgeom[s,0])) > texsummaryfiletime;
2138         else
2139            printf(" & %s", texcompstr(timegeom[s,0], reftimegeom[s,0])) > texsummaryfiletime;
2140      }
2141      printf("\\\\\n") > texsummaryfiletime;
2142      printf("& \\testset{%s}", textestset) >> texsummaryfilenodes;
2143      for( o = 1; o < nsolver; o++ )
2144      {
2145         s = printorder[o];
2146         if( texsummaryshifted )
2147            printf(" & %s", texcompstr(nodeshiftedgeom[s,0], refnodeshiftedgeom[s,0])) > texsummaryfilenodes;
2148         else
2149            printf(" & %s", texcompstr(nodegeom[s,0], refnodegeom[s,0])) > texsummaryfilenodes;
2150      }
2151      printf("\\\\\n") > texsummaryfilenodes;
2152
2153      # add tex comment to summary file which is later be used to generate overall statistics
2154      for( o = 1; o < nsolver; o++ )
2155      {
2156         s = printorder[o];
2157         weight = (texsummaryweight == 0 ? nevalprobs[s,0] : texsummaryweight);
2158         if( texsummaryshifted )
2159            printf("%% =mean=  %s %.4f %.4f %g\n", solvername[s],
2160		   timeshiftedgeom[s,0]/reftimeshiftedgeom[s,0], nodeshiftedgeom[s,0]/refnodeshiftedgeom[s,0],
2161		   weight) >> texsummaryfile;
2162         else
2163            printf("%% =mean=  %s %.4f %.4f %g\n", solvername[s],
2164		   timegeom[s,0]/reftimegeom[s,0], nodegeom[s,0]/refnodegeom[s,0], weight) >> texsummaryfile;
2165      }
2166   }
2167
2168   if( diagramfile != "" )
2169   {
2170      refsolver = 0;
2171      mintime = timeshiftedgeom[0,0];
2172      maxtime = timeshiftedgeom[0,0];
2173      for( s = 0; s < nsolver; ++s )
2174      {
2175         if( substr(solvername[s], 1, 4) == "SCIP" )
2176            refsolver = s;
2177
2178         if( timeshiftedgeom[s,0] < mintime )
2179            mintime = timeshiftedgeom[s,0];
2180         else if( timeshiftedgeom[s,0] > mintime )
2181            maxtime = timeshiftedgeom[s,0];
2182      }
2183      # bound, over which solvers get cut off
2184      upperbound = diagramnsteps*timeshiftedgeom[refsolver,0];
2185
2186      yscale = 3000/upperbound;
2187
2188      printf("\\documentclass{article}\n\\usepackage{tikz}\n\\pagestyle{empty}\n\\begin{document}\n\n") > diagramfile;
2189
2190      printf("\\definecolor{c1}{HTML}{000060}\n") > diagramfile;
2191      printf("\\definecolor{c2}{HTML}{0000FF}\n") > diagramfile;
2192      printf("\\definecolor{c3}{HTML}{36648B}\n") > diagramfile;
2193      printf("\\definecolor{c4}{HTML}{4682B4}\n") > diagramfile;
2194      printf("\\definecolor{c5}{HTML}{5CACEE}\n") > diagramfile;
2195      printf("\\definecolor{c6}{HTML}{00FFFF}\n") > diagramfile;
2196      #printf("\\definecolor{c7}{HTML}{A0FFFF}\n") > diagramfile;
2197      printf("\\definecolor{c7}{HTML}{008888}\n") > diagramfile;
2198      printf("\\definecolor{c8}{HTML}{00DD99}\n") > diagramfile;
2199      printf("\\definecolor{c9}{HTML}{527B10}\n") > diagramfile;
2200      printf("\\definecolor{c10}{HTML}{7BC618}\n") > diagramfile;
2201      printf("\\definecolor{c11}{HTML}{A00060}\n") > diagramfile;
2202      printf("\\definecolor{c12}{HTML}{A000FF}\n") > diagramfile;
2203      printf("\\definecolor{c13}{HTML}{E6648B}\n") > diagramfile;
2204      printf("\\definecolor{c14}{HTML}{F682B4}\n") > diagramfile;
2205      printf("\\definecolor{c15}{HTML}{DCACEE}\n") > diagramfile;
2206      printf("\\definecolor{c16}{HTML}{A0FFFF}\n") > diagramfile;
2207      #printf("\\definecolor{c7}{HTML}{A0FFFF}\n") > diagramfile;
2208      printf("\\definecolor{c17}{HTML}{A08888}\n") > diagramfile;
2209      printf("\\definecolor{c18}{HTML}{A0DD99}\n") > diagramfile;
2210      printf("\\definecolor{c19}{HTML}{D27B10}\n") > diagramfile;
2211      printf("\\definecolor{c20}{HTML}{FBC618}\n") > diagramfile;
2212
2213      printf("\\definecolor{darkgreen}{HTML}{006600}\n") > diagramfile;
2214      if( diagramyellowbg )
2215	 printf("\\definecolor{background}{HTML}{FFFFE6}\n\n") > diagramfile;
2216      else
2217	 printf("\\definecolor{background}{HTML}{FFFFFF}\n\n") > diagramfile;
2218
2219      printf("\\begin{tikzpicture}[auto,scale=0.8,yscale=%1.2f]\n",yscale) > diagramfile;
2220      printf("\n%% tikz styles\n") > diagramfile;
2221      printf("\\tikzstyle{box} +=[draw=black,rectangle,inner sep=0mm, minimum size = 2.5mm,left];\n") > diagramfile;
2222      printf("\\tikzstyle{legend} +=[inner sep=1mm, right];\n") > diagramfile;
2223      printf("\\tikzstyle{tic} +=[left];\n") > diagramfile;
2224      printf("\\tikzstyle{bel} +=[below,inner sep=0.8mm];\n") > diagramfile;
2225      printf("\\tikzstyle{abo} +=[above,inner sep=0.3mm];\n\n") > diagramfile;
2226
2227      #extendedub = (1+1/(2*diagramnsteps-2))*upperbound;
2228      extendedub = 1.2*upperbound;
2229      printf("\\draw[background,fill=background] (-1.5,-0.3) rectangle (%1.2f,%1.2f);\n",nsolver+6.5,extendedub/1000.0+0.1) > diagramfile;
2230
2231      printf("%% Beschriftungen und Hoehenlinien\n") > diagramfile;
2232      for( i = 0; i < diagramnsteps; ++i )
2233      {
2234         perc = i/(diagramnsteps-1.0)*upperbound/1000.0;
2235         printf("\\node () at (0.2,%1.2f) [tic] {\\small %d};\n",perc,perc*1000) > diagramfile;
2236         printf("\\draw[dotted] (0.2,%1.2f) -- (%1.2f,%1.2f);\n",perc,nsolver+0.8,perc) > diagramfile;
2237      }
2238      if( diagramgerman )
2239         printf("\\node () at (-1,%1.2f) [rotate=90]{\\footnotesize Zeit in Sekunden};\n",extendedub/2000.0) > diagramfile;
2240      else
2241         printf("\\node () at (-1,%1.2f) [rotate=90]{\\footnotesize time in seconds};\n",extendedub/2000.0) > diagramfile;
2242      printf("\\draw[] (0.2,0) rectangle (%1.2f,%1.2f);\n\n",nsolver+0.8,extendedub/1000.0) > diagramfile;
2243
2244      printf("%% BALKEN\n") > diagramfile;
2245      for( i = 1; i <= nsolver; ++i )
2246      {
2247         pos[i] = i-1;
2248         if( timeshiftedgeom[pos[i],0] >= upperbound)
2249            printf("\\draw[fill=c%d] (%d.5,0) rectangle (%d.5,%1.3f);\n",i,i-1,i,extendedub/1000.0) > diagramfile;
2250         else
2251            printf("\\draw[fill=c%d] (%d.5,0) rectangle (%d.5,%1.3f);\n",i,i-1,i,timeshiftedgeom[pos[i],0]/1000.0) > diagramfile;
2252      }
2253      printf("\n") > diagramfile;
2254
2255      printf("%% TRENNLINIEN\n") > diagramfile;
2256      upperbreak = 0.92 * extendedub/1000.0;
2257      lowerbreak = 0.90 * extendedub/1000.0;
2258      printf("\\fill[background] (0.1,%1.4f) rectangle (%d.5,%1.4f);\n",lowerbreak,nsolver,upperbreak) > diagramfile;
2259      printf("\\draw[black] (0.1,%1.4f) --  (0.3,%1.4f);\n",lowerbreak,lowerbreak) > diagramfile;
2260      printf("\\draw[black] (0.1,%1.4f) --  (0.3,%1.4f);\n",upperbreak,upperbreak) > diagramfile;
2261
2262      printf("%% BALKENBESCHRIFTUNG \n") > diagramfile;
2263      printf("\\draw[c%d] (0.2,%1.3f) -- (%1.2f,%1.3f);\n",refsolver+1,timeshiftedgeom[refsolver,0]/1000.0,
2264	     nsolver+0.8,timeshiftedgeom[refsolver,0]/1000.0) > diagramfile;
2265
2266      for( i = 1; i <= nsolver; ++i )
2267      {
2268	 if( timeshiftedgeom[pos[i],0] >= upperbound )
2269	    printf("\\node () at (%d,%1.3f) [bel,inner sep=0.3mm] {\\footnotesize\\textcolor{white}{%2.1fx}};\n",i,
2270		   extendedub/1000.0,timeshiftedgeom[pos[i],0]/timeshiftedgeom[refsolver,0]) > diagramfile;
2271	 else if( pos[i] == refsolver )
2272	    printf("\\node () at (%d,%1.3f) [abo] {\\footnotesize\\textbf{%1.2fx}};\n",i,
2273		   timeshiftedgeom[pos[i],0]/1000.0,timeshiftedgeom[pos[i],0]/timeshiftedgeom[refsolver,0]) > diagramfile;
2274	 else
2275	    printf("\\node () at (%d,%1.3f) [abo] {\\footnotesize %1.2fx};\n",i,
2276		   timeshiftedgeom[pos[i],0]/1000.0,timeshiftedgeom[pos[i],0]/timeshiftedgeom[refsolver,0]) > diagramfile;
2277      }
2278      printf("\n") > diagramfile;
2279
2280      printf("%% NOT SOLVED\n") > diagramfile;
2281      if( diagramgerman )
2282         printf("\\node () at (-0.4, %1.3f) [bel]{\\footnotesize\\textcolor{blue}{Nicht gel\\\"ost}};\n", -upperbound/6000.0) > diagramfile;
2283      else
2284         printf("\\node () at (-0.4, %1.3f) [bel]{\\footnotesize\\textcolor{blue}{not solved}};\n", -upperbound/6000.0) > diagramfile;
2285      for( i = 1; i <= nsolver; ++i )
2286         printf("\\node () at (%d, %1.3f) [bel]{\\footnotesize\\textcolor{blue}{%2.0f\\%}};\n",
2287		i,-upperbound/6000.0,100.0-100.0*((nsolved[pos[i],0]+0.0)/(nprocessedprobs[pos[i],0]+0.0))) > diagramfile;
2288      printf("\n") > diagramfile;
2289
2290      printf("%% LEGEND\n") > diagramfile;
2291      for( i = 1; i <= nsolver; ++i )
2292      {
2293         perc = (nsolver-i)/(nsolver-1.0)*extendedub/1000.0;
2294         if( pos[i] == refsolver )
2295            printf("\\node () at (%1.4f,%1.4f) [legend]{\\small \\textbf{%s}};\n",nsolver+1.4,perc,solvername[pos[i]]) > diagramfile;
2296         else
2297            printf("\\node () at (%1.4f,%1.4f) [legend]{\\small %s};\n",nsolver+1.4,perc,solvername[pos[i]]) > diagramfile;
2298         printf("\\node () at (%1.4f,%1.4f) [box,color=c%d,fill=c%d]{};\n",nsolver+1.4,perc,i,i) > diagramfile;
2299      }
2300      printf("\\node () at (%1.4f,%1.3f) [legend]{\\footnotesize timings w.r.t. %d instances};\n",nsolver+1.4,-upperbound/6000.0,nevalprobs[printorder[0],0]) > diagramfile;
2301      printf("\\end{tikzpicture}\n") > diagramfile;
2302      printf("\n\\end{document}\n") > diagramfile;
2303   }
2304}
2305