1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2005, 2006, 2007, 2009, 2010, 2011, 2012, 2013, 2014, 2016 Free Software Foundation, Inc
3
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
16
17 #include <config.h>
18
19 #include <math.h>
20 #include <gsl/gsl_cdf.h>
21
22 #include "data/case.h"
23 #include "data/casegrouper.h"
24 #include "data/casereader.h"
25 #include "data/dataset.h"
26 #include "data/dictionary.h"
27 #include "data/format.h"
28 #include "data/variable.h"
29 #include "data/subcase.h"
30 #include "data/casewriter.h"
31 #include "data/short-names.h"
32 #include "language/command.h"
33 #include "language/lexer/lexer.h"
34 #include "language/lexer/variable-parser.h"
35 #include "language/stats/sort-criteria.h"
36 #include "math/sort.h"
37 #include "libpspp/assertion.h"
38 #include "libpspp/i18n.h"
39 #include "libpspp/message.h"
40 #include "libpspp/misc.h"
41 #include "libpspp/pool.h"
42 #include "libpspp/string-set.h"
43 #include "libpspp/taint.h"
44 #include "output/pivot-table.h"
45
46 #include "gettext.h"
47 #define _(msgid) gettext (msgid)
48 #define N_(msgid) (msgid)
49
50 struct rank;
51
52 typedef double (*rank_function_t) (const struct rank*, double c, double cc, double cc_1,
53 int i, double w);
54
55 static double rank_proportion (const struct rank *, double c, double cc, double cc_1,
56 int i, double w);
57
58 static double rank_normal (const struct rank *, double c, double cc, double cc_1,
59 int i, double w);
60
61 static double rank_percent (const struct rank *, double c, double cc, double cc_1,
62 int i, double w);
63
64 static double rank_rfraction (const struct rank *, double c, double cc, double cc_1,
65 int i, double w);
66
67 static double rank_rank (const struct rank *, double c, double cc, double cc_1,
68 int i, double w);
69
70 static double rank_n (const struct rank *, double c, double cc, double cc_1,
71 int i, double w);
72
73 static double rank_savage (const struct rank *, double c, double cc, double cc_1,
74 int i, double w);
75
76 static double rank_ntiles (const struct rank *, double c, double cc, double cc_1,
77 int i, double w);
78
79
80 enum rank_func
81 {
82 RANK,
83 NORMAL,
84 PERCENT,
85 RFRACTION,
86 PROPORTION,
87 N,
88 NTILES,
89 SAVAGE,
90 n_RANK_FUNCS
91 };
92
93 static const struct fmt_spec dest_format[n_RANK_FUNCS] = {
94 {FMT_F, 9, 3}, /* rank */
95 {FMT_F, 6, 4}, /* normal */
96 {FMT_F, 6, 2}, /* percent */
97 {FMT_F, 6, 4}, /* rfraction */
98 {FMT_F, 6, 4}, /* proportion */
99 {FMT_F, 6, 0}, /* n */
100 {FMT_F, 3, 0}, /* ntiles */
101 {FMT_F, 8, 4} /* savage */
102 };
103
104 static const char * const function_name[n_RANK_FUNCS] = {
105 "RANK",
106 "NORMAL",
107 "PERCENT",
108 "RFRACTION",
109 "PROPORTION",
110 "N",
111 "NTILES",
112 "SAVAGE"
113 };
114
115 static const rank_function_t rank_func[n_RANK_FUNCS] = {
116 rank_rank,
117 rank_normal,
118 rank_percent,
119 rank_rfraction,
120 rank_proportion,
121 rank_n,
122 rank_ntiles,
123 rank_savage
124 };
125
126
127 enum ties
128 {
129 TIES_LOW,
130 TIES_HIGH,
131 TIES_MEAN,
132 TIES_CONDENSE
133 };
134
135 enum fraction
136 {
137 FRAC_BLOM,
138 FRAC_RANKIT,
139 FRAC_TUKEY,
140 FRAC_VW
141 };
142
143 struct rank_spec
144 {
145 enum rank_func rfunc;
146 const char **dest_names;
147 const char **dest_labels;
148 };
149
150 /* If NEW_NAME exists in DICT or NEW_NAMES, returns NULL without changing
151 anything. Otherwise, inserts NEW_NAME in NEW_NAMES and returns the copy of
152 NEW_NAME now in NEW_NAMES. */
153 static const char *
try_new_name(const char * new_name,const struct dictionary * dict,struct string_set * new_names)154 try_new_name (const char *new_name,
155 const struct dictionary *dict, struct string_set *new_names)
156 {
157 return (!dict_lookup_var (dict, new_name)
158 && string_set_insert (new_names, new_name)
159 ? string_set_find_node (new_names, new_name)->string
160 : NULL);
161 }
162
163 /* Returns a variable name for storing ranks of a variable named SRC_NAME
164 according to the rank function F. The name chosen will not be one already in
165 DICT or NEW_NAMES.
166
167 If successful, adds the new name to NEW_NAMES and returns the name added.
168 If no name can be generated, returns NULL. */
169 static const char *
rank_choose_dest_name(struct dictionary * dict,struct string_set * new_names,enum rank_func f,const char * src_name)170 rank_choose_dest_name (struct dictionary *dict, struct string_set *new_names,
171 enum rank_func f, const char *src_name)
172 {
173 char *src_name_7;
174 char name[128];
175 const char *s;
176 int i;
177
178 /* Try the first character of the ranking function followed by the first 7
179 bytes of the srcinal variable name. */
180 src_name_7 = utf8_encoding_trunc (src_name, dict_get_encoding (dict), 7);
181 snprintf (name, sizeof name, "%c%s", function_name[f][0], src_name_7);
182 free (src_name_7);
183 s = try_new_name (name, dict, new_names);
184 if (s != NULL)
185 return s;
186
187 /* Try "fun###". */
188 for (i = 1; i <= 999; i++)
189 {
190 sprintf (name, "%.3s%03d", function_name[f], i);
191 s = try_new_name (name, dict, new_names);
192 if (s != NULL)
193 return s;
194 }
195
196 /* Try "RNKfn##". */
197 for (i = 1; i <= 99; i++)
198 {
199 sprintf (name, "RNK%.2s%02d", function_name[f], i);
200 s = try_new_name (name, dict, new_names);
201 if (s != NULL)
202 return s;
203 }
204
205 msg (ME, _("Cannot generate variable name for ranking %s with %s. "
206 "All candidates in use."),
207 src_name, function_name[f]);
208 return NULL;
209 }
210
211 struct rank
212 {
213 struct dictionary *dict;
214
215 struct subcase sc;
216
217 const struct variable **vars;
218 size_t n_vars;
219
220 const struct variable **group_vars;
221 size_t n_group_vars;
222
223
224 enum mv_class exclude;
225
226 struct rank_spec *rs;
227 size_t n_rs;
228
229 enum ties ties;
230
231 enum fraction fraction;
232 int k_ntiles;
233
234 bool print;
235
236 /* Pool on which cell functions may allocate data */
237 struct pool *pool;
238 };
239
240
241 static void
destroy_rank(struct rank * rank)242 destroy_rank (struct rank *rank)
243 {
244 free (rank->vars);
245 free (rank->group_vars);
246 subcase_destroy (&rank->sc);
247 pool_destroy (rank->pool);
248 }
249
250 static bool
parse_into(struct lexer * lexer,struct rank * cmd,struct string_set * new_names)251 parse_into (struct lexer *lexer, struct rank *cmd,
252 struct string_set *new_names)
253 {
254 int var_count = 0;
255 struct rank_spec *rs = NULL;
256
257 cmd->rs = pool_realloc (cmd->pool, cmd->rs, sizeof (*cmd->rs) * (cmd->n_rs + 1));
258 rs = &cmd->rs[cmd->n_rs];
259
260 if (lex_match_id (lexer, "RANK"))
261 {
262 rs->rfunc = RANK;
263 }
264 else if (lex_match_id (lexer, "NORMAL"))
265 {
266 rs->rfunc = NORMAL;
267 }
268 else if (lex_match_id (lexer, "RFRACTION"))
269 {
270 rs->rfunc = RFRACTION;
271 }
272 else if (lex_match_id (lexer, "N"))
273 {
274 rs->rfunc = N;
275 }
276 else if (lex_match_id (lexer, "SAVAGE"))
277 {
278 rs->rfunc = SAVAGE;
279 }
280 else if (lex_match_id (lexer, "PERCENT"))
281 {
282 rs->rfunc = PERCENT;
283 }
284 else if (lex_match_id (lexer, "PROPORTION"))
285 {
286 rs->rfunc = PROPORTION;
287 }
288 else if (lex_match_id (lexer, "NTILES"))
289 {
290 if (!lex_force_match (lexer, T_LPAREN))
291 return false;
292
293 if (! lex_force_int (lexer))
294 return false;
295
296 cmd->k_ntiles = lex_integer (lexer);
297 lex_get (lexer);
298
299 if (!lex_force_match (lexer, T_RPAREN))
300 return false;
301
302 rs->rfunc = NTILES;
303 }
304 else
305 {
306 lex_error (lexer, NULL);
307 return false;
308 }
309
310 cmd->n_rs++;
311 rs->dest_names = pool_calloc (cmd->pool, cmd->n_vars,
312 sizeof *rs->dest_names);
313
314 if (lex_match_id (lexer, "INTO"))
315 {
316 while(lex_token (lexer) == T_ID)
317 {
318 const char *name = lex_tokcstr (lexer);
319
320 if (var_count >= subcase_get_n_fields (&cmd->sc))
321 msg (SE, _("Too many variables in %s clause."), "INTO");
322 else if (dict_lookup_var (cmd->dict, name) != NULL)
323 msg (SE, _("Variable %s already exists."), name);
324 else if (string_set_contains (new_names, name))
325 msg (SE, _("Duplicate variable name %s."), name);
326 else
327 {
328 string_set_insert (new_names, name);
329 rs->dest_names[var_count++] = pool_strdup (cmd->pool, name);
330 lex_get (lexer);
331 continue;
332 }
333
334 /* Error path. */
335 return false;
336 }
337 }
338
339 return true;
340 }
341
342 /* Hardly a rank function !! */
343 static double
rank_n(const struct rank * cmd UNUSED,double c UNUSED,double cc UNUSED,double cc_1 UNUSED,int i UNUSED,double w)344 rank_n (const struct rank *cmd UNUSED, double c UNUSED, double cc UNUSED, double cc_1 UNUSED,
345 int i UNUSED, double w)
346 {
347 return w;
348 }
349
350
351 static double
rank_rank(const struct rank * cmd,double c,double cc,double cc_1,int i,double w UNUSED)352 rank_rank (const struct rank *cmd, double c, double cc, double cc_1,
353 int i, double w UNUSED)
354 {
355 double rank;
356
357 if (c >= 1.0)
358 {
359 switch (cmd->ties)
360 {
361 case TIES_LOW:
362 rank = cc_1 + 1;
363 break;
364 case TIES_HIGH:
365 rank = cc;
366 break;
367 case TIES_MEAN:
368 rank = cc_1 + (c + 1.0)/ 2.0;
369 break;
370 case TIES_CONDENSE:
371 rank = i;
372 break;
373 default:
374 NOT_REACHED ();
375 }
376 }
377 else
378 {
379 switch (cmd->ties)
380 {
381 case TIES_LOW:
382 rank = cc_1;
383 break;
384 case TIES_HIGH:
385 rank = cc;
386 break;
387 case TIES_MEAN:
388 rank = cc_1 + c / 2.0 ;
389 break;
390 case TIES_CONDENSE:
391 rank = i;
392 break;
393 default:
394 NOT_REACHED ();
395 }
396 }
397
398 return rank;
399 }
400
401
402 static double
rank_rfraction(const struct rank * cmd,double c,double cc,double cc_1,int i,double w)403 rank_rfraction (const struct rank *cmd, double c, double cc, double cc_1,
404 int i, double w)
405 {
406 return rank_rank (cmd, c, cc, cc_1, i, w) / w ;
407 }
408
409
410 static double
rank_percent(const struct rank * cmd,double c,double cc,double cc_1,int i,double w)411 rank_percent (const struct rank *cmd, double c, double cc, double cc_1,
412 int i, double w)
413 {
414 return rank_rank (cmd, c, cc, cc_1, i, w) * 100.0 / w ;
415 }
416
417
418 static double
rank_proportion(const struct rank * cmd,double c,double cc,double cc_1,int i,double w)419 rank_proportion (const struct rank *cmd, double c, double cc, double cc_1,
420 int i, double w)
421 {
422 const double r = rank_rank (cmd, c, cc, cc_1, i, w) ;
423
424 double f;
425
426 switch (cmd->fraction)
427 {
428 case FRAC_BLOM:
429 f = (r - 3.0/8.0) / (w + 0.25);
430 break;
431 case FRAC_RANKIT:
432 f = (r - 0.5) / w ;
433 break;
434 case FRAC_TUKEY:
435 f = (r - 1.0/3.0) / (w + 1.0/3.0);
436 break;
437 case FRAC_VW:
438 f = r / (w + 1.0);
439 break;
440 default:
441 NOT_REACHED ();
442 }
443
444
445 return (f > 0) ? f : SYSMIS;
446 }
447
448 static double
rank_normal(const struct rank * cmd,double c,double cc,double cc_1,int i,double w)449 rank_normal (const struct rank *cmd, double c, double cc, double cc_1,
450 int i, double w)
451 {
452 double f = rank_proportion (cmd, c, cc, cc_1, i, w);
453
454 return gsl_cdf_ugaussian_Pinv (f);
455 }
456
457 static double
rank_ntiles(const struct rank * cmd,double c,double cc,double cc_1,int i,double w)458 rank_ntiles (const struct rank *cmd, double c, double cc, double cc_1,
459 int i, double w)
460 {
461 double r = rank_rank (cmd, c, cc, cc_1, i, w);
462
463
464 return (floor ((r * cmd->k_ntiles) / (w + 1)) + 1);
465 }
466
467 /* Expected value of the order statistics from an exponential distribution */
468 static double
ee(int j,double w_star)469 ee (int j, double w_star)
470 {
471 int k;
472 double sum = 0.0;
473
474 for (k = 1 ; k <= j; k++)
475 sum += 1.0 / (w_star + 1 - k);
476
477 return sum;
478 }
479
480
481 static double
rank_savage(const struct rank * cmd UNUSED,double c,double cc,double cc_1,int i UNUSED,double w)482 rank_savage (const struct rank *cmd UNUSED, double c, double cc, double cc_1,
483 int i UNUSED, double w)
484 {
485 double int_part;
486 const int i_1 = floor (cc_1);
487 const int i_2 = floor (cc);
488
489 const double w_star = (modf (w, &int_part) == 0) ? w : floor (w) + 1;
490
491 const double g_1 = cc_1 - i_1;
492 const double g_2 = cc - i_2;
493
494 /* The second factor is infinite, when the first is zero.
495 Therefore, evaluate the second, only when the first is non-zero */
496 const double expr1 = (1 - g_1) ? (1 - g_1) * ee(i_1+1, w_star) : (1 - g_1);
497 const double expr2 = g_2 ? g_2 * ee (i_2+1, w_star) : g_2 ;
498
499 if (i_1 == i_2)
500 return ee (i_1 + 1, w_star) - 1;
501
502 if (i_1 + 1 == i_2)
503 return ((expr1 + expr2)/c) - 1;
504
505 if (i_1 + 2 <= i_2)
506 {
507 int j;
508 double sigma = 0.0;
509 for (j = i_1 + 2 ; j <= i_2; ++j)
510 sigma += ee (j, w_star);
511 return ((expr1 + expr2 + sigma) / c) -1;
512 }
513
514 NOT_REACHED();
515 }
516
517 static double
sum_weights(const struct casereader * input,int weight_idx)518 sum_weights (const struct casereader *input, int weight_idx)
519 {
520 if (weight_idx == -1)
521 return casereader_count_cases (input);
522 else
523 {
524 struct casereader *pass;
525 struct ccase *c;
526 double w;
527
528 w = 0.0;
529 pass = casereader_clone (input);
530 for (; (c = casereader_read (pass)) != NULL; case_unref (c))
531 w += case_num_idx (c, weight_idx);
532 casereader_destroy (pass);
533
534 return w;
535 }
536 }
537
538 static void
rank_sorted_file(struct casereader * input,struct casewriter * output,int weight_idx,const struct rank * cmd)539 rank_sorted_file (struct casereader *input,
540 struct casewriter *output,
541 int weight_idx,
542 const struct rank *cmd)
543 {
544 struct casegrouper *tie_grouper;
545 struct casereader *tied_cases;
546 struct subcase input_var;
547 int tie_group = 1;
548 struct ccase *c;
549 double cc = 0.0;
550 double w;
551
552 /* Get total group weight. */
553 w = sum_weights (input, weight_idx);
554
555 /* Do ranking. */
556 subcase_init (&input_var, 0, 0, SC_ASCEND);
557 tie_grouper = casegrouper_create_subcase (input, &input_var);
558 subcase_destroy (&input_var);
559 for (; casegrouper_get_next_group (tie_grouper, &tied_cases);
560 casereader_destroy (tied_cases))
561 {
562 double tw = sum_weights (tied_cases, weight_idx);
563 double cc_1 = cc;
564 cc += tw;
565
566 taint_propagate (casereader_get_taint (tied_cases),
567 casewriter_get_taint (output));
568
569 /* Rank tied cases. */
570 for (; (c = casereader_read (tied_cases)) != NULL; case_unref (c))
571 {
572 struct ccase *out_case;
573 size_t i;
574
575 out_case = case_create (casewriter_get_proto (output));
576 case_data_rw_idx (out_case, 0)->f = case_num_idx (c, 1);
577 for (i = 0; i < cmd->n_rs; ++i)
578 {
579 rank_function_t func = rank_func[cmd->rs[i].rfunc];
580 double rank = func (cmd, tw, cc, cc_1, tie_group, w);
581 case_data_rw_idx (out_case, i + 1)->f = rank;
582 }
583
584 casewriter_write (output, out_case);
585 }
586 tie_group++;
587 }
588 casegrouper_destroy (tie_grouper);
589 }
590
591
592 static bool
593 rank_cmd (struct dataset *ds, const struct rank *cmd);
594
595 static const char *
fraction_name(const struct rank * cmd)596 fraction_name (const struct rank *cmd)
597 {
598 switch (cmd->fraction)
599 {
600 case FRAC_BLOM: return "BLOM";
601 case FRAC_RANKIT: return "RANKIT";
602 case FRAC_TUKEY: return "TUKEY";
603 case FRAC_VW: return "VW";
604 default: NOT_REACHED ();
605 }
606 }
607
608 /* Returns a label for a variable derived from SRC_VAR with function F. */
609 static const char *
create_var_label(struct rank * cmd,const struct variable * src_var,enum rank_func f)610 create_var_label (struct rank *cmd, const struct variable *src_var,
611 enum rank_func f)
612 {
613 struct string label;
614 const char *pool_label;
615
616 ds_init_empty (&label);
617
618 if (cmd->n_group_vars > 0)
619 {
620 struct string group_var_str;
621 int g;
622
623 ds_init_empty (&group_var_str);
624
625 for (g = 0 ; g < cmd->n_group_vars ; ++g)
626 {
627 if (g > 0) ds_put_cstr (&group_var_str, " ");
628 ds_put_cstr (&group_var_str, var_get_name (cmd->group_vars[g]));
629 }
630
631 ds_put_format (&label, _("%s of %s by %s"), function_name[f],
632 var_get_name (src_var), ds_cstr (&group_var_str));
633 ds_destroy (&group_var_str);
634 }
635 else
636 ds_put_format (&label, _("%s of %s"),
637 function_name[f], var_get_name (src_var));
638
639 pool_label = pool_strdup (cmd->pool, ds_cstr (&label));
640
641 ds_destroy (&label);
642
643 return pool_label;
644 }
645
646 int
cmd_rank(struct lexer * lexer,struct dataset * ds)647 cmd_rank (struct lexer *lexer, struct dataset *ds)
648 {
649 struct string_set new_names;
650 struct rank rank;
651 struct rank_spec *rs;
652
653 subcase_init_empty (&rank.sc);
654
655 rank.rs = NULL;
656 rank.n_rs = 0;
657 rank.exclude = MV_ANY;
658 rank.n_group_vars = 0;
659 rank.group_vars = NULL;
660 rank.dict = dataset_dict (ds);
661 rank.ties = TIES_MEAN;
662 rank.fraction = FRAC_BLOM;
663 rank.print = true;
664 rank.pool = pool_create ();
665
666 string_set_init (&new_names);
667
668 if (lex_match_id (lexer, "VARIABLES"))
669 if (! lex_force_match (lexer, T_EQUALS))
670 goto error;
671
672 if (!parse_sort_criteria (lexer, rank.dict,
673 &rank.sc,
674 &rank.vars, NULL))
675 goto error;
676
677 rank.n_vars = rank.sc.n_fields;
678
679 if (lex_match (lexer, T_BY))
680 {
681 if (! parse_variables_const (lexer, rank.dict,
682 &rank.group_vars, &rank.n_group_vars,
683 PV_NO_DUPLICATE | PV_NO_SCRATCH))
684 goto error;
685 }
686
687
688 while (lex_token (lexer) != T_ENDCMD)
689 {
690 if (! lex_force_match (lexer, T_SLASH))
691 goto error;
692 if (lex_match_id (lexer, "TIES"))
693 {
694 if (! lex_force_match (lexer, T_EQUALS))
695 goto error;
696 if (lex_match_id (lexer, "MEAN"))
697 {
698 rank.ties = TIES_MEAN;
699 }
700 else if (lex_match_id (lexer, "LOW"))
701 {
702 rank.ties = TIES_LOW;
703 }
704 else if (lex_match_id (lexer, "HIGH"))
705 {
706 rank.ties = TIES_HIGH;
707 }
708 else if (lex_match_id (lexer, "CONDENSE"))
709 {
710 rank.ties = TIES_CONDENSE;
711 }
712 else
713 {
714 lex_error (lexer, NULL);
715 goto error;
716 }
717 }
718 else if (lex_match_id (lexer, "FRACTION"))
719 {
720 if (! lex_force_match (lexer, T_EQUALS))
721 goto error;
722 if (lex_match_id (lexer, "BLOM"))
723 {
724 rank.fraction = FRAC_BLOM;
725 }
726 else if (lex_match_id (lexer, "TUKEY"))
727 {
728 rank.fraction = FRAC_TUKEY;
729 }
730 else if (lex_match_id (lexer, "VW"))
731 {
732 rank.fraction = FRAC_VW;
733 }
734 else if (lex_match_id (lexer, "RANKIT"))
735 {
736 rank.fraction = FRAC_RANKIT;
737 }
738 else
739 {
740 lex_error (lexer, NULL);
741 goto error;
742 }
743 }
744 else if (lex_match_id (lexer, "PRINT"))
745 {
746 if (! lex_force_match (lexer, T_EQUALS))
747 goto error;
748 if (lex_match_id (lexer, "YES"))
749 {
750 rank.print = true;
751 }
752 else if (lex_match_id (lexer, "NO"))
753 {
754 rank.print = false;
755 }
756 else
757 {
758 lex_error (lexer, NULL);
759 goto error;
760 }
761 }
762 else if (lex_match_id (lexer, "MISSING"))
763 {
764 if (! lex_force_match (lexer, T_EQUALS))
765 goto error;
766 if (lex_match_id (lexer, "INCLUDE"))
767 {
768 rank.exclude = MV_SYSTEM;
769 }
770 else if (lex_match_id (lexer, "EXCLUDE"))
771 {
772 rank.exclude = MV_ANY;
773 }
774 else
775 {
776 lex_error (lexer, NULL);
777 goto error;
778 }
779 }
780 else if (! parse_into (lexer, &rank, &new_names))
781 goto error;
782 }
783
784
785 /* If no rank specs are given, then apply a default */
786 if (rank.n_rs == 0)
787 {
788 struct rank_spec *rs;
789
790 rs = pool_calloc (rank.pool, 1, sizeof *rs);
791 rs->rfunc = RANK;
792 rs->dest_names = pool_calloc (rank.pool, rank.n_vars,
793 sizeof *rs->dest_names);
794
795 rank.rs = rs;
796 rank.n_rs = 1;
797 }
798
799 /* Choose variable names for all rank destinations which haven't already been
800 created with INTO. */
801 for (rs = rank.rs; rs < &rank.rs[rank.n_rs]; rs++)
802 {
803 rs->dest_labels = pool_calloc (rank.pool, rank.n_vars,
804 sizeof *rs->dest_labels);
805 for (int v = 0 ; v < rank.n_vars ; v ++)
806 {
807 const char **dst_name = &rs->dest_names[v];
808 if (*dst_name == NULL)
809 {
810 *dst_name = rank_choose_dest_name (rank.dict, &new_names,
811 rs->rfunc,
812 var_get_name (rank.vars[v]));
813 if (*dst_name == NULL)
814 goto error;
815 }
816
817 rs->dest_labels[v] = create_var_label (&rank, rank.vars[v],
818 rs->rfunc);
819 }
820 }
821
822 if (rank.print)
823 {
824 struct pivot_table *table = pivot_table_create (
825 N_("Variables Created by RANK"));
826 table->omit_empty = true;
827
828 pivot_dimension_create (table, PIVOT_AXIS_COLUMN, N_("New Variable"),
829 N_("New Variable"), N_("Function"),
830 N_("Fraction"), N_("Grouping Variables"));
831
832 struct pivot_dimension *variables = pivot_dimension_create (
833 table, PIVOT_AXIS_ROW, N_("Existing Variable"),
834 N_("Existing Variable"));
835 variables->root->show_label = true;
836
837 for (size_t i = 0 ; i < rank.n_rs ; ++i)
838 {
839 for (size_t v = 0 ; v < rank.n_vars ; v ++)
840 {
841 int row_idx = pivot_category_create_leaf (
842 variables->root, pivot_value_new_variable (rank.vars[v]));
843
844 struct string group_vars = DS_EMPTY_INITIALIZER;
845 for (int g = 0 ; g < rank.n_group_vars ; ++g)
846 {
847 if (g)
848 ds_put_byte (&group_vars, ' ');
849 ds_put_cstr (&group_vars, var_get_name (rank.group_vars[g]));
850 }
851
852 enum rank_func rfunc = rank.rs[i].rfunc;
853 bool has_fraction = rfunc == NORMAL || rfunc == PROPORTION;
854 const char *entries[] =
855 {
856 rank.rs[i].dest_names[v],
857 function_name[rank.rs[i].rfunc],
858 has_fraction ? fraction_name (&rank) : NULL,
859 rank.n_group_vars ? ds_cstr (&group_vars) : NULL,
860 };
861 for (size_t j = 0; j < sizeof entries / sizeof *entries; j++)
862 {
863 const char *entry = entries[j];
864 if (entry)
865 pivot_table_put2 (table, j, row_idx,
866 pivot_value_new_user_text (entry, -1));
867 }
868 ds_destroy (&group_vars);
869 }
870 }
871
872 pivot_table_submit (table);
873 }
874
875 /* Do the ranking */
876 rank_cmd (ds, &rank);
877
878 destroy_rank (&rank);
879 string_set_destroy (&new_names);
880 return CMD_SUCCESS;
881
882 error:
883
884 destroy_rank (&rank);
885 string_set_destroy (&new_names);
886 return CMD_FAILURE;
887 }
888
889 /* RANK transformation. */
890 struct rank_trns
891 {
892 int order_case_idx;
893
894 struct rank_trns_input_var *input_vars;
895 size_t n_input_vars;
896
897 size_t n_funcs;
898 };
899
900 struct rank_trns_input_var
901 {
902 struct casereader *input;
903 struct ccase *current;
904
905 struct variable **output_vars;
906 };
907
908 static void
advance_ranking(struct rank_trns_input_var * iv)909 advance_ranking (struct rank_trns_input_var *iv)
910 {
911 case_unref (iv->current);
912 iv->current = casereader_read (iv->input);
913 }
914
915 static int
rank_trns_proc(void * trns_,struct ccase ** c,casenumber case_idx UNUSED)916 rank_trns_proc (void *trns_, struct ccase **c, casenumber case_idx UNUSED)
917 {
918 struct rank_trns *trns = trns_;
919 double order = case_num_idx (*c, trns->order_case_idx);
920 struct rank_trns_input_var *iv;
921
922 *c = case_unshare (*c);
923 for (iv = trns->input_vars; iv < &trns->input_vars[trns->n_input_vars]; iv++)
924 while (iv->current != NULL)
925 {
926 double iv_order = case_num_idx (iv->current, 0);
927 if (iv_order == order)
928 {
929 size_t i;
930
931 for (i = 0; i < trns->n_funcs; i++)
932 case_data_rw (*c, iv->output_vars[i])->f
933 = case_num_idx (iv->current, i + 1);
934 advance_ranking (iv);
935 break;
936 }
937 else if (iv_order > order)
938 break;
939 else
940 advance_ranking (iv);
941 }
942 return TRNS_CONTINUE;
943 }
944
945 static bool
rank_trns_free(void * trns_)946 rank_trns_free (void *trns_)
947 {
948 struct rank_trns *trns = trns_;
949 struct rank_trns_input_var *iv;
950
951 for (iv = trns->input_vars; iv < &trns->input_vars[trns->n_input_vars]; iv++)
952 {
953 casereader_destroy (iv->input);
954 case_unref (iv->current);
955
956 free (iv->output_vars);
957 }
958 free (trns->input_vars);
959 free (trns);
960
961 return true;
962 }
963
964 static bool
rank_cmd(struct dataset * ds,const struct rank * cmd)965 rank_cmd (struct dataset *ds, const struct rank *cmd)
966 {
967 struct dictionary *d = dataset_dict (ds);
968 struct variable *weight_var = dict_get_weight (d);
969 struct casewriter **outputs;
970 struct variable *order_var;
971 struct casereader *input;
972 struct rank_trns *trns;
973 bool ok = true;
974 int i;
975
976 order_var = add_permanent_ordering_transformation (ds);
977
978 /* Create output files. */
979 {
980 struct caseproto *output_proto;
981 struct subcase by_order;
982
983 output_proto = caseproto_create ();
984 for (i = 0; i < cmd->n_rs + 1; i++)
985 output_proto = caseproto_add_width (output_proto, 0);
986
987 subcase_init (&by_order, 0, 0, SC_ASCEND);
988
989 outputs = xnmalloc (cmd->n_vars, sizeof *outputs);
990 for (i = 0; i < cmd->n_vars; i++)
991 outputs[i] = sort_create_writer (&by_order, output_proto);
992
993 subcase_destroy (&by_order);
994 caseproto_unref (output_proto);
995 }
996
997 /* Open the active file and make one pass per input variable. */
998 input = proc_open (ds);
999 input = casereader_create_filter_weight (input, d, NULL, NULL);
1000 for (i = 0 ; i < cmd->n_vars ; ++i)
1001 {
1002 const struct variable *input_var = cmd->vars[i];
1003 struct casereader *input_pass;
1004 struct casegrouper *split_grouper;
1005 struct casereader *split_group;
1006 struct subcase rank_ordering;
1007 struct subcase projection;
1008 struct subcase split_vars;
1009 struct subcase group_vars;
1010 int weight_idx;
1011 int j;
1012
1013 /* Discard cases that have missing values of input variable. */
1014 input_pass = i == cmd->n_vars - 1 ? input : casereader_clone (input);
1015 input_pass = casereader_create_filter_missing (input_pass, &input_var, 1,
1016 cmd->exclude, NULL, NULL);
1017
1018 /* Keep only the columns we really need, to save time and space when we
1019 sort them just below.
1020
1021 After this projection, the input_pass case indexes look like:
1022
1023 - 0: input_var.
1024 - 1: order_var.
1025 - 2 and up: cmd->n_group_vars group variables
1026 - 2 + cmd->n_group_vars and up: split variables
1027 - 2 + cmd->n_group_vars + n_split_vars: weight var
1028 */
1029 subcase_init_empty (&projection);
1030 subcase_add_var_always (&projection, input_var, SC_ASCEND);
1031 subcase_add_var_always (&projection, order_var, SC_ASCEND);
1032 subcase_add_vars_always (&projection,
1033 cmd->group_vars, cmd->n_group_vars);
1034 subcase_add_vars_always (&projection, dict_get_split_vars (d),
1035 dict_get_split_cnt (d));
1036 if (weight_var != NULL)
1037 {
1038 subcase_add_var_always (&projection, weight_var, SC_ASCEND);
1039 weight_idx = 2 + cmd->n_group_vars + dict_get_split_cnt (d);
1040 }
1041 else
1042 weight_idx = -1;
1043 input_pass = casereader_project (input_pass, &projection);
1044 subcase_destroy (&projection);
1045
1046 /* Prepare 'group_vars' as the set of grouping variables. */
1047 subcase_init_empty (&group_vars);
1048 for (j = 0; j < cmd->n_group_vars; j++)
1049 subcase_add_always (&group_vars,
1050 j + 2, var_get_width (cmd->group_vars[j]),
1051 SC_ASCEND);
1052
1053 /* Prepare 'rank_ordering' for sorting with the group variables as
1054 primary key and the input variable as secondary key. */
1055 subcase_clone (&rank_ordering, &group_vars);
1056 subcase_add (&rank_ordering, 0, 0, subcase_get_direction (&cmd->sc, i));
1057
1058 /* Group by split variables */
1059 subcase_init_empty (&split_vars);
1060 for (j = 0; j < dict_get_split_cnt (d); j++)
1061 subcase_add_always (&split_vars, 2 + j + cmd->n_group_vars,
1062 var_get_width (dict_get_split_vars (d)[j]),
1063 SC_ASCEND);
1064 split_grouper = casegrouper_create_subcase (input_pass, &split_vars);
1065 subcase_destroy (&split_vars);
1066 while (casegrouper_get_next_group (split_grouper, &split_group))
1067 {
1068 struct casereader *ordered;
1069 struct casegrouper *by_grouper;
1070 struct casereader *by_group;
1071
1072 ordered = sort_execute (split_group, &rank_ordering);
1073 by_grouper = casegrouper_create_subcase (ordered, &group_vars);
1074 while (casegrouper_get_next_group (by_grouper, &by_group))
1075 rank_sorted_file (by_group, outputs[i], weight_idx, cmd);
1076 ok = casegrouper_destroy (by_grouper) && ok;
1077 }
1078 subcase_destroy (&group_vars);
1079 subcase_destroy (&rank_ordering);
1080
1081 ok = casegrouper_destroy (split_grouper) && ok;
1082 }
1083 ok = proc_commit (ds) && ok;
1084
1085 /* Re-fetch the dictionary and order variable, because if TEMPORARY was in
1086 effect then there's a new dictionary. */
1087 d = dataset_dict (ds);
1088 order_var = dict_lookup_var_assert (d, "$ORDER");
1089
1090 /* Merge the original data set with the ranks (which we already sorted on
1091 $ORDER). */
1092 trns = xmalloc (sizeof *trns);
1093 trns->order_case_idx = var_get_case_index (order_var);
1094 trns->input_vars = xnmalloc (cmd->n_vars, sizeof *trns->input_vars);
1095 trns->n_input_vars = cmd->n_vars;
1096 trns->n_funcs = cmd->n_rs;
1097 for (i = 0; i < trns->n_input_vars; i++)
1098 {
1099 struct rank_trns_input_var *iv = &trns->input_vars[i];
1100 int j;
1101
1102 iv->input = casewriter_make_reader (outputs[i]);
1103 iv->current = casereader_read (iv->input);
1104 iv->output_vars = xnmalloc (trns->n_funcs, sizeof *iv->output_vars);
1105 for (j = 0; j < trns->n_funcs; j++)
1106 {
1107 struct rank_spec *rs = &cmd->rs[j];
1108 struct variable *var;
1109
1110 var = dict_create_var_assert (d, rs->dest_names[i], 0);
1111 var_set_both_formats (var, &dest_format[rs->rfunc]);
1112 var_set_label (var, rs->dest_labels[i]);
1113
1114 iv->output_vars[j] = var;
1115 }
1116 }
1117 free (outputs);
1118
1119 add_transformation (ds, rank_trns_proc, rank_trns_free, trns);
1120
1121 /* Delete our sort key, which we don't need anymore. */
1122 dict_delete_var (d, order_var);
1123
1124 return ok;
1125 }
1126