1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006, 2009, 2010, 2011, 2012, 2013, 2014 Free Software Foundation, Inc.
3
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
16 #include <config.h>
17
18 #include <ctype.h>
19 #include <errno.h>
20 #include <float.h>
21 #include <stdlib.h>
22
23 #include "data/any-reader.h"
24 #include "data/attributes.h"
25 #include "data/casereader.h"
26 #include "data/dataset.h"
27 #include "data/dictionary.h"
28 #include "data/file-handle-def.h"
29 #include "data/format.h"
30 #include "data/missing-values.h"
31 #include "data/value-labels.h"
32 #include "data/variable.h"
33 #include "data/vector.h"
34 #include "language/command.h"
35 #include "language/data-io/file-handle.h"
36 #include "language/lexer/lexer.h"
37 #include "language/lexer/variable-parser.h"
38 #include "libpspp/array.h"
39 #include "libpspp/hash-functions.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
42 #include "libpspp/misc.h"
43 #include "libpspp/pool.h"
44 #include "libpspp/string-array.h"
45 #include "output/pivot-table.h"
46 #include "output/text-item.h"
47 #include "output/table-item.h"
48
49 #include "gl/count-one-bits.h"
50 #include "gl/localcharset.h"
51 #include "gl/intprops.h"
52 #include "gl/minmax.h"
53 #include "gl/xalloc.h"
54
55 #include "gettext.h"
56 #define _(msgid) gettext (msgid)
57 #define N_(msgid) (msgid)
58
59 /* Information to include in displaying a dictionary. */
60 enum
61 {
62 /* Variable table. */
63 DF_NAME = 1 << 0,
64 DF_POSITION = 1 << 1,
65 DF_LABEL = 1 << 2,
66 DF_MEASUREMENT_LEVEL = 1 << 3,
67 DF_ROLE = 1 << 4,
68 DF_WIDTH = 1 << 5,
69 DF_ALIGNMENT = 1 << 6,
70 DF_PRINT_FORMAT = 1 << 7,
71 DF_WRITE_FORMAT = 1 << 8,
72 DF_MISSING_VALUES = 1 << 9,
73 #define DF_ALL_VARIABLE ((1 << 10) - 1)
74
75 /* Value labels table. */
76 DF_VALUE_LABELS = 1 << 10,
77
78 /* Attribute table. */
79 DF_AT_ATTRIBUTES = 1 << 11, /* Attributes whose names begin with @. */
80 DF_ATTRIBUTES = 1 << 12, /* All other attributes. */
81 };
82
83 static void display_variables (const struct variable **, size_t, int flags);
84 static void display_value_labels (const struct variable **, size_t);
85 static void display_attributes (const struct attrset *,
86 const struct variable **, size_t, int flags);
87
88 static void report_encodings (const struct file_handle *, struct pool *,
89 char **titles, bool *ids,
90 char **strings, size_t n_strings);
91
92 static char *get_documents_as_string (const struct dictionary *);
93
94 static void
add_row(struct pivot_table * table,const char * attribute,struct pivot_value * value)95 add_row (struct pivot_table *table, const char *attribute,
96 struct pivot_value *value)
97 {
98 int row = pivot_category_create_leaf (table->dimensions[0]->root,
99 pivot_value_new_text (attribute));
100 if (value)
101 pivot_table_put1 (table, row, value);
102 }
103
104 /* SYSFILE INFO utility. */
105 int
cmd_sysfile_info(struct lexer * lexer,struct dataset * ds)106 cmd_sysfile_info (struct lexer *lexer, struct dataset *ds)
107 {
108 struct any_reader *any_reader;
109 struct file_handle *h;
110 struct dictionary *d;
111 struct casereader *reader;
112 struct any_read_info info;
113 char *encoding;
114
115 h = NULL;
116 encoding = NULL;
117 for (;;)
118 {
119 lex_match (lexer, T_SLASH);
120
121 if (lex_match_id (lexer, "FILE") || lex_is_string (lexer))
122 {
123 lex_match (lexer, T_EQUALS);
124
125 fh_unref (h);
126 h = fh_parse (lexer, FH_REF_FILE, NULL);
127 if (h == NULL)
128 goto error;
129 }
130 else if (lex_match_id (lexer, "ENCODING"))
131 {
132 lex_match (lexer, T_EQUALS);
133
134 if (!lex_force_string (lexer))
135 goto error;
136
137 free (encoding);
138 encoding = ss_xstrdup (lex_tokss (lexer));
139
140 lex_get (lexer);
141 }
142 else
143 break;
144 }
145
146 if (h == NULL)
147 {
148 lex_sbc_missing ("FILE");
149 goto error;
150 }
151
152 any_reader = any_reader_open (h);
153 if (!any_reader)
154 {
155 free (encoding);
156 return CMD_FAILURE;
157 }
158
159 if (encoding && !strcasecmp (encoding, "detect"))
160 {
161 char **titles, **strings;
162 struct pool *pool;
163 size_t n_strings;
164 bool *ids;
165
166 pool = pool_create ();
167 n_strings = any_reader_get_strings (any_reader, pool,
168 &titles, &ids, &strings);
169 any_reader_close (any_reader);
170
171 report_encodings (h, pool, titles, ids, strings, n_strings);
172 fh_unref (h);
173 pool_destroy (pool);
174 free (encoding);
175
176 return CMD_SUCCESS;
177 }
178
179 reader = any_reader_decode (any_reader, encoding, &d, &info);
180 if (!reader)
181 goto error;
182 casereader_destroy (reader);
183
184 struct pivot_table *table = pivot_table_create (N_("File Information"));
185 pivot_dimension_create (table, PIVOT_AXIS_ROW, N_("Attribute"));
186
187 add_row (table, N_("File"),
188 pivot_value_new_user_text (fh_get_file_name (h), -1));
189
190 const char *label = dict_get_label (d);
191 add_row (table, N_("Label"),
192 label ? pivot_value_new_user_text (label, -1) : NULL);
193
194 add_row (table, N_("Created"),
195 pivot_value_new_user_text_nocopy (
196 xasprintf ("%s %s by %s", info.creation_date,
197 info.creation_time, info.product)));
198
199 if (info.product_ext)
200 add_row (table, N_("Product"),
201 pivot_value_new_user_text (info.product_ext, -1));
202
203 add_row (table, N_("Integer Format"),
204 pivot_value_new_text (
205 info.integer_format == INTEGER_MSB_FIRST ? N_("Big Endian")
206 : info.integer_format == INTEGER_LSB_FIRST ? N_("Little Endian")
207 : N_("Unknown")));
208
209 add_row (table, N_("Real Format"),
210 pivot_value_new_text (
211 info.float_format == FLOAT_IEEE_DOUBLE_LE ? N_("IEEE 754 LE.")
212 : info.float_format == FLOAT_IEEE_DOUBLE_BE ? N_("IEEE 754 BE.")
213 : info.float_format == FLOAT_VAX_D ? N_("VAX D.")
214 : info.float_format == FLOAT_VAX_G ? N_("VAX G.")
215 : info.float_format == FLOAT_Z_LONG ? N_("IBM 390 Hex Long.")
216 : N_("Unknown")));
217
218 add_row (table, N_("Variables"),
219 pivot_value_new_integer (dict_get_var_cnt (d)));
220
221 add_row (table, N_("Cases"),
222 (info.case_cnt == -1
223 ? pivot_value_new_text (N_("Unknown"))
224 : pivot_value_new_integer (info.case_cnt)));
225
226 add_row (table, N_("Type"),
227 pivot_value_new_text (info.klass->name));
228
229 struct variable *weight_var = dict_get_weight (d);
230 add_row (table, N_("Weight"),
231 (weight_var
232 ? pivot_value_new_variable (weight_var)
233 : pivot_value_new_text (N_("Not weighted"))));
234
235 add_row (table, N_("Compression"),
236 (info.compression == ANY_COMP_NONE
237 ? pivot_value_new_text (N_("None"))
238 : pivot_value_new_user_text (
239 info.compression == ANY_COMP_SIMPLE ? "SAV" : "ZSAV", -1)));
240
241 add_row (table, N_("Encoding"),
242 pivot_value_new_user_text (dict_get_encoding (d), -1));
243
244 if (dict_get_document_line_cnt (d) > 0)
245 add_row (table, N_("Documents"),
246 pivot_value_new_user_text_nocopy (get_documents_as_string (d)));
247
248 pivot_table_submit (table);
249
250 size_t n_vars = dict_get_var_cnt (d);
251 const struct variable **vars = xnmalloc (n_vars, sizeof *vars);
252 for (size_t i = 0; i < dict_get_var_cnt (d); i++)
253 vars[i] = dict_get_var (d, i);
254 display_variables (vars, n_vars, DF_ALL_VARIABLE);
255 display_value_labels (vars, n_vars);
256 display_attributes (dict_get_attributes (dataset_dict (ds)),
257 vars, n_vars, DF_ATTRIBUTES);
258 free (vars);
259
260 dict_unref (d);
261
262 fh_unref (h);
263 free (encoding);
264 any_read_info_destroy (&info);
265 return CMD_SUCCESS;
266
267 error:
268 fh_unref (h);
269 free (encoding);
270 return CMD_FAILURE;
271 }
272
273 /* DISPLAY utility. */
274
275 static void display_macros (void);
276 static void display_documents (const struct dictionary *dict);
277 static void display_vectors (const struct dictionary *dict, int sorted);
278
279 int
cmd_display(struct lexer * lexer,struct dataset * ds)280 cmd_display (struct lexer *lexer, struct dataset *ds)
281 {
282 /* Whether to sort the list of variables alphabetically. */
283 int sorted;
284
285 /* Variables to display. */
286 size_t n;
287 const struct variable **vl;
288
289 if (lex_match_id (lexer, "MACROS"))
290 display_macros ();
291 else if (lex_match_id (lexer, "DOCUMENTS"))
292 display_documents (dataset_dict (ds));
293 else if (lex_match_id (lexer, "FILE"))
294 {
295 if (!lex_force_match_id (lexer, "LABEL"))
296 return CMD_FAILURE;
297
298 const char *label = dict_get_label (dataset_dict (ds));
299
300 struct pivot_table *table = pivot_table_create (N_("File Label"));
301 pivot_dimension_create (table, PIVOT_AXIS_ROW, N_("Label"),
302 N_("Label"));
303 pivot_table_put1 (table, 0,
304 (label ? pivot_value_new_user_text (label, -1)
305 : pivot_value_new_text (N_("(none)"))));
306 pivot_table_submit (table);
307 }
308 else
309 {
310 int flags;
311
312 sorted = lex_match_id (lexer, "SORTED");
313
314 if (lex_match_id (lexer, "VECTORS"))
315 {
316 display_vectors (dataset_dict(ds), sorted);
317 return CMD_SUCCESS;
318 }
319 else if (lex_match_id (lexer, "SCRATCH"))
320 {
321 dict_get_vars (dataset_dict (ds), &vl, &n, DC_ORDINARY);
322 flags = DF_NAME;
323 }
324 else
325 {
326 struct subcommand
327 {
328 const char *name;
329 int flags;
330 };
331 static const struct subcommand subcommands[] =
332 {
333 {"@ATTRIBUTES", DF_ATTRIBUTES | DF_AT_ATTRIBUTES},
334 {"ATTRIBUTES", DF_ATTRIBUTES},
335 {"DICTIONARY", (DF_NAME | DF_POSITION | DF_LABEL
336 | DF_MEASUREMENT_LEVEL | DF_ROLE | DF_WIDTH
337 | DF_ALIGNMENT | DF_PRINT_FORMAT
338 | DF_WRITE_FORMAT | DF_MISSING_VALUES
339 | DF_VALUE_LABELS)},
340 {"INDEX", DF_NAME | DF_POSITION},
341 {"LABELS", DF_NAME | DF_POSITION | DF_LABEL},
342 {"NAMES", DF_NAME},
343 {"VARIABLES", (DF_NAME | DF_POSITION | DF_PRINT_FORMAT
344 | DF_WRITE_FORMAT | DF_MISSING_VALUES)},
345 {NULL, 0},
346 };
347 const struct subcommand *sbc;
348 struct dictionary *dict = dataset_dict (ds);
349
350 flags = 0;
351 for (sbc = subcommands; sbc->name != NULL; sbc++)
352 if (lex_match_id (lexer, sbc->name))
353 {
354 flags = sbc->flags;
355 break;
356 }
357
358 lex_match (lexer, T_SLASH);
359 lex_match_id (lexer, "VARIABLES");
360 lex_match (lexer, T_EQUALS);
361
362 if (lex_token (lexer) != T_ENDCMD)
363 {
364 if (!parse_variables_const (lexer, dict, &vl, &n, PV_NONE))
365 {
366 free (vl);
367 return CMD_FAILURE;
368 }
369 }
370 else
371 dict_get_vars (dict, &vl, &n, 0);
372 }
373
374 if (n > 0)
375 {
376 sort (vl, n, sizeof *vl, (sorted
377 ? compare_var_ptrs_by_name
378 : compare_var_ptrs_by_dict_index), NULL);
379
380 int variable_flags = flags & DF_ALL_VARIABLE;
381 if (variable_flags)
382 display_variables (vl, n, variable_flags);
383
384 if (flags & DF_VALUE_LABELS)
385 display_value_labels (vl, n);
386
387 int attribute_flags = flags & (DF_ATTRIBUTES | DF_AT_ATTRIBUTES);
388 if (attribute_flags)
389 display_attributes (dict_get_attributes (dataset_dict (ds)),
390 vl, n, attribute_flags);
391 }
392 else
393 msg (SW, _("No variables to display."));
394
395 free (vl);
396 }
397
398 return CMD_SUCCESS;
399 }
400
401 static void
display_macros(void)402 display_macros (void)
403 {
404 msg (SW, _("Macros not supported."));
405 }
406
407 static char *
get_documents_as_string(const struct dictionary * dict)408 get_documents_as_string (const struct dictionary *dict)
409 {
410 const struct string_array *documents = dict_get_documents (dict);
411 struct string s = DS_EMPTY_INITIALIZER;
412 for (size_t i = 0; i < documents->n; i++)
413 {
414 if (i)
415 ds_put_byte (&s, '\n');
416 ds_put_cstr (&s, documents->strings[i]);
417 }
418 return ds_steal_cstr (&s);
419 }
420
421 static void
display_documents(const struct dictionary * dict)422 display_documents (const struct dictionary *dict)
423 {
424 struct pivot_table *table = pivot_table_create (N_("Documents"));
425 struct pivot_dimension *d = pivot_dimension_create (
426 table, PIVOT_AXIS_COLUMN, N_("Documents"), N_("Document"));
427 d->hide_all_labels = true;
428
429 if (!dict_get_documents (dict)->n)
430 pivot_table_put1 (table, 0, pivot_value_new_text (N_("(none)")));
431 else
432 {
433 char *docs = get_documents_as_string (dict);
434 pivot_table_put1 (table, 0, pivot_value_new_user_text_nocopy (docs));
435 }
436
437 pivot_table_submit (table);
438 }
439
440 static void
display_variables(const struct variable ** vl,size_t n,int flags)441 display_variables (const struct variable **vl, size_t n, int flags)
442 {
443 struct pivot_table *table = pivot_table_create (N_("Variables"));
444
445 struct pivot_dimension *attributes = pivot_dimension_create (
446 table, PIVOT_AXIS_COLUMN, N_("Attributes"));
447
448 struct heading
449 {
450 int flag;
451 const char *title;
452 };
453 static const struct heading headings[] = {
454 { DF_POSITION, N_("Position") },
455 { DF_LABEL, N_("Label") },
456 { DF_MEASUREMENT_LEVEL, N_("Measurement Level") },
457 { DF_ROLE, N_("Role") },
458 { DF_WIDTH, N_("Width") },
459 { DF_ALIGNMENT, N_("Alignment") },
460 { DF_PRINT_FORMAT, N_("Print Format") },
461 { DF_WRITE_FORMAT, N_("Write Format") },
462 { DF_MISSING_VALUES, N_("Missing Values") },
463 };
464 for (size_t i = 0; i < sizeof headings / sizeof *headings; i++)
465 if (flags & headings[i].flag)
466 pivot_category_create_leaf (attributes->root,
467 pivot_value_new_text (headings[i].title));
468
469 struct pivot_dimension *names = pivot_dimension_create (
470 table, PIVOT_AXIS_ROW, N_("Name"));
471 names->root->show_label = true;
472
473 for (size_t i = 0; i < n; i++)
474 {
475 const struct variable *v = vl[i];
476
477 struct pivot_value *name = pivot_value_new_variable (v);
478 name->variable.show = SETTINGS_VALUE_SHOW_VALUE;
479 int row = pivot_category_create_leaf (names->root, name);
480
481 int x = 0;
482 if (flags & DF_POSITION)
483 pivot_table_put2 (table, x++, row, pivot_value_new_integer (
484 var_get_dict_index (v) + 1));
485
486 if (flags & DF_LABEL)
487 {
488 const char *label = var_get_label (v);
489 if (label)
490 pivot_table_put2 (table, x, row,
491 pivot_value_new_user_text (label, -1));
492 x++;
493 }
494
495 if (flags & DF_MEASUREMENT_LEVEL)
496 pivot_table_put2 (
497 table, x++, row,
498 pivot_value_new_text (measure_to_string (var_get_measure (v))));
499
500 if (flags & DF_ROLE)
501 pivot_table_put2 (
502 table, x++, row,
503 pivot_value_new_text (var_role_to_string (var_get_role (v))));
504
505 if (flags & DF_WIDTH)
506 pivot_table_put2 (
507 table, x++, row,
508 pivot_value_new_integer (var_get_display_width (v)));
509
510 if (flags & DF_ALIGNMENT)
511 pivot_table_put2 (
512 table, x++, row,
513 pivot_value_new_text (alignment_to_string (
514 var_get_alignment (v))));
515
516 if (flags & DF_PRINT_FORMAT)
517 {
518 const struct fmt_spec *print = var_get_print_format (v);
519 char s[FMT_STRING_LEN_MAX + 1];
520
521 pivot_table_put2 (
522 table, x++, row,
523 pivot_value_new_user_text (fmt_to_string (print, s), -1));
524 }
525
526 if (flags & DF_WRITE_FORMAT)
527 {
528 const struct fmt_spec *write = var_get_write_format (v);
529 char s[FMT_STRING_LEN_MAX + 1];
530
531 pivot_table_put2 (
532 table, x++, row,
533 pivot_value_new_user_text (fmt_to_string (write, s), -1));
534 }
535
536 if (flags & DF_MISSING_VALUES)
537 {
538 char *s = mv_to_string (var_get_missing_values (v),
539 var_get_encoding (v));
540 if (s)
541 pivot_table_put2 (
542 table, x, row,
543 pivot_value_new_user_text_nocopy (s));
544
545 x++;
546 }
547 }
548
549 pivot_table_submit (table);
550 }
551
552 static bool
any_value_labels(const struct variable ** vars,size_t n_vars)553 any_value_labels (const struct variable **vars, size_t n_vars)
554 {
555 for (size_t i = 0; i < n_vars; i++)
556 if (val_labs_count (var_get_value_labels (vars[i])))
557 return true;
558 return false;
559 }
560
561 static void
display_value_labels(const struct variable ** vars,size_t n_vars)562 display_value_labels (const struct variable **vars, size_t n_vars)
563 {
564 if (!any_value_labels (vars, n_vars))
565 return;
566
567 struct pivot_table *table = pivot_table_create (N_("Value Labels"));
568
569 pivot_dimension_create (table, PIVOT_AXIS_COLUMN,
570 N_("Label"), N_("Label"));
571
572 struct pivot_dimension *values = pivot_dimension_create (
573 table, PIVOT_AXIS_ROW, N_("Variable Value"));
574 values->root->show_label = true;
575
576 struct pivot_footnote *missing_footnote = pivot_table_create_footnote (
577 table, pivot_value_new_text (N_("User-missing value")));
578
579 for (size_t i = 0; i < n_vars; i++)
580 {
581 const struct val_labs *val_labs = var_get_value_labels (vars[i]);
582 size_t n_labels = val_labs_count (val_labs);
583 if (!n_labels)
584 continue;
585
586 struct pivot_category *group = pivot_category_create_group__ (
587 values->root, pivot_value_new_variable (vars[i]));
588
589 const struct val_lab **labels = val_labs_sorted (val_labs);
590 for (size_t j = 0; j < n_labels; j++)
591 {
592 const struct val_lab *vl = labels[j];
593
594 struct pivot_value *value = pivot_value_new_var_value (
595 vars[i], &vl->value);
596 if (value->type == PIVOT_VALUE_NUMERIC)
597 value->numeric.show = SETTINGS_VALUE_SHOW_VALUE;
598 else
599 value->string.show = SETTINGS_VALUE_SHOW_VALUE;
600 if (var_is_value_missing (vars[i], &vl->value, MV_USER))
601 pivot_value_add_footnote (value, missing_footnote);
602 int row = pivot_category_create_leaf (group, value);
603
604 struct pivot_value *label = pivot_value_new_var_value (
605 vars[i], &vl->value);
606 char *escaped_label = xstrdup (val_lab_get_escaped_label (vl));
607 if (label->type == PIVOT_VALUE_NUMERIC)
608 {
609 free (label->numeric.value_label);
610 label->numeric.value_label = escaped_label;
611 label->numeric.show = SETTINGS_VALUE_SHOW_LABEL;
612 }
613 else
614 {
615 free (label->string.value_label);
616 label->string.value_label = escaped_label;
617 label->string.show = SETTINGS_VALUE_SHOW_LABEL;
618 }
619 pivot_table_put2 (table, 0, row, label);
620 }
621 free (labels);
622 }
623 pivot_table_submit (table);
624 }
625
626 static bool
is_at_name(const char * name)627 is_at_name (const char *name)
628 {
629 return name[0] == '@' || (name[0] == '$' && name[1] == '@');
630 }
631
632 static size_t
count_attributes(const struct attrset * set,int flags)633 count_attributes (const struct attrset *set, int flags)
634 {
635 struct attrset_iterator i;
636 struct attribute *attr;
637 size_t n_attrs;
638
639 n_attrs = 0;
640 for (attr = attrset_first (set, &i); attr != NULL;
641 attr = attrset_next (set, &i))
642 if (flags & DF_AT_ATTRIBUTES || !is_at_name (attribute_get_name (attr)))
643 n_attrs += attribute_get_n_values (attr);
644 return n_attrs;
645 }
646
647 static void
display_attrset(struct pivot_table * table,struct pivot_value * set_name,const struct attrset * set,int flags)648 display_attrset (struct pivot_table *table, struct pivot_value *set_name,
649 const struct attrset *set, int flags)
650 {
651 size_t n_total = count_attributes (set, flags);
652 if (!n_total)
653 {
654 pivot_value_destroy (set_name);
655 return;
656 }
657
658 struct pivot_category *group = pivot_category_create_group__ (
659 table->dimensions[1]->root, set_name);
660
661 size_t n_attrs = attrset_count (set);
662 struct attribute **attrs = attrset_sorted (set);
663 for (size_t i = 0; i < n_attrs; i++)
664 {
665 const struct attribute *attr = attrs[i];
666 const char *name = attribute_get_name (attr);
667
668 if (!(flags & DF_AT_ATTRIBUTES) && is_at_name (name))
669 continue;
670
671 size_t n_values = attribute_get_n_values (attr);
672 for (size_t j = 0; j < n_values; j++)
673 {
674 int row = pivot_category_create_leaf (
675 group,
676 (n_values > 1
677 ? pivot_value_new_user_text_nocopy (xasprintf (
678 "%s[%zu]", name, j + 1))
679 : pivot_value_new_user_text (name, -1)));
680 pivot_table_put2 (table, 0, row,
681 pivot_value_new_user_text (
682 attribute_get_value (attr, j), -1));
683 }
684 }
685 free (attrs);
686 }
687
688 static void
display_attributes(const struct attrset * dict_attrset,const struct variable ** vars,size_t n_vars,int flags)689 display_attributes (const struct attrset *dict_attrset,
690 const struct variable **vars, size_t n_vars, int flags)
691 {
692 struct pivot_table *table = pivot_table_create (
693 N_("Variable and Dataset Attributes"));
694
695 pivot_dimension_create (table, PIVOT_AXIS_COLUMN,
696 N_("Value"), N_("Value"));
697
698 struct pivot_dimension *variables = pivot_dimension_create (
699 table, PIVOT_AXIS_ROW, N_("Variable and Name"));
700 variables->root->show_label = true;
701
702 display_attrset (table, pivot_value_new_text (N_("(dataset)")),
703 dict_attrset, flags);
704 for (size_t i = 0; i < n_vars; i++)
705 display_attrset (table, pivot_value_new_variable (vars[i]),
706 var_get_attributes (vars[i]), flags);
707
708 if (pivot_table_is_empty (table))
709 pivot_table_unref (table);
710 else
711 pivot_table_submit (table);
712 }
713
714 /* Display a list of vectors. If SORTED is nonzero then they are
715 sorted alphabetically. */
716 static void
display_vectors(const struct dictionary * dict,int sorted)717 display_vectors (const struct dictionary *dict, int sorted)
718 {
719 size_t n_vectors = dict_get_vector_cnt (dict);
720 if (n_vectors == 0)
721 {
722 msg (SW, _("No vectors defined."));
723 return;
724 }
725
726 const struct vector **vectors = xnmalloc (n_vectors, sizeof *vectors);
727 for (size_t i = 0; i < n_vectors; i++)
728 vectors[i] = dict_get_vector (dict, i);
729 if (sorted)
730 qsort (vectors, n_vectors, sizeof *vectors, compare_vector_ptrs_by_name);
731
732 struct pivot_table *table = pivot_table_create (N_("Vectors"));
733 pivot_dimension_create (table, PIVOT_AXIS_COLUMN, N_("Attributes"),
734 N_("Variable"), N_("Print Format"));
735 struct pivot_dimension *vector_dim = pivot_dimension_create (
736 table, PIVOT_AXIS_ROW, N_("Vector and Position"));
737 vector_dim->root->show_label = true;
738
739 for (size_t i = 0; i < n_vectors; i++)
740 {
741 const struct vector *vec = vectors[i];
742
743 struct pivot_category *group = pivot_category_create_group__ (
744 vector_dim->root, pivot_value_new_user_text (
745 vector_get_name (vectors[i]), -1));
746
747 for (size_t j = 0; j < vector_get_var_cnt (vec); j++)
748 {
749 struct variable *var = vector_get_var (vec, j);
750
751 int row = pivot_category_create_leaf (
752 group, pivot_value_new_integer (j + 1));
753
754 pivot_table_put2 (table, 0, row, pivot_value_new_variable (var));
755 char fmt_string[FMT_STRING_LEN_MAX + 1];
756 fmt_to_string (var_get_print_format (var), fmt_string);
757 pivot_table_put2 (table, 1, row,
758 pivot_value_new_user_text (fmt_string, -1));
759 }
760 }
761
762 pivot_table_submit (table);
763
764 free (vectors);
765 }
766
767 /* Encoding analysis. */
768
769 static const char *encoding_names[] = {
770 /* These encodings are from http://encoding.spec.whatwg.org/, as retrieved
771 February 2014. Encodings not supported by glibc and encodings relevant
772 only to HTML have been removed. */
773 "utf-8",
774 "windows-1252",
775 "iso-8859-2",
776 "iso-8859-3",
777 "iso-8859-4",
778 "iso-8859-5",
779 "iso-8859-6",
780 "iso-8859-7",
781 "iso-8859-8",
782 "iso-8859-10",
783 "iso-8859-13",
784 "iso-8859-14",
785 "iso-8859-16",
786 "macintosh",
787 "windows-874",
788 "windows-1250",
789 "windows-1251",
790 "windows-1253",
791 "windows-1254",
792 "windows-1255",
793 "windows-1256",
794 "windows-1257",
795 "windows-1258",
796 "koi8-r",
797 "koi8-u",
798 "ibm866",
799 "gb18030",
800 "big5",
801 "euc-jp",
802 "iso-2022-jp",
803 "shift_jis",
804 "euc-kr",
805
806 /* Added by user request. */
807 "ibm850",
808 "din_66003",
809 };
810 #define N_ENCODING_NAMES (sizeof encoding_names / sizeof *encoding_names)
811
812 struct encoding
813 {
814 uint64_t encodings;
815 char **utf8_strings;
816 unsigned int hash;
817 };
818
819 static char **
recode_strings(struct pool * pool,char ** strings,bool * ids,size_t n,const char * encoding)820 recode_strings (struct pool *pool,
821 char **strings, bool *ids, size_t n,
822 const char *encoding)
823 {
824 char **utf8_strings;
825 size_t i;
826
827 utf8_strings = pool_alloc (pool, n * sizeof *utf8_strings);
828 for (i = 0; i < n; i++)
829 {
830 struct substring utf8;
831 int error;
832
833 error = recode_pedantically ("UTF-8", encoding, ss_cstr (strings[i]),
834 pool, &utf8);
835 if (!error)
836 {
837 ss_rtrim (&utf8, ss_cstr (" "));
838 utf8.string[utf8.length] = '\0';
839
840 if (ids[i] && !id_is_plausible (utf8.string, false))
841 error = EINVAL;
842 }
843
844 if (error)
845 return NULL;
846
847 utf8_strings[i] = utf8.string;
848 }
849
850 return utf8_strings;
851 }
852
853 static struct encoding *
find_duplicate_encoding(struct encoding * encodings,size_t n_encodings,char ** utf8_strings,size_t n_strings,unsigned int hash)854 find_duplicate_encoding (struct encoding *encodings, size_t n_encodings,
855 char **utf8_strings, size_t n_strings,
856 unsigned int hash)
857 {
858 struct encoding *e;
859
860 for (e = encodings; e < &encodings[n_encodings]; e++)
861 {
862 int i;
863
864 if (e->hash != hash)
865 goto next_encoding;
866
867 for (i = 0; i < n_strings; i++)
868 if (strcmp (utf8_strings[i], e->utf8_strings[i]))
869 goto next_encoding;
870
871 return e;
872 next_encoding:;
873 }
874
875 return NULL;
876 }
877
878 static bool
all_equal(const struct encoding * encodings,size_t n_encodings,size_t string_idx)879 all_equal (const struct encoding *encodings, size_t n_encodings,
880 size_t string_idx)
881 {
882 const char *s0;
883 size_t i;
884
885 s0 = encodings[0].utf8_strings[string_idx];
886 for (i = 1; i < n_encodings; i++)
887 if (strcmp (s0, encodings[i].utf8_strings[string_idx]))
888 return false;
889
890 return true;
891 }
892
893 static int
equal_prefix(const struct encoding * encodings,size_t n_encodings,size_t string_idx)894 equal_prefix (const struct encoding *encodings, size_t n_encodings,
895 size_t string_idx)
896 {
897 const char *s0;
898 size_t prefix;
899 size_t i;
900
901 s0 = encodings[0].utf8_strings[string_idx];
902 prefix = strlen (s0);
903 for (i = 1; i < n_encodings; i++)
904 {
905 const char *si = encodings[i].utf8_strings[string_idx];
906 size_t j;
907
908 for (j = 0; j < prefix; j++)
909 if (s0[j] != si[j])
910 {
911 prefix = j;
912 if (!prefix)
913 return 0;
914 break;
915 }
916 }
917
918 while (prefix > 0 && s0[prefix - 1] != ' ')
919 prefix--;
920 return prefix;
921 }
922
923 static int
equal_suffix(const struct encoding * encodings,size_t n_encodings,size_t string_idx)924 equal_suffix (const struct encoding *encodings, size_t n_encodings,
925 size_t string_idx)
926 {
927 const char *s0;
928 size_t s0_len;
929 size_t suffix;
930 size_t i;
931
932 s0 = encodings[0].utf8_strings[string_idx];
933 s0_len = strlen (s0);
934 suffix = s0_len;
935 for (i = 1; i < n_encodings; i++)
936 {
937 const char *si = encodings[i].utf8_strings[string_idx];
938 size_t si_len = strlen (si);
939 size_t j;
940
941 if (si_len < suffix)
942 suffix = si_len;
943 for (j = 0; j < suffix; j++)
944 if (s0[s0_len - j - 1] != si[si_len - j - 1])
945 {
946 suffix = j;
947 if (!suffix)
948 return 0;
949 break;
950 }
951 }
952
953 while (suffix > 0 && s0[s0_len - suffix] != ' ')
954 suffix--;
955 return suffix;
956 }
957
958 static void
report_encodings(const struct file_handle * h,struct pool * pool,char ** titles,bool * ids,char ** strings,size_t n_strings)959 report_encodings (const struct file_handle *h, struct pool *pool,
960 char **titles, bool *ids, char **strings, size_t n_strings)
961 {
962 struct encoding encodings[N_ENCODING_NAMES];
963 size_t n_encodings, n_unique_strings;
964
965 n_encodings = 0;
966 for (size_t i = 0; i < N_ENCODING_NAMES; i++)
967 {
968 char **utf8_strings;
969 struct encoding *e;
970 unsigned int hash;
971
972 utf8_strings = recode_strings (pool, strings, ids, n_strings,
973 encoding_names[i]);
974 if (!utf8_strings)
975 continue;
976
977 /* Hash utf8_strings. */
978 hash = 0;
979 for (size_t j = 0; j < n_strings; j++)
980 hash = hash_string (utf8_strings[j], hash);
981
982 /* If there's a duplicate encoding, just mark it. */
983 e = find_duplicate_encoding (encodings, n_encodings,
984 utf8_strings, n_strings, hash);
985 if (e)
986 {
987 e->encodings |= UINT64_C (1) << i;
988 continue;
989 }
990
991 e = &encodings[n_encodings++];
992 e->encodings = UINT64_C (1) << i;
993 e->utf8_strings = utf8_strings;
994 e->hash = hash;
995 }
996 if (!n_encodings)
997 {
998 msg (SW, _("No valid encodings found."));
999 return;
1000 }
1001
1002 /* Table of valid encodings. */
1003 struct pivot_table *table = pivot_table_create__ (
1004 pivot_value_new_text_format (N_("Usable encodings for %s."),
1005 fh_get_name (h)), "Usable Encodings");
1006 table->caption = pivot_value_new_text_format (
1007 N_("Encodings that can successfully read %s (by specifying the encoding "
1008 "name on the GET command's ENCODING subcommand). Encodings that "
1009 "yield identical text are listed together."),
1010 fh_get_name (h));
1011
1012 pivot_dimension_create (table, PIVOT_AXIS_COLUMN, N_("Encodings"),
1013 N_("Encodings"));
1014 struct pivot_dimension *number = pivot_dimension_create__ (
1015 table, PIVOT_AXIS_ROW, pivot_value_new_user_text ("#", -1));
1016 number->root->show_label = true;
1017
1018 for (size_t i = 0; i < n_encodings; i++)
1019 {
1020 struct string s = DS_EMPTY_INITIALIZER;
1021 for (size_t j = 0; j < 64; j++)
1022 if (encodings[i].encodings & (UINT64_C (1) << j))
1023 ds_put_format (&s, "%s, ", encoding_names[j]);
1024 ds_chomp (&s, ss_cstr (", "));
1025
1026 int row = pivot_category_create_leaf (number->root,
1027 pivot_value_new_integer (i + 1));
1028 pivot_table_put2 (
1029 table, 0, row, pivot_value_new_user_text_nocopy (ds_steal_cstr (&s)));
1030 }
1031 pivot_table_submit (table);
1032
1033 n_unique_strings = 0;
1034 for (size_t i = 0; i < n_strings; i++)
1035 if (!all_equal (encodings, n_encodings, i))
1036 n_unique_strings++;
1037 if (!n_unique_strings)
1038 return;
1039
1040 /* Table of alternative interpretations. */
1041 table = pivot_table_create__ (
1042 pivot_value_new_text_format (N_("%s Encoded Text Strings"),
1043 fh_get_name (h)),
1044 "Alternate Encoded Text Strings");
1045 table->caption = pivot_value_new_text (
1046 N_("Text strings in the file dictionary that the previously listed "
1047 "encodings interpret differently, along with the interpretations."));
1048
1049 pivot_dimension_create (table, PIVOT_AXIS_COLUMN, N_("Text"), N_("Text"));
1050
1051 number = pivot_dimension_create__ (table, PIVOT_AXIS_ROW,
1052 pivot_value_new_user_text ("#", -1));
1053 number->root->show_label = true;
1054 for (size_t i = 0; i < n_encodings; i++)
1055 pivot_category_create_leaf (number->root,
1056 pivot_value_new_integer (i + 1));
1057
1058 struct pivot_dimension *purpose = pivot_dimension_create (
1059 table, PIVOT_AXIS_ROW, N_("Purpose"));
1060 purpose->root->show_label = true;
1061
1062 for (size_t i = 0; i < n_strings; i++)
1063 if (!all_equal (encodings, n_encodings, i))
1064 {
1065 int prefix = equal_prefix (encodings, n_encodings, i);
1066 int suffix = equal_suffix (encodings, n_encodings, i);
1067
1068 int purpose_idx = pivot_category_create_leaf (
1069 purpose->root, pivot_value_new_user_text (titles[i], -1));
1070
1071 for (size_t j = 0; j < n_encodings; j++)
1072 {
1073 const char *s = encodings[j].utf8_strings[i] + prefix;
1074
1075 if (prefix || suffix)
1076 {
1077 size_t len = strlen (s) - suffix;
1078 struct string entry;
1079
1080 ds_init_empty (&entry);
1081 if (prefix)
1082 ds_put_cstr (&entry, "...");
1083 ds_put_substring (&entry, ss_buffer (s, len));
1084 if (suffix)
1085 ds_put_cstr (&entry, "...");
1086
1087 pivot_table_put3 (table, 0, j, purpose_idx,
1088 pivot_value_new_user_text_nocopy (
1089 ds_steal_cstr (&entry)));
1090 }
1091 else
1092 pivot_table_put3 (table, 0, j, purpose_idx,
1093 pivot_value_new_user_text (s, -1));
1094 }
1095 }
1096
1097 pivot_table_submit (table);
1098 }
1099