1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012,
3 2013, 2015, 2016 Free Software Foundation, Inc.
4
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, either version 3 of the License, or
8 (at your option) any later version.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>. */
17
18 #include <config.h>
19
20 #include <stdlib.h>
21
22 #include <string.h>
23
24 #include "data/dataset.h"
25 #include "data/dictionary.h"
26 #include "data/format.h"
27 #include "data/gnumeric-reader.h"
28 #include "data/ods-reader.h"
29 #include "data/spreadsheet-reader.h"
30 #include "data/psql-reader.h"
31 #include "data/settings.h"
32 #include "language/command.h"
33 #include "language/data-io/data-parser.h"
34 #include "language/data-io/data-reader.h"
35 #include "language/data-io/file-handle.h"
36 #include "language/data-io/placement-parser.h"
37 #include "language/lexer/format-parser.h"
38 #include "language/lexer/lexer.h"
39 #include "libpspp/cast.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
42
43 #include "gl/xalloc.h"
44
45 #include "gettext.h"
46 #define _(msgid) gettext (msgid)
47 #define N_(msgid) (msgid)
48
49 static bool parse_spreadsheet (struct lexer *lexer, char **filename,
50 struct spreadsheet_read_options *opts);
51
52 static void destroy_spreadsheet_read_info (struct spreadsheet_read_options *);
53
54 static int parse_get_txt (struct lexer *lexer, struct dataset *);
55 static int parse_get_psql (struct lexer *lexer, struct dataset *);
56
57 int
cmd_get_data(struct lexer * lexer,struct dataset * ds)58 cmd_get_data (struct lexer *lexer, struct dataset *ds)
59 {
60 char *tok = NULL;
61 struct spreadsheet_read_options opts;
62
63 opts.sheet_name = NULL;
64 opts.sheet_index = -1;
65 opts.cell_range = NULL;
66 opts.read_names = false;
67 opts.asw = -1;
68
69 if (! lex_force_match (lexer, T_SLASH))
70 goto error;
71
72 if (!lex_force_match_id (lexer, "TYPE"))
73 goto error;
74
75 if (!lex_force_match (lexer, T_EQUALS))
76 goto error;
77
78 const char *s = lex_tokcstr (lexer);
79
80 if (s)
81 tok = strdup (s);
82
83 if (lex_match_id (lexer, "TXT"))
84 {
85 free (tok);
86 return parse_get_txt (lexer, ds);
87 }
88 else if (lex_match_id (lexer, "PSQL"))
89 {
90 free (tok);
91 return parse_get_psql (lexer, ds);
92 }
93 else if (lex_match_id (lexer, "GNM") ||
94 lex_match_id (lexer, "ODS"))
95 {
96 char *filename = NULL;
97 if (!parse_spreadsheet (lexer, &filename, &opts))
98 goto error;
99
100 struct spreadsheet *spreadsheet = NULL;
101 if (0 == strncasecmp (tok, "GNM", 3))
102 spreadsheet = gnumeric_probe (filename, true);
103 else if (0 == strncasecmp (tok, "ODS", 3))
104 spreadsheet = ods_probe (filename, true);
105
106 if (spreadsheet == NULL)
107 {
108 msg (SE, _("error reading file `%s'"), filename);
109 free (filename);
110 goto error;
111 }
112 free (filename);
113
114 struct casereader *reader = spreadsheet_make_reader (spreadsheet, &opts);
115 if (reader)
116 {
117 dataset_set_dict (ds, dict_clone (spreadsheet->dict));
118 dataset_set_source (ds, reader);
119 free (tok);
120 destroy_spreadsheet_read_info (&opts);
121 spreadsheet_unref (spreadsheet);
122 return CMD_SUCCESS;
123 }
124 spreadsheet_unref (spreadsheet);
125 }
126 else
127 msg (SE, _("Unsupported TYPE %s."), tok);
128
129
130 error:
131 destroy_spreadsheet_read_info (&opts);
132 free (tok);
133 return CMD_FAILURE;
134 }
135
136 static int
parse_get_psql(struct lexer * lexer,struct dataset * ds)137 parse_get_psql (struct lexer *lexer, struct dataset *ds)
138 {
139 struct psql_read_info psql;
140 psql.allow_clear = false;
141 psql.conninfo = NULL;
142 psql.str_width = -1;
143 psql.bsize = -1;
144 ds_init_empty (&psql.sql);
145
146 if (! lex_force_match (lexer, T_SLASH))
147 goto error;
148
149 if (!lex_force_match_id (lexer, "CONNECT"))
150 goto error;
151
152 if (! lex_force_match (lexer, T_EQUALS))
153 goto error;
154
155 if (!lex_force_string (lexer))
156 goto error;
157
158 psql.conninfo = ss_xstrdup (lex_tokss (lexer));
159
160 lex_get (lexer);
161
162 while (lex_match (lexer, T_SLASH))
163 {
164 if (lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
165 {
166 lex_match (lexer, T_EQUALS);
167 if (lex_force_int (lexer))
168 {
169 psql.str_width = lex_integer (lexer);
170 lex_get (lexer);
171 }
172 }
173 else if (lex_match_id (lexer, "BSIZE"))
174 {
175 lex_match (lexer, T_EQUALS);
176 if (lex_force_int (lexer))
177 {
178 psql.bsize = lex_integer (lexer);
179 lex_get (lexer);
180 }
181 }
182 else if (lex_match_id (lexer, "UNENCRYPTED"))
183 {
184 psql.allow_clear = true;
185 }
186 else if (lex_match_id (lexer, "SQL"))
187 {
188 lex_match (lexer, T_EQUALS);
189 if (! lex_force_string (lexer))
190 goto error;
191
192 ds_put_substring (&psql.sql, lex_tokss (lexer));
193 lex_get (lexer);
194 }
195 }
196 {
197 struct dictionary *dict = NULL;
198 struct casereader *reader = psql_open_reader (&psql, &dict);
199
200 if (reader)
201 {
202 dataset_set_dict (ds, dict);
203 dataset_set_source (ds, reader);
204 }
205 }
206
207 ds_destroy (&psql.sql);
208 free (psql.conninfo);
209
210 return CMD_SUCCESS;
211
212 error:
213
214 ds_destroy (&psql.sql);
215 free (psql.conninfo);
216
217 return CMD_FAILURE;
218 }
219
220 static bool
parse_spreadsheet(struct lexer * lexer,char ** filename,struct spreadsheet_read_options * opts)221 parse_spreadsheet (struct lexer *lexer, char **filename,
222 struct spreadsheet_read_options *opts)
223 {
224 opts->sheet_index = 1;
225 opts->sheet_name = NULL;
226 opts->cell_range = NULL;
227 opts->read_names = true;
228 opts->asw = -1;
229
230 if (! lex_force_match (lexer, T_SLASH))
231 goto error;
232
233 if (!lex_force_match_id (lexer, "FILE"))
234 goto error;
235
236 if (! lex_force_match (lexer, T_EQUALS))
237 goto error;
238
239 if (!lex_force_string (lexer))
240 goto error;
241
242 *filename = utf8_to_filename (lex_tokcstr (lexer));
243
244 lex_get (lexer);
245
246 while (lex_match (lexer, T_SLASH))
247 {
248 if (lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
249 {
250 lex_match (lexer, T_EQUALS);
251 if (lex_force_int (lexer))
252 {
253 opts->asw = lex_integer (lexer);
254 lex_get (lexer);
255 }
256 }
257 else if (lex_match_id (lexer, "SHEET"))
258 {
259 lex_match (lexer, T_EQUALS);
260 if (lex_match_id (lexer, "NAME"))
261 {
262 if (! lex_force_string (lexer))
263 goto error;
264
265 opts->sheet_name = ss_xstrdup (lex_tokss (lexer));
266 opts->sheet_index = -1;
267
268 lex_get (lexer);
269 }
270 else if (lex_match_id (lexer, "INDEX"))
271 {
272 if (!lex_force_int (lexer))
273 goto error;
274
275 opts->sheet_index = lex_integer (lexer);
276 if (opts->sheet_index <= 0)
277 {
278 msg (SE, _("The sheet index must be greater than or equal to 1"));
279 goto error;
280 }
281 lex_get (lexer);
282 }
283 else
284 {
285 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
286 "/SHEET", "NAME", "INDEX");
287 goto error;
288 }
289 }
290 else if (lex_match_id (lexer, "CELLRANGE"))
291 {
292 lex_match (lexer, T_EQUALS);
293
294 if (lex_match_id (lexer, "FULL"))
295 {
296 opts->cell_range = NULL;
297 }
298 else if (lex_match_id (lexer, "RANGE"))
299 {
300 if (! lex_force_string (lexer))
301 goto error;
302
303 opts->cell_range = ss_xstrdup (lex_tokss (lexer));
304 lex_get (lexer);
305 }
306 else
307 {
308 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
309 "/CELLRANGE", "FULL", "RANGE");
310 goto error;
311 }
312 }
313 else if (lex_match_id (lexer, "READNAMES"))
314 {
315 lex_match (lexer, T_EQUALS);
316
317 if (lex_match_id (lexer, "ON"))
318 {
319 opts->read_names = true;
320 }
321 else if (lex_match_id (lexer, "OFF"))
322 {
323 opts->read_names = false;
324 }
325 else
326 {
327 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
328 "/READNAMES", "ON", "OFF");
329 goto error;
330 }
331 }
332 else
333 {
334 lex_error (lexer, NULL);
335 goto error;
336 }
337 }
338
339 return true;
340
341 error:
342 return false;
343 }
344
345
346 static bool
set_type(struct data_parser * parser,const char * subcommand,enum data_parser_type type,bool * has_type)347 set_type (struct data_parser *parser, const char *subcommand,
348 enum data_parser_type type, bool *has_type)
349 {
350 if (!*has_type)
351 {
352 data_parser_set_type (parser, type);
353 *has_type = true;
354 }
355 else if (type != data_parser_get_type (parser))
356 {
357 msg (SE, _("%s is allowed only with %s arrangement, but %s arrangement "
358 "was stated or implied earlier in this command."),
359 subcommand,
360 type == DP_FIXED ? "FIXED" : "DELIMITED",
361 type == DP_FIXED ? "DELIMITED" : "FIXED");
362 return false;
363 }
364 return true;
365 }
366
367 static int
parse_get_txt(struct lexer * lexer,struct dataset * ds)368 parse_get_txt (struct lexer *lexer, struct dataset *ds)
369 {
370 struct data_parser *parser = NULL;
371 struct dictionary *dict = dict_create (get_default_encoding ());
372 struct file_handle *fh = NULL;
373 struct dfm_reader *reader = NULL;
374 char *encoding = NULL;
375 char *name = NULL;
376
377 int record;
378 enum data_parser_type type;
379 bool has_type;
380
381 if (! lex_force_match (lexer, T_SLASH))
382 goto error;
383
384 if (!lex_force_match_id (lexer, "FILE"))
385 goto error;
386 if (! lex_force_match (lexer, T_EQUALS))
387 goto error;
388 fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE, NULL);
389 if (fh == NULL)
390 goto error;
391
392 parser = data_parser_create (dict);
393 has_type = false;
394 data_parser_set_type (parser, DP_DELIMITED);
395 data_parser_set_span (parser, false);
396 data_parser_set_quotes (parser, ss_empty ());
397 data_parser_set_quote_escape (parser, true);
398 data_parser_set_empty_line_has_field (parser, true);
399
400 for (;;)
401 {
402 if (!lex_force_match (lexer, T_SLASH))
403 goto error;
404
405 if (lex_match_id (lexer, "ENCODING"))
406 {
407 lex_match (lexer, T_EQUALS);
408 if (!lex_force_string (lexer))
409 goto error;
410
411 free (encoding);
412 encoding = ss_xstrdup (lex_tokss (lexer));
413
414 lex_get (lexer);
415 }
416 else if (lex_match_id (lexer, "ARRANGEMENT"))
417 {
418 bool ok;
419
420 lex_match (lexer, T_EQUALS);
421 if (lex_match_id (lexer, "FIXED"))
422 ok = set_type (parser, "ARRANGEMENT=FIXED", DP_FIXED, &has_type);
423 else if (lex_match_id (lexer, "DELIMITED"))
424 ok = set_type (parser, "ARRANGEMENT=DELIMITED",
425 DP_DELIMITED, &has_type);
426 else
427 {
428 lex_error_expecting (lexer, "FIXED", "DELIMITED");
429 goto error;
430 }
431 if (!ok)
432 goto error;
433 }
434 else if (lex_match_id (lexer, "FIRSTCASE"))
435 {
436 lex_match (lexer, T_EQUALS);
437 if (!lex_force_int (lexer))
438 goto error;
439 if (lex_integer (lexer) < 1)
440 {
441 msg (SE, _("Value of %s must be 1 or greater."), "FIRSTCASE");
442 goto error;
443 }
444 data_parser_set_skip (parser, lex_integer (lexer) - 1);
445 lex_get (lexer);
446 }
447 else if (lex_match_id_n (lexer, "DELCASE", 4))
448 {
449 if (!set_type (parser, "DELCASE", DP_DELIMITED, &has_type))
450 goto error;
451 lex_match (lexer, T_EQUALS);
452 if (lex_match_id (lexer, "LINE"))
453 data_parser_set_span (parser, false);
454 else if (lex_match_id (lexer, "VARIABLES"))
455 {
456 data_parser_set_span (parser, true);
457
458 /* VARIABLES takes an integer argument, but for no
459 good reason. We just ignore it. */
460 if (!lex_force_int (lexer))
461 goto error;
462 lex_get (lexer);
463 }
464 else
465 {
466 lex_error_expecting (lexer, "LINE", "VARIABLES");
467 goto error;
468 }
469 }
470 else if (lex_match_id (lexer, "FIXCASE"))
471 {
472 if (!set_type (parser, "FIXCASE", DP_FIXED, &has_type))
473 goto error;
474 lex_match (lexer, T_EQUALS);
475 if (!lex_force_int (lexer))
476 goto error;
477 if (lex_integer (lexer) < 1)
478 {
479 msg (SE, _("Value of %s must be 1 or greater."), "FIXCASE");
480 goto error;
481 }
482 data_parser_set_records (parser, lex_integer (lexer));
483 lex_get (lexer);
484 }
485 else if (lex_match_id (lexer, "IMPORTCASES"))
486 {
487 lex_match (lexer, T_EQUALS);
488 if (lex_match (lexer, T_ALL))
489 {
490 /* Nothing to do. */
491 }
492 else if (lex_match_id (lexer, "FIRST"))
493 {
494 if (!lex_force_int (lexer))
495 goto error;
496 lex_get (lexer);
497 }
498 else if (lex_match_id (lexer, "PERCENT"))
499 {
500 if (!lex_force_int (lexer))
501 goto error;
502 lex_get (lexer);
503 }
504 msg (SW, _("Ignoring obsolete IMPORTCASES subcommand. (N OF CASES "
505 "or SAMPLE may be used to substitute.)"));
506 }
507 else if (lex_match_id_n (lexer, "DELIMITERS", 4))
508 {
509 struct string hard_seps = DS_EMPTY_INITIALIZER;
510 const char *soft_seps = "";
511 struct substring s;
512 int c;
513
514 if (!set_type (parser, "DELIMITERS", DP_DELIMITED, &has_type))
515 goto error;
516 lex_match (lexer, T_EQUALS);
517
518 if (!lex_force_string (lexer))
519 goto error;
520
521 /* XXX should support multibyte UTF-8 characters */
522 s = lex_tokss (lexer);
523 if (ss_match_string (&s, ss_cstr ("\\t")))
524 ds_put_cstr (&hard_seps, "\t");
525 if (ss_match_string (&s, ss_cstr ("\\\\")))
526 ds_put_cstr (&hard_seps, "\\");
527 while ((c = ss_get_byte (&s)) != EOF)
528 if (c == ' ')
529 soft_seps = " ";
530 else
531 ds_put_byte (&hard_seps, c);
532 data_parser_set_soft_delimiters (parser, ss_cstr (soft_seps));
533 data_parser_set_hard_delimiters (parser, ds_ss (&hard_seps));
534 ds_destroy (&hard_seps);
535
536 lex_get (lexer);
537 }
538 else if (lex_match_id (lexer, "QUALIFIERS"))
539 {
540 if (!set_type (parser, "QUALIFIERS", DP_DELIMITED, &has_type))
541 goto error;
542 lex_match (lexer, T_EQUALS);
543
544 if (!lex_force_string (lexer))
545 goto error;
546
547 /* XXX should support multibyte UTF-8 characters */
548 if (settings_get_syntax () == COMPATIBLE
549 && ss_length (lex_tokss (lexer)) != 1)
550 {
551 msg (SE, _("In compatible syntax mode, the QUALIFIER string "
552 "must contain exactly one character."));
553 goto error;
554 }
555
556 data_parser_set_quotes (parser, lex_tokss (lexer));
557 lex_get (lexer);
558 }
559 else if (lex_match_id (lexer, "VARIABLES"))
560 break;
561 else
562 {
563 lex_error_expecting (lexer, "VARIABLES");
564 goto error;
565 }
566 }
567 lex_match (lexer, T_EQUALS);
568
569 record = 1;
570 type = data_parser_get_type (parser);
571 do
572 {
573 struct fmt_spec input, output;
574 struct variable *v;
575 int fc, lc;
576
577 while (type == DP_FIXED && lex_match (lexer, T_SLASH))
578 {
579 if (!lex_force_int (lexer))
580 goto error;
581 if (lex_integer (lexer) < record)
582 {
583 msg (SE, _("The record number specified, %ld, is at or "
584 "before the previous record, %d. Data "
585 "fields must be listed in order of "
586 "increasing record number."),
587 lex_integer (lexer), record);
588 goto error;
589 }
590 if (lex_integer (lexer) > data_parser_get_records (parser))
591 {
592 msg (SE, _("The record number specified, %ld, exceeds "
593 "the number of records per case specified "
594 "on FIXCASE, %d."),
595 lex_integer (lexer), data_parser_get_records (parser));
596 goto error;
597 }
598 record = lex_integer (lexer);
599 lex_get (lexer);
600 }
601
602 const char * tstr = lex_tokcstr (lexer);
603 if (tstr == NULL)
604 {
605 lex_error (lexer, NULL);
606 goto error;
607 }
608 name = xstrdup (tstr);
609 if (!lex_force_id (lexer)
610 || !dict_id_is_valid (dict, name, true))
611 {
612 goto error;
613 }
614 lex_get (lexer);
615 if (type == DP_DELIMITED)
616 {
617 if (!parse_format_specifier (lexer, &input)
618 || !fmt_check_input (&input))
619 {
620 goto error;
621 }
622 output = fmt_for_output_from_input (&input);
623 }
624 else
625 {
626 char fmt_type_name[FMT_TYPE_LEN_MAX + 1];
627 enum fmt_type fmt_type;
628 int w, d;
629
630 if (!parse_column_range (lexer, 0, &fc, &lc, NULL))
631 goto error;
632
633 /* Accept a format (e.g. F8.2) or just a type name (e.g. DOLLAR). */
634 if (!parse_abstract_format_specifier (lexer, fmt_type_name, &w, &d))
635 goto error;
636 if (!fmt_from_name (fmt_type_name, &fmt_type))
637 {
638 msg (SE, _("Unknown format type `%s'."), fmt_type_name);
639 goto error;
640 }
641 /* Compose input format. */
642 input.type = fmt_type;
643 input.w = lc - fc + 1;
644 input.d = 0;
645 if (!fmt_check_input (&input))
646 goto error;
647 /* Compose output format. */
648 if (w != 0)
649 {
650 output.type = fmt_type;
651 output.w = w;
652 output.d = d;
653 if (!fmt_check_output (&output))
654 goto error;
655 }
656 else
657 output = fmt_for_output_from_input (&input);
658 }
659 v = dict_create_var (dict, name, fmt_var_width (&input));
660 if (v == NULL)
661 {
662 msg (SE, _("%s is a duplicate variable name."), name);
663 goto error;
664 }
665 var_set_both_formats (v, &output);
666 if (type == DP_DELIMITED)
667 data_parser_add_delimited_field (parser, &input,
668 var_get_case_index (v),
669 name);
670 else
671 data_parser_add_fixed_field (parser, &input, var_get_case_index (v),
672 name, record, fc);
673 free (name);
674 name = NULL;
675 }
676 while (lex_token (lexer) != T_ENDCMD);
677
678 reader = dfm_open_reader (fh, lexer, encoding);
679 if (reader == NULL)
680 goto error;
681
682 data_parser_make_active_file (parser, ds, reader, dict, NULL, NULL);
683 fh_unref (fh);
684 free (encoding);
685 return CMD_SUCCESS;
686
687 error:
688 data_parser_destroy (parser);
689 dict_unref (dict);
690 fh_unref (fh);
691 free (name);
692 free (encoding);
693 return CMD_CASCADING_FAILURE;
694 }
695
696
697 static void
destroy_spreadsheet_read_info(struct spreadsheet_read_options * opts)698 destroy_spreadsheet_read_info (struct spreadsheet_read_options *opts)
699 {
700 free (opts->cell_range);
701 free (opts->sheet_name);
702 }
703