1 /********************************************************************\
2  * gnc-import-price.cpp - import prices from csv files              *
3  *                                                                  *
4  * This program is free software; you can redistribute it and/or    *
5  * modify it under the terms of the GNU General Public License as   *
6  * published by the Free Software Foundation; either version 2 of   *
7  * the License, or (at your option) any later version.              *
8  *                                                                  *
9  * This program is distributed in the hope that it will be useful,  *
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of   *
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the    *
12  * GNU General Public License for more details.                     *
13  *                                                                  *
14  * You should have received a copy of the GNU General Public License*
15  * along with this program; if not, contact:                        *
16  *                                                                  *
17  * Free Software Foundation           Voice:  +1-617-542-5942       *
18  * 51 Franklin Street, Fifth Floor    Fax:    +1-617-542-2652       *
19  * Boston, MA  02110-1301,  USA       gnu@gnu.org                   *
20  *                                                                  *
21 \********************************************************************/
22 
23 #include <guid.hpp>
24 
25 extern "C" {
26 #include <platform.h>
27 #if PLATFORM(WINDOWS)
28 #include <windows.h>
29 #endif
30 
31 #include <glib/gi18n.h>
32 
33 #include "gnc-ui-util.h" //get book
34 #include "gnc-commodity.h"
35 #include "gnc-pricedb.h"
36 }
37 
38 #include <algorithm>
39 #include <exception>
40 #include <iostream>
41 #include <memory>
42 #include <string>
43 #include <tuple>
44 #include <vector>
45 
46 #include <boost/regex.hpp>
47 #include <boost/regex/icu.hpp>
48 #include <boost/optional.hpp>
49 
50 #include "gnc-import-price.hpp"
51 #include "gnc-imp-props-price.hpp"
52 #include "gnc-tokenizer-csv.hpp"
53 #include "gnc-tokenizer-fw.hpp"
54 #include "gnc-imp-settings-csv-price.hpp"
55 
56 G_GNUC_UNUSED static QofLogModule log_module = GNC_MOD_IMPORT;
57 
58 const int num_currency_formats_price = 3;
59 const gchar* currency_format_user_price[] = {N_("Locale"),
60                                        N_("Period: 123,456.78"),
61                                        N_("Comma: 123.456,78")
62                                       };
63 
64 
65 /** Constructor for GncPriceImport.
66  */
GncPriceImport(GncImpFileFormat format)67 GncPriceImport::GncPriceImport(GncImpFileFormat format)
68 {
69     /* All of the data pointers are initially NULL. This is so that, if
70      * gnc_csv_parse_data_free is called before all of the data is
71      * initialized, only the data that needs to be freed is freed. */
72     m_skip_errors = false;
73     file_format(m_settings.m_file_format = format);
74 }
75 
76 /** Destructor for GncPriceImport.
77  */
~GncPriceImport()78 GncPriceImport::~GncPriceImport()
79 {
80 }
81 
82 /** Sets the file format for the file to import, which
83  *  may cause the file to be reloaded as well if the
84  *  previously set file format was different and a
85  *  filename was already set.
86  *  @param format the new format to set
87  *  @exception std::ifstream::failure if file reloading fails
88  */
file_format(GncImpFileFormat format)89 void GncPriceImport::file_format(GncImpFileFormat format)
90 {
91     if (m_tokenizer && m_settings.m_file_format == format)
92         return;
93 
94     auto new_encoding = std::string("UTF-8");
95     auto new_imp_file = std::string();
96 
97     // Recover common settings from old tokenizer
98     if (m_tokenizer)
99     {
100         new_encoding = m_tokenizer->encoding();
101         new_imp_file = m_tokenizer->current_file();
102         if (file_format() == GncImpFileFormat::FIXED_WIDTH)
103         {
104             auto fwtok = dynamic_cast<GncFwTokenizer*>(m_tokenizer.get());
105             if (!fwtok->get_columns().empty())
106                 m_settings.m_column_widths = fwtok->get_columns();
107         }
108     }
109 
110     m_settings.m_file_format = format;
111     m_tokenizer = gnc_tokenizer_factory(m_settings.m_file_format);
112 
113     // Set up new tokenizer with common settings
114     // recovered from old tokenizer
115     m_tokenizer->encoding(new_encoding);
116     load_file(new_imp_file);
117 
118     // Restore potentially previously set separators or column_widths
119     if ((file_format() == GncImpFileFormat::CSV)
120         && !m_settings.m_separators.empty())
121         separators (m_settings.m_separators);
122     else if ((file_format() == GncImpFileFormat::FIXED_WIDTH)
123         && !m_settings.m_column_widths.empty())
124     {
125         auto fwtok = dynamic_cast<GncFwTokenizer*>(m_tokenizer.get());
126         fwtok->columns (m_settings.m_column_widths);
127     }
128 }
129 
file_format()130 GncImpFileFormat GncPriceImport::file_format()
131 {
132     return m_settings.m_file_format;
133 }
134 
over_write(bool over)135 void GncPriceImport::over_write (bool over)
136 {
137     m_over_write = over;
138 }
over_write()139 bool GncPriceImport::over_write () { return m_over_write; }
140 
141 /** Sets a from commodity. This is the commodity all import data relates to.
142  *  When a from commodity is set, there can't be any from columns selected
143  *  in the import data.
144  * @param from_commodity pointer to a commodity or NULL.
145  */
from_commodity(gnc_commodity * from_commodity)146 void GncPriceImport::from_commodity (gnc_commodity* from_commodity)
147 {
148     m_settings.m_from_commodity = from_commodity;
149     if (m_settings.m_from_commodity)
150     {
151         auto col_type_sym = std::find (m_settings.m_column_types_price.begin(),
152                 m_settings.m_column_types_price.end(), GncPricePropType::FROM_SYMBOL);
153 
154         if (col_type_sym != m_settings.m_column_types_price.end())
155             set_column_type_price (col_type_sym -m_settings.m_column_types_price.begin(),
156                             GncPricePropType::NONE);
157 
158         auto col_type_name = std::find (m_settings.m_column_types_price.begin(),
159                 m_settings.m_column_types_price.end(), GncPricePropType::FROM_NAMESPACE);
160 
161         if (col_type_name != m_settings.m_column_types_price.end())
162             set_column_type_price (col_type_name -m_settings.m_column_types_price.begin(),
163                             GncPricePropType::NONE);
164 
165         // force a refresh of the to_currency if the from_commodity is changed
166         std::vector<GncPricePropType> commodities = { GncPricePropType::TO_CURRENCY };
167         reset_formatted_column (commodities);
168     }
169 }
from_commodity()170 gnc_commodity *GncPriceImport::from_commodity () { return m_settings.m_from_commodity; }
171 
172 /** Sets a to currency. This is the to currency all import data relates to.
173  *  When a to currency is set, there can't be any to currency columns selected
174  *  in the import data.
175  * @param to_currency pointer to a commodity or NULL.
176  */
to_currency(gnc_commodity * to_currency)177 void GncPriceImport::to_currency (gnc_commodity* to_currency)
178 {
179     m_settings.m_to_currency = to_currency;
180     if (m_settings.m_to_currency)
181     {
182         auto col_type_currency = std::find (m_settings.m_column_types_price.begin(),
183                 m_settings.m_column_types_price.end(), GncPricePropType::TO_CURRENCY);
184 
185         if (col_type_currency != m_settings.m_column_types_price.end())
186             set_column_type_price (col_type_currency -m_settings.m_column_types_price.begin(),
187                             GncPricePropType::NONE);
188 
189         // force a refresh of the from_commodity if the to_currency is changed
190         // either namespace or symbol will be sufice
191         std::vector<GncPricePropType> commodities = { GncPricePropType::FROM_SYMBOL };
192         reset_formatted_column (commodities);
193     }
194 }
to_currency()195 gnc_commodity *GncPriceImport::to_currency () { return m_settings.m_to_currency; }
196 
reset_formatted_column(std::vector<GncPricePropType> & col_types)197 void GncPriceImport::reset_formatted_column (std::vector<GncPricePropType>& col_types)
198 {
199     for (auto col_type: col_types)
200     {
201         auto col = std::find (m_settings.m_column_types_price.begin(),
202                 m_settings.m_column_types_price.end(), col_type);
203         if (col != m_settings.m_column_types_price.end())
204             set_column_type_price (col - m_settings.m_column_types_price.begin(), col_type, true);
205     }
206 }
207 
currency_format(int currency_format)208 void GncPriceImport::currency_format (int currency_format)
209 {
210     m_settings.m_currency_format = currency_format;
211 
212     /* Reparse all currency related columns */
213     std::vector<GncPricePropType> commodities = { GncPricePropType::AMOUNT };
214     reset_formatted_column (commodities);
215 }
currency_format()216 int GncPriceImport::currency_format () { return m_settings.m_currency_format; }
217 
date_format(int date_format)218 void GncPriceImport::date_format (int date_format)
219 {
220     m_settings.m_date_format = date_format;
221 
222     /* Reparse all date related columns */
223     std::vector<GncPricePropType> dates = { GncPricePropType::DATE };
224     reset_formatted_column (dates);
225 }
date_format()226 int GncPriceImport::date_format () { return m_settings.m_date_format; }
227 
228 /** Converts raw file data using a new encoding. This function must be
229  * called after load_file only if load_file guessed
230  * the wrong encoding.
231  * @param encoding Encoding that data should be translated using
232  */
encoding(const std::string & encoding)233 void GncPriceImport::encoding (const std::string& encoding)
234 {
235     // TODO investigate if we can catch conversion errors and report them
236     if (m_tokenizer)
237     {
238         m_tokenizer->encoding(encoding); // May throw
239         try
240         {
241             tokenize(false);
242         }
243         catch (...)
244         { };
245     }
246 
247     m_settings.m_encoding = encoding;
248 }
249 
encoding()250 std::string GncPriceImport::encoding () { return m_settings.m_encoding; }
251 
update_skipped_lines(boost::optional<uint32_t> start,boost::optional<uint32_t> end,boost::optional<bool> alt,boost::optional<bool> errors)252 void GncPriceImport::update_skipped_lines(boost::optional<uint32_t> start, boost::optional<uint32_t> end,
253         boost::optional<bool> alt, boost::optional<bool> errors)
254 {
255     if (start)
256         m_settings.m_skip_start_lines = *start;
257     if (end)
258         m_settings.m_skip_end_lines = *end;
259     if (alt)
260         m_settings.m_skip_alt_lines = *alt;
261     if (errors)
262         m_skip_errors = *errors;
263 
264     for (uint32_t i = 0; i < m_parsed_lines.size(); i++)
265     {
266         std::get<PL_SKIP>(m_parsed_lines[i]) =
267             ((i < skip_start_lines()) ||             // start rows to skip
268              (i >= m_parsed_lines.size() - skip_end_lines()) ||          // end rows to skip
269              (((i - skip_start_lines()) % 2 == 1) && // skip every second row...
270                   skip_alt_lines()) ||                   // ...if requested
271              (m_skip_errors && !std::get<PL_ERROR>(m_parsed_lines[i]).empty())); // skip lines with errors
272     }
273 }
274 
skip_start_lines()275 uint32_t GncPriceImport::skip_start_lines () { return m_settings.m_skip_start_lines; }
skip_end_lines()276 uint32_t GncPriceImport::skip_end_lines () { return m_settings.m_skip_end_lines; }
skip_alt_lines()277 bool GncPriceImport::skip_alt_lines () { return m_settings.m_skip_alt_lines; }
skip_err_lines()278 bool GncPriceImport::skip_err_lines () { return m_skip_errors; }
279 
separators(std::string separators)280 void GncPriceImport::separators (std::string separators)
281 {
282     if (file_format() != GncImpFileFormat::CSV)
283         return;
284 
285     m_settings.m_separators = separators;
286     auto csvtok = dynamic_cast<GncCsvTokenizer*>(m_tokenizer.get());
287     csvtok->set_separators (separators);
288 
289 }
separators()290 std::string GncPriceImport::separators () { return m_settings.m_separators; }
291 
settings(const CsvPriceImpSettings & settings)292 void GncPriceImport::settings (const CsvPriceImpSettings& settings)
293 {
294     /* First apply file format as this may recreate the tokenizer */
295     file_format (settings.m_file_format);
296     /* Only then apply the other settings */
297     m_settings = settings;
298     from_commodity (m_settings.m_from_commodity);
299     to_currency (m_settings.m_to_currency);
300     encoding (m_settings.m_encoding);
301 
302     if (file_format() == GncImpFileFormat::CSV)
303         separators (m_settings.m_separators);
304     else if (file_format() == GncImpFileFormat::FIXED_WIDTH)
305     {
306         auto fwtok = dynamic_cast<GncFwTokenizer*>(m_tokenizer.get());
307         fwtok->columns (m_settings.m_column_widths);
308     }
309     try
310     {
311         tokenize(false);
312     }
313     catch (...)
314     { };
315 
316     /* Tokenizing will clear column types, reset them here
317      * based on the loaded settings.
318      */
319     std::copy_n (settings.m_column_types_price.begin(),
320             std::min (m_settings.m_column_types_price.size(), settings.m_column_types_price.size()),
321             m_settings.m_column_types_price.begin());
322 }
323 
save_settings()324 bool GncPriceImport::save_settings ()
325 {
326     if (preset_is_reserved_name (m_settings.m_name))
327         return true;
328 
329     /* separators are already copied to m_settings in the separators
330      * function above. However this is not the case for the column
331      * widths in fw mode, so do this now.
332      */
333     if (file_format() == GncImpFileFormat::FIXED_WIDTH)
334     {
335         auto fwtok = dynamic_cast<GncFwTokenizer*>(m_tokenizer.get());
336         m_settings.m_column_widths = fwtok->get_columns();
337     }
338     return m_settings.save();
339 }
340 
settings_name(std::string name)341 void GncPriceImport::settings_name (std::string name) { m_settings.m_name = name; }
settings_name()342 std::string GncPriceImport::settings_name () { return m_settings.m_name; }
343 
344 /** Loads a file into a GncPriceImport. This is the first function
345  * that must be called after creating a new GncPriceImport. As long as
346  * this function didn't run successfully, the importer can't proceed.
347  * @param filename Name of the file that should be opened
348  * @exception may throw std::ifstream::failure on any io error
349  */
load_file(const std::string & filename)350 void GncPriceImport::load_file (const std::string& filename)
351 {
352     /* Get the raw data first and handle an error if one occurs. */
353     try
354     {
355         m_tokenizer->load_file (filename);
356         return;
357     }
358     catch (std::ifstream::failure& ios_err)
359     {
360         // Just log the error and pass it on the call stack for proper handling
361         PWARN ("Error: %s", ios_err.what());
362         throw;
363     }
364 }
365 
366 /** Splits a file into cells. This requires having an encoding that
367  * works (see GncPriceImport::convert_encoding). Tokenizing related options
368  * should be set to the user's selections before calling this
369  * function.
370  * Notes: - this function must be called with guessColTypes set to true once
371  *          before calling it with guessColTypes set to false.
372  *        - if guessColTypes is true, all the column types will be set
373  *          GncPricePropType::NONE right now as real guessing isn't implemented yet
374  * @param guessColTypes true to guess what the types of columns are based on the cell contents
375  * @exception std::range_error if tokenizing failed
376  */
tokenize(bool guessColTypes)377 void GncPriceImport::tokenize (bool guessColTypes)
378 {
379     if (!m_tokenizer)
380         return;
381 
382     uint32_t max_cols = 0;
383     m_tokenizer->tokenize();
384     m_parsed_lines.clear();
385     for (auto tokenized_line : m_tokenizer->get_tokens())
386     {
387         auto length = tokenized_line.size();
388         if (length > 0)
389             m_parsed_lines.push_back (std::make_tuple (tokenized_line, std::string(),
390                     std::make_shared<GncImportPrice>(date_format(), currency_format()),
391                     false));
392         if (length > max_cols)
393             max_cols = length;
394     }
395 
396     /* If it failed, generate an error. */
397     if (m_parsed_lines.size() == 0)
398     {
399         throw (std::range_error ("Tokenizing failed."));
400         return;
401     }
402 
403     m_settings.m_column_types_price.resize(max_cols, GncPricePropType::NONE);
404 
405     /* Force reinterpretation of already set columns and/or base_account */
406     for (uint32_t i = 0; i < m_settings.m_column_types_price.size(); i++)
407         set_column_type_price (i, m_settings.m_column_types_price[i], true);
408 
409     if (guessColTypes)
410     {
411         /* Guess column_types based
412          * on the contents of each column. */
413         /* TODO Make it actually guess. */
414     }
415 }
416 
417 struct ErrorListPrice
418 {
419 public:
420     void add_error (std::string msg);
421     std::string str();
emptyErrorListPrice422     bool empty() { return m_error.empty(); }
423 private:
424     std::string m_error;
425 };
426 
add_error(std::string msg)427 void ErrorListPrice::add_error (std::string msg)
428 {
429     m_error += "- " + msg + "\n";
430 }
431 
str()432 std::string ErrorListPrice::str()
433 {
434     return m_error.substr(0, m_error.size() - 1);
435 }
436 
437 /* Test for the required minimum number of columns selected and
438  * the selection is consistent.
439  * @param An ErrorListPrice object to which all found issues are added.
440  */
verify_column_selections(ErrorListPrice & error_msg)441 void GncPriceImport::verify_column_selections (ErrorListPrice& error_msg)
442 {
443     /* Verify if a date column is selected and it's parsable.
444      */
445     if (!check_for_column_type(GncPricePropType::DATE))
446         error_msg.add_error( _("Please select a date column."));
447 
448     /* Verify an amount column is selected.
449      */
450     if (!check_for_column_type(GncPricePropType::AMOUNT))
451         error_msg.add_error( _("Please select an amount column."));
452 
453     /* Verify a Currency to column is selected.
454      */
455     if (!check_for_column_type(GncPricePropType::TO_CURRENCY))
456     {
457         if (!m_settings.m_to_currency)
458             error_msg.add_error( _("Please select a 'Currency to' column or set a Currency in the 'Currency To' field."));
459     }
460 
461     /* Verify a From Symbol column is selected.
462      */
463     if (!check_for_column_type(GncPricePropType::FROM_SYMBOL))
464     {
465         if (!m_settings.m_from_commodity)
466             error_msg.add_error( _("Please select a 'From Symbol' column or set a Commodity in the 'Commodity From' field."));
467     }
468 
469     /* Verify a From Namespace column is selected.
470      */
471     if (!check_for_column_type(GncPricePropType::FROM_NAMESPACE))
472     {
473         if (!m_settings.m_from_commodity)
474             error_msg.add_error( _("Please select a 'From Namespace' column or set a Commodity in the 'Commodity From' field."));
475     }
476 
477     /* Verify a 'Commodity from' does not equal 'Currency to'.
478      */
479     if ((m_settings.m_to_currency) && (m_settings.m_from_commodity))
480     {
481         if (gnc_commodity_equal (m_settings.m_to_currency, m_settings.m_from_commodity))
482             error_msg.add_error( _("'Commodity From' can not be the same as 'Currency To'."));
483     }
484 }
485 
486 /* Check whether the chosen settings can successfully parse
487  * the import data. This will check:
488  * - there's at least one line selected for import
489  * - the minimum number of columns is selected
490  * - the values in the selected columns can be parsed meaningfully.
491  * @return An empty string if all checks passed or the reason
492  *         verification failed otherwise.
493  */
verify()494 std::string GncPriceImport::verify ()
495 {
496     auto newline = std::string();
497     auto error_msg = ErrorListPrice();
498 
499     /* Check if the import file did actually contain any information */
500     if (m_parsed_lines.size() == 0)
501     {
502         error_msg.add_error(_("No valid data found in the selected file. It may be empty or the selected encoding is wrong."));
503         return error_msg.str();
504     }
505 
506     /* Check if at least one line is selected for importing */
507     auto skip_alt_offset = m_settings.m_skip_alt_lines ? 1 : 0;
508     if (m_settings.m_skip_start_lines + m_settings.m_skip_end_lines + skip_alt_offset >= m_parsed_lines.size())
509     {
510         error_msg.add_error(_("No lines are selected for importing. Please reduce the number of lines to skip."));
511         return error_msg.str();
512     }
513 
514     verify_column_selections (error_msg);
515 
516     update_skipped_lines (boost::none, boost::none, boost::none, boost::none);
517 
518     auto have_line_errors = false;
519     for (auto line : m_parsed_lines)
520     {
521         if (!std::get<PL_SKIP>(line) && !std::get<PL_ERROR>(line).empty())
522         {
523             have_line_errors = true;
524             break;
525         }
526     }
527 
528     if (have_line_errors)
529         error_msg.add_error( _("Not all fields could be parsed. Please correct the issues reported for each line or adjust the lines to skip."));
530 
531     return error_msg.str();
532 }
533 
534 /** Checks whether the parsed line contains all essential properties.
535  * @param parsed_line The line we are checking
536  * @exception std::invalid_argument in an essential property is missing
537  */
price_properties_verify_essentials(std::vector<parse_line_t>::iterator & parsed_line)538 static void price_properties_verify_essentials (std::vector<parse_line_t>::iterator& parsed_line)
539 {
540     std::string error_message;
541     std::shared_ptr<GncImportPrice> price_props;
542     std::tie(std::ignore, error_message, price_props, std::ignore) = *parsed_line;
543 
544     auto price_error = price_props->verify_essentials();
545 
546     error_message.clear();
547     if (!price_error.empty())
548     {
549         error_message += price_error;
550         error_message += "\n";
551     }
552 
553     if (!error_message.empty())
554         throw std::invalid_argument(error_message);
555 }
556 
create_price(std::vector<parse_line_t>::iterator & parsed_line)557 void GncPriceImport::create_price (std::vector<parse_line_t>::iterator& parsed_line)
558 {
559     StrVec line;
560     std::string error_message;
561     std::shared_ptr<GncImportPrice> price_props = nullptr;
562     bool skip_line = false;
563     std::tie(line, error_message, price_props, skip_line) = *parsed_line;
564 
565     if (skip_line)
566         return;
567 
568     error_message.clear();
569 
570     // Add a TO_CURRENCY property with the selected 'to_currency' if no 'Currency To' column was set by the user
571     auto line_to_currency = price_props->get_to_currency();
572     if (!line_to_currency)
573     {
574         if (m_settings.m_to_currency)
575             price_props->set_to_currency(m_settings.m_to_currency);
576         else
577         {
578             // Oops - the user didn't select a 'currency to' column *and* we didn't get a selected value either!
579             // Note if you get here this suggests a bug in the code!
580             error_message = _("No 'Currency to' column selected and no selected Currency specified either.\n"
581                                        "This should never happen. Please report this as a bug.");
582             PINFO("User warning: %s", error_message.c_str());
583             throw std::invalid_argument(error_message);
584         }
585     }
586 
587     // Add a FROM_COMMODITY property with the selected 'from_commodity' if no 'From Namespace/Symbol' columns were set by the user
588     auto line_from_commodity = price_props->get_from_commodity();
589     if (!line_from_commodity)
590     {
591         if (m_settings.m_from_commodity)
592             price_props->set_from_commodity(m_settings.m_from_commodity);
593         else
594         {
595             // Oops - the user didn't select a 'commodity from' column *and* we didn't get a selected value either!
596             // Note if you get here this suggests a bug in the code!
597             error_message = _("No 'From Namespace/From Symbol' columns selected and no selected Commodity From specified either.\n"
598                                        "This should never happen. Please report this as a bug.");
599             PINFO("User warning: %s", error_message.c_str());
600             throw std::invalid_argument(error_message);
601         }
602     }
603 
604     /* If column parsing was successful, convert price properties into a price. */
605     try
606     {
607         price_properties_verify_essentials (parsed_line);
608 
609         QofBook* book = gnc_get_current_book();
610         GNCPriceDB *pdb = gnc_pricedb_get_db (book);
611 
612         /* If all went well, add this price to the list. */
613         auto price_created = price_props->create_price (book, pdb, m_over_write);
614         if (price_created == ADDED)
615             m_prices_added++;
616         else if (price_created == DUPLICATED)
617             m_prices_duplicated++;
618         else if (price_created == REPLACED)
619             m_prices_replaced++;
620     }
621     catch (const std::invalid_argument& e)
622     {
623         error_message = e.what();
624         PINFO("User warning: %s", error_message.c_str());
625     }
626 }
627 
628 /** Creates a list of prices from parsed data. The parsed data
629  * will first be validated. If any errors are found in lines that are marked
630  * for processing (ie not marked to skip) this function will
631  * throw an error.
632  * @param skip_errors true skip over lines with errors
633  * @exception throws std::invalid_argument if data validation or processing fails.
634  */
create_prices()635 void GncPriceImport::create_prices ()
636 {
637     /* Start with verifying the current data. */
638     auto verify_result = verify();
639     if (!verify_result.empty())
640         throw std::invalid_argument (verify_result);
641 
642     m_prices_added = 0;
643     m_prices_duplicated = 0;
644     m_prices_replaced = 0;
645 
646     /* Iterate over all parsed lines */
647     for (auto parsed_lines_it = m_parsed_lines.begin();
648             parsed_lines_it != m_parsed_lines.end();
649             ++parsed_lines_it)
650     {
651         /* Skip current line if the user specified so */
652         if ((std::get<PL_SKIP>(*parsed_lines_it)))
653             continue;
654 
655         /* Should not throw anymore, otherwise verify needs revision */
656         create_price (parsed_lines_it);
657     }
658     PINFO("Number of lines is %d, added %d, duplicated %d, replaced %d",
659          (int)m_parsed_lines.size(), m_prices_added, m_prices_duplicated, m_prices_replaced);
660 }
661 
662 bool
check_for_column_type(GncPricePropType type)663 GncPriceImport::check_for_column_type (GncPricePropType type)
664 {
665     return (std::find (m_settings.m_column_types_price.begin(),
666                        m_settings.m_column_types_price.end(), type)
667                         != m_settings.m_column_types_price.end());
668 }
669 
670 /* A helper function intended to be called only from set_column_type_price */
update_price_props(uint32_t row,uint32_t col,GncPricePropType prop_type)671 void GncPriceImport::update_price_props (uint32_t row, uint32_t col, GncPricePropType prop_type)
672 {
673     if (prop_type == GncPricePropType::NONE)
674         return; /* Only deal with price related properties. */
675 
676     auto price_props = std::make_shared<GncImportPrice> (*(std::get<PL_PREPRICE>(m_parsed_lines[row])).get());
677 
678     if (col >= std::get<PL_INPUT>(m_parsed_lines[row]).size())
679         price_props->reset (prop_type); //reset errors
680     else
681     {
682         auto value = std::get<PL_INPUT>(m_parsed_lines[row]).at(col);
683         bool enable_test_empty = true;
684         try
685         {
686             // set the from_commodity based on combo so we can test for same.
687             if (prop_type == GncPricePropType::TO_CURRENCY)
688             {
689                 if (m_settings.m_from_commodity)
690                     price_props->set_from_commodity (m_settings.m_from_commodity);
691 
692                 if (m_settings.m_to_currency)
693                     enable_test_empty = false;
694             }
695             // set the to_currency based on combo so we can test for same.
696             if (prop_type == GncPricePropType::FROM_SYMBOL)
697             {
698                 if (m_settings.m_to_currency)
699                     price_props->set_to_currency (m_settings.m_to_currency);
700 
701                 if (m_settings.m_from_commodity)
702                     enable_test_empty = false;
703             }
704             price_props->set(prop_type, value, enable_test_empty);
705         }
706         catch (const std::exception& e)
707         {
708             /* Do nothing, just prevent the exception from escalating up
709              * However log the error if it happens on a row that's not skipped
710              */
711             if (!std::get<PL_SKIP>(m_parsed_lines[row]))
712                 PINFO("User warning: %s", e.what());
713         }
714     }
715     /* Store the result */
716     std::get<PL_PREPRICE>(m_parsed_lines[row]) = price_props;
717 }
718 
719 void
set_column_type_price(uint32_t position,GncPricePropType type,bool force)720 GncPriceImport::set_column_type_price (uint32_t position, GncPricePropType type, bool force)
721 {
722     if (position >= m_settings.m_column_types_price.size())
723         return;
724 
725     auto old_type = m_settings.m_column_types_price[position];
726     if ((type == old_type) && !force)
727         return; /* Nothing to do */
728 
729     // Column types should be unique, so remove any previous occurrence of the new type
730     std::replace(m_settings.m_column_types_price.begin(), m_settings.m_column_types_price.end(),
731             type, GncPricePropType::NONE);
732 
733     m_settings.m_column_types_price.at (position) = type;
734 
735     // If the user has set a 'from namespace' column, we can't have a 'commodity from' selected
736     if (type == GncPricePropType::FROM_NAMESPACE)
737         from_commodity (nullptr);
738 
739     // If the user has set a 'from symbol' column, we can't have a 'commodity from' selected
740     if (type == GncPricePropType::FROM_SYMBOL)
741         from_commodity (nullptr);
742 
743     // If the user has set a 'currency to' column, we can't have a 'currency to' selected
744     if (type == GncPricePropType::TO_CURRENCY)
745         to_currency (nullptr);
746 
747     /* Update the preparsed data */
748     for (auto parsed_lines_it = m_parsed_lines.begin();
749             parsed_lines_it != m_parsed_lines.end();
750             ++parsed_lines_it)
751     {
752         /* Reset date and currency formats for each price props object
753          * to ensure column updates use the most recent one
754          */
755         std::get<PL_PREPRICE>(*parsed_lines_it)->set_date_format (m_settings.m_date_format);
756         std::get<PL_PREPRICE>(*parsed_lines_it)->set_currency_format (m_settings.m_currency_format);
757 
758         uint32_t row = parsed_lines_it - m_parsed_lines.begin();
759 
760         /* If the column type actually changed, first reset the property
761          * represented by the old column type
762          */
763         if (old_type != type)
764         {
765             auto old_col = std::get<PL_INPUT>(*parsed_lines_it).size(); // Deliberately out of bounds to trigger a reset!
766             if ((old_type > GncPricePropType::NONE)
767                     && (old_type <= GncPricePropType::PRICE_PROPS))
768                 update_price_props (row, old_col, old_type);
769         }
770         /* Then set the property represented by the new column type */
771         if ((type > GncPricePropType::NONE)
772                 && (type <= GncPricePropType::PRICE_PROPS))
773             update_price_props (row, position, type);
774 
775         /* Report errors if there are any */
776         auto price_errors = std::get<PL_PREPRICE>(*parsed_lines_it)->errors();
777         std::get<PL_ERROR>(*parsed_lines_it) =
778                 price_errors +
779                 (price_errors.empty() ? std::string() : "\n");
780     }
781 }
782 
column_types_price()783 std::vector<GncPricePropType> GncPriceImport::column_types_price ()
784 {
785     return m_settings.m_column_types_price;
786 }
787 
788