1 /*=========================================================================
2
3 Program: Visualization Toolkit
4 Module: vtkTecplotTableReader.cxx
5
6 Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
7 All rights reserved.
8 See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
9
10 This software is distributed WITHOUT ANY WARRANTY; without even
11 the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
12 PURPOSE. See the above copyright notice for more information.
13
14 =========================================================================*/
15 /*-------------------------------------------------------------------------
16 Copyright 2016 Menno Deij - van Rijswijk (MARIN)
17 -------------------------------------------------------------------------*/
18
19 #include "vtkTecplotTableReader.h"
20 #include "vtkCommand.h"
21 #include "vtkDataSetAttributes.h"
22 #include "vtkIdTypeArray.h"
23 #include "vtkInformation.h"
24 #include "vtkInformationVector.h"
25 #include "vtkObjectFactory.h"
26 #include "vtkSmartPointer.h"
27 #include "vtkStreamingDemandDrivenPipeline.h"
28 #include "vtkTable.h"
29 #include "vtkDoubleArray.h"
30
31 #include "vtkTextCodec.h"
32 #include "vtkTextCodecFactory.h"
33
34 #include <sstream>
35 #include <iostream>
36 #include <algorithm>
37 #include <iterator>
38 #include <stdexcept>
39 #include <set>
40 #include <vector>
41
42 #include <cctype>
43
44 ////////////////////////////////////////////////////////////////////////////////
45 // DelimitedTextIterator
46
47 /// Output iterator object that parses a stream of Unicode characters into records and
48 /// fields, inserting them into a vtkTable. Based on the iterator from
49 /// vtkIOInfoVis::DelimitedTextReader but tailored to Tecplot table files
50
51 namespace {
52
53 class DelimitedTextIterator : public vtkTextCodec::OutputIterator
54 {
55 public:
56 typedef std::forward_iterator_tag iterator_category;
57 typedef vtkUnicodeStringValueType value_type;
58 typedef std::string::difference_type difference_type;
59 typedef value_type* pointer;
60 typedef value_type& reference;
61
DelimitedTextIterator(vtkTable * const outputTable,const vtkIdType maxRecords,const vtkIdType headerLines,const vtkIdType columnHeadersOnLine,const vtkIdType skipColumnNames)62 DelimitedTextIterator(
63 vtkTable* const outputTable,
64 const vtkIdType maxRecords,
65 const vtkIdType headerLines,
66 const vtkIdType columnHeadersOnLine,
67 const vtkIdType skipColumnNames
68 ):
69 MaxRecords(maxRecords),
70 MaxRecordIndex(maxRecords + headerLines), // first two lines are title + column names
71 WhiteSpaceOnlyString(true),
72 OutputTable(outputTable),
73 CurrentRecordIndex(0),
74 CurrentFieldIndex(0),
75 HeaderLines(headerLines),
76 ColumnNamesOnLine(columnHeadersOnLine),
77 SkipColumnNames(skipColumnNames),
78 RecordAdjacent(true),
79 MergeConsDelims(true),
80 ProcessEscapeSequence(false),
81 UseStringDelimiter(true),
82 WithinString(0)
83 {
84 // how records (e.g. lines) are separated
85 RecordDelimiters.insert('\n');
86 RecordDelimiters.insert('\r');
87
88 // how fields (e.g. entries) are separated
89 FieldDelimiters.insert(' ');
90 FieldDelimiters.insert('\t');
91
92 // how string entries are separated
93 StringDelimiters.insert('"');
94 StringDelimiters.insert(' ');
95
96 // what is whitespace
97 Whitespace.insert(' ');
98 Whitespace.insert('\t');
99 }
100
~DelimitedTextIterator()101 ~DelimitedTextIterator() override
102 {
103 // Ensure that all table columns have the same length ...
104 for(vtkIdType i = 0; i != this->OutputTable->GetNumberOfColumns(); ++i)
105 {
106 if(this->OutputTable->GetColumn(i)->GetNumberOfTuples() !=
107 this->OutputTable->GetColumn(0)->GetNumberOfTuples())
108 {
109 this->OutputTable->GetColumn(i)
110 ->Resize(this->OutputTable->GetColumn(0)->GetNumberOfTuples());
111 }
112 }
113 }
114
operator ++(int)115 DelimitedTextIterator& operator++(int) override
116 {
117 return *this;
118 }
119
operator *()120 DelimitedTextIterator& operator*() override
121 {
122 return *this;
123 }
124
125 // Handle windows files that do not have a carriage return line feed on the last line of the file ...
ReachedEndOfInput()126 void ReachedEndOfInput()
127 {
128 if(this->CurrentField.empty())
129 {
130 return;
131 }
132 vtkUnicodeString::value_type value =
133 this->CurrentField[this->CurrentField.character_count()-1];
134 if(!this->RecordDelimiters.count(value) && !this->Whitespace.count(value))
135 {
136 this->InsertField();
137 }
138 }
139
operator =(const vtkUnicodeString::value_type value)140 DelimitedTextIterator& operator=(const vtkUnicodeString::value_type value) override
141 {
142 // If we've already read our maximum number of records, we're done ...
143 if(this->MaxRecords && this->CurrentRecordIndex == this->MaxRecordIndex)
144 {
145 return *this;
146 }
147
148 // Strip adjacent record delimiters and whitespace...
149 if(this->RecordAdjacent && (this->RecordDelimiters.count(value) ||
150 this->Whitespace.count(value)))
151 {
152 return *this;
153 }
154 else
155 {
156 this->RecordAdjacent = false;
157 }
158
159 // Look for record delimiters ...
160 if(this->RecordDelimiters.count(value))
161 {
162 // keep skipping until column names line
163 if (this->CurrentRecordIndex < ColumnNamesOnLine)
164 {
165 this->CurrentRecordIndex += 1;
166 return *this;
167 }
168
169 this->InsertField();
170 this->CurrentRecordIndex += 1;
171 this->CurrentFieldIndex = 0;
172 this->CurrentField.clear();
173 this->RecordAdjacent = true;
174 this->WithinString = 0;
175 this->WhiteSpaceOnlyString = true;
176 return *this;
177 }
178
179 if (this->CurrentRecordIndex < ColumnNamesOnLine)
180 {
181 return *this; // keep skipping until column names line
182 }
183
184 // Look for field delimiters unless we're in a string ...
185 if(!this->WithinString && this->FieldDelimiters.count(value))
186 {
187 // Handle special case of merging consective delimiters ...
188 if( !(this->CurrentField.empty() && this->MergeConsDelims) )
189 {
190 if (!(this->CurrentFieldIndex < SkipColumnNames && this->CurrentRecordIndex == ColumnNamesOnLine))
191 //if (!(this->CurrentFieldIndex == 0 && this->CurrentRecordIndex == 1))
192 {
193 this->InsertField();
194 }
195 this->CurrentFieldIndex += 1;
196 this->CurrentField.clear();
197 }
198 return *this;
199 }
200
201 // Check for start of escape sequence ...
202 if(!this->ProcessEscapeSequence && this->EscapeDelimiter.count(value))
203 {
204 this->ProcessEscapeSequence = true;
205 return *this;
206 }
207
208 // Process escape sequence ...
209 if(this->ProcessEscapeSequence)
210 {
211 vtkUnicodeString curr_char;
212 curr_char += value;
213 if(curr_char == vtkUnicodeString::from_utf8("0"))
214 {
215 this->CurrentField += vtkUnicodeString::from_utf8("\0");
216 }
217 else if(curr_char == vtkUnicodeString::from_utf8("a"))
218 {
219 this->CurrentField += vtkUnicodeString::from_utf8("\a");
220 }
221 else if(curr_char == vtkUnicodeString::from_utf8("b"))
222 {
223 this->CurrentField += vtkUnicodeString::from_utf8("\b");
224 }
225 else if(curr_char == vtkUnicodeString::from_utf8("t"))
226 {
227 this->CurrentField += vtkUnicodeString::from_utf8("\t");
228 }
229 else if(curr_char == vtkUnicodeString::from_utf8("n"))
230 {
231 this->CurrentField += vtkUnicodeString::from_utf8("\n");
232 }
233 else if(curr_char == vtkUnicodeString::from_utf8("v"))
234 {
235 this->CurrentField += vtkUnicodeString::from_utf8("\v");
236 }
237 else if(curr_char == vtkUnicodeString::from_utf8("f"))
238 {
239 this->CurrentField += vtkUnicodeString::from_utf8("\f");
240 }
241 else if(curr_char == vtkUnicodeString::from_utf8("r"))
242 {
243 this->CurrentField += vtkUnicodeString::from_utf8("\r");
244 }
245 else if(curr_char == vtkUnicodeString::from_utf8("\\"))
246 {
247 this->CurrentField += vtkUnicodeString::from_utf8("\\");
248 }
249 else
250 {
251 this->CurrentField += value;
252 }
253 this->ProcessEscapeSequence = false;
254 return *this;
255 }
256
257 // Start a string ...
258 if(!this->WithinString && this->StringDelimiters.count(value) &&
259 this->UseStringDelimiter)
260 {
261 this->WithinString = value;
262 this->CurrentField.clear();
263 return *this;
264 }
265
266 // End a string ...
267 if(this->WithinString && (this->WithinString == value) &&
268 this->UseStringDelimiter)
269 {
270 this->WithinString = 0;
271 return *this;
272 }
273
274 if(!this->Whitespace.count(value))
275 {
276 this->WhiteSpaceOnlyString = false;
277 }
278 // Keep growing the current field ...
279 this->CurrentField += value;
280 return *this;
281 }
282
283 private:
InsertField()284 void InsertField()
285 {
286 vtkIdType fieldIndex = this->CurrentFieldIndex;
287 if (this->CurrentRecordIndex == ColumnNamesOnLine)
288 {
289 fieldIndex -= SkipColumnNames;
290 }
291
292 if(fieldIndex >= this->OutputTable->GetNumberOfColumns() && ColumnNamesOnLine == this->CurrentRecordIndex)
293 {
294 vtkDoubleArray* array = vtkDoubleArray::New();
295
296 array->SetName(this->CurrentField.utf8_str());
297 this->OutputTable->AddColumn(array);
298 array->Delete();
299 }
300 else if(fieldIndex < this->OutputTable->GetNumberOfColumns())
301 {
302 // Handle case where input file has header information ...
303 vtkIdType recordIndex;
304 recordIndex = this->CurrentRecordIndex - HeaderLines;
305 vtkDoubleArray* array = vtkArrayDownCast<vtkDoubleArray>(this->OutputTable->GetColumn(fieldIndex));
306
307 vtkStdString str;
308 str = this->CurrentField.utf8_str();
309 bool ok;
310 double doubleValue = vtkVariant(str).ToDouble(&ok);
311 if (ok)
312 {
313 array->InsertValue(recordIndex, doubleValue);
314 }
315 else
316 {
317 array->InsertValue(recordIndex, std::numeric_limits<double>::quiet_NaN());
318 }
319 }
320 }
321
322 vtkIdType MaxRecords;
323 vtkIdType MaxRecordIndex;
324 std::set<vtkUnicodeString::value_type> RecordDelimiters;
325 std::set<vtkUnicodeString::value_type> FieldDelimiters;
326 std::set<vtkUnicodeString::value_type> StringDelimiters;
327 std::set<vtkUnicodeString::value_type> Whitespace;
328 std::set<vtkUnicodeString::value_type> EscapeDelimiter;
329
330 bool WhiteSpaceOnlyString;
331 vtkTable* OutputTable;
332 vtkIdType CurrentRecordIndex;
333 vtkIdType CurrentFieldIndex;
334 vtkUnicodeString CurrentField;
335
336 vtkIdType HeaderLines;
337 vtkIdType ColumnNamesOnLine;
338 vtkIdType SkipColumnNames;
339
340
341 bool RecordAdjacent;
342 bool MergeConsDelims;
343 bool ProcessEscapeSequence;
344 bool UseStringDelimiter;
345 vtkUnicodeString::value_type WithinString;
346 };
347
348 } // End anonymous namespace
349
350 /////////////////////////////////////////////////////////////////////////////////////////
351 // vtkTecplotTableReader
352
353 vtkStandardNewMacro(vtkTecplotTableReader);
354
vtkTecplotTableReader()355 vtkTecplotTableReader::vtkTecplotTableReader() :
356 FileName(nullptr),
357 MaxRecords(0),
358 HeaderLines(2),
359 ColumnNamesOnLine(1),
360 SkipColumnNames(1)
361 {
362 this->SetNumberOfInputPorts(0);
363 this->SetNumberOfOutputPorts(1);
364 this->PedigreeIdArrayName = nullptr;
365 this->SetPedigreeIdArrayName("id");
366 this->GeneratePedigreeIds = false;
367 this->OutputPedigreeIds = false;
368 }
369
~vtkTecplotTableReader()370 vtkTecplotTableReader::~vtkTecplotTableReader()
371 {
372 this->SetPedigreeIdArrayName(nullptr);
373 this->SetFileName(nullptr);
374 }
375
PrintSelf(ostream & os,vtkIndent indent)376 void vtkTecplotTableReader::PrintSelf(ostream& os, vtkIndent indent)
377 {
378 this->Superclass::PrintSelf(os, indent);
379 os << indent << "FileName: "
380 << (this->FileName ? this->FileName : "(none)") << endl;
381 os << indent << "MaxRecords: " << this->MaxRecords
382 << endl;
383 os << indent << "GeneratePedigreeIds: "
384 << this->GeneratePedigreeIds << endl;
385 os << indent << "PedigreeIdArrayName: "
386 << this->PedigreeIdArrayName << endl;
387 os << indent << "OutputPedigreeIds: "
388 << (this->OutputPedigreeIds? "true" : "false") << endl;
389 }
390
GetLastError()391 vtkStdString vtkTecplotTableReader::GetLastError()
392 {
393 return this->LastError;
394 }
395
RequestData(vtkInformation *,vtkInformationVector **,vtkInformationVector * outputVector)396 int vtkTecplotTableReader::RequestData(
397 vtkInformation*,
398 vtkInformationVector**,
399 vtkInformationVector* outputVector)
400 {
401 vtkTable* const output_table = vtkTable::GetData(outputVector);
402
403 this->LastError = "";
404
405 try
406 {
407 // We only retrieve one piece ...
408 vtkInformation* const outInfo = outputVector->GetInformationObject(0);
409 if(outInfo->Has(vtkStreamingDemandDrivenPipeline::UPDATE_PIECE_NUMBER()) &&
410 outInfo->Get(vtkStreamingDemandDrivenPipeline::UPDATE_PIECE_NUMBER()) > 0)
411 {
412 return 1;
413 }
414
415 if (!this->PedigreeIdArrayName)
416 {
417 vtkErrorMacro(<<"You must specify a pedigree id array name");
418 return 0;
419 }
420
421 istream* input_stream_pt = nullptr;
422 ifstream file_stream;
423
424 // If the filename hasn't been specified, we're done ...
425 if(!this->FileName)
426 {
427 return 1;
428 }
429 // Get the total size of the input file in bytes
430 file_stream.open(this->FileName, ios::binary);
431 if(!file_stream.good())
432 {
433 vtkErrorMacro(<<"Unable to open input file" << std::string(this->FileName));
434 return 0;
435 }
436
437 file_stream.seekg(0, ios::end);
438 file_stream.seekg(0, ios::beg);
439
440 input_stream_pt = dynamic_cast<istream*>(&file_stream);
441 vtkTextCodec* transCodec = vtkTextCodecFactory::CodecToHandle(*input_stream_pt);
442
443 if (nullptr == transCodec)
444 {
445 // should this use the locale instead??
446 return 1;
447 }
448
449 DelimitedTextIterator iterator(
450 output_table,
451 this->MaxRecords,
452 this->HeaderLines,
453 this->ColumnNamesOnLine,
454 this->SkipColumnNames);
455
456 vtkTextCodec::OutputIterator& outIter = iterator;
457
458 transCodec->ToUnicode(*input_stream_pt, outIter);
459 iterator.ReachedEndOfInput();
460 transCodec->Delete();
461
462 if(this->OutputPedigreeIds)
463 {
464 vtkAbstractArray *arr =
465 output_table->GetColumnByName(this->PedigreeIdArrayName);
466
467 if (this->GeneratePedigreeIds || !arr)
468 {
469 vtkSmartPointer<vtkIdTypeArray> pedigreeIds =
470 vtkSmartPointer<vtkIdTypeArray>::New();
471 vtkIdType numRows = output_table->GetNumberOfRows();
472 pedigreeIds->SetNumberOfTuples(numRows);
473 pedigreeIds->SetName(this->PedigreeIdArrayName);
474 for (vtkIdType i = 0; i < numRows; ++i)
475 {
476 pedigreeIds->InsertValue(i, i);
477 }
478 output_table->GetRowData()->SetPedigreeIds(pedigreeIds);
479 }
480 else
481 {
482 if (arr)
483 {
484 output_table->GetRowData()->SetPedigreeIds(arr);
485 }
486 else
487 {
488 vtkErrorMacro(<< "Could not find pedigree id array: " << std::string(this->PedigreeIdArrayName));
489 return 0;
490 }
491 }
492 }
493 }
494 catch(std::exception& e)
495 {
496 vtkErrorMacro(<< "caught exception: " << e.what());
497 this->LastError = e.what();
498 output_table->Initialize();
499 return 0;
500 }
501 catch(...)
502 {
503 vtkErrorMacro(<< "caught unknown exception.");
504 this->LastError = "Unknown exception.";
505 output_table->Initialize();
506 return 0;
507 }
508
509 return 1;
510 }
511