1 /***************************************************************************
2 * Copyright (C) 2004 by Gary Sherman *
3 * sherman at mrcc.com *
4 * *
5 * GUI for loading a delimited text file as a layer in QGIS *
6 * This plugin works in conjunction with the delimited text data *
7 * provider plugin *
8 * *
9 * This program is free software; you can redistribute it and/or modify *
10 * it under the terms of the GNU General Public License as published by *
11 * the Free Software Foundation; either version 2 of the License, or *
12 * (at your option) any later version. *
13 ***************************************************************************/
14 #include "qgsdelimitedtextsourceselect.h"
15
16 #include "qgisinterface.h"
17 #include "qgslogger.h"
18 #include "qgsvectordataprovider.h"
19 #include "qgsdelimitedtextprovider.h"
20 #include "qgsdelimitedtextfile.h"
21 #include "qgssettings.h"
22 #include "qgsproviderregistry.h"
23 #include "qgsgui.h"
24
25 #include <QButtonGroup>
26 #include <QFile>
27 #include <QFileDialog>
28 #include <QFileInfo>
29 #include <QMessageBox>
30 #include <QRegularExpression>
31 #include <QTextStream>
32 #include <QTextCodec>
33 #include <QUrl>
34 #include <QUrlQuery>
35
36 const int MAX_SAMPLE_LENGTH = 200;
37
QgsDelimitedTextSourceSelect(QWidget * parent,Qt::WindowFlags fl,QgsProviderRegistry::WidgetMode theWidgetMode)38 QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget *parent, Qt::WindowFlags fl, QgsProviderRegistry::WidgetMode theWidgetMode )
39 : QgsAbstractDataSourceWidget( parent, fl, theWidgetMode )
40 , mFile( std::make_unique<QgsDelimitedTextFile>() )
41 , mSettingsKey( QStringLiteral( "/Plugin-DelimitedText" ) )
42 {
43
44 setupUi( this );
45 QgsGui::instance()->enableAutoGeometryRestore( this );
46 setupButtons( buttonBox );
47 connect( buttonBox, &QDialogButtonBox::helpRequested, this, &QgsDelimitedTextSourceSelect::showHelp );
48
49 bgFileFormat = new QButtonGroup( this );
50 bgFileFormat->addButton( delimiterCSV, swFileFormat->indexOf( swpCSVOptions ) );
51 bgFileFormat->addButton( delimiterChars, swFileFormat->indexOf( swpDelimOptions ) );
52 bgFileFormat->addButton( delimiterRegexp, swFileFormat->indexOf( swpRegexpOptions ) );
53
54 bgGeomType = new QButtonGroup( this );
55 bgGeomType->addButton( geomTypeXY, swGeomType->indexOf( swpGeomXY ) );
56 bgGeomType->addButton( geomTypeWKT, swGeomType->indexOf( swpGeomWKT ) );
57 bgGeomType->addButton( geomTypeNone, swGeomType->indexOf( swpGeomNone ) );
58
59 connect( bgFileFormat, static_cast < void ( QButtonGroup::* )( int ) > ( &QButtonGroup::buttonClicked ), swFileFormat, &QStackedWidget::setCurrentIndex );
60 connect( bgGeomType, static_cast < void ( QButtonGroup::* )( int ) > ( &QButtonGroup::buttonClicked ), swGeomType, &QStackedWidget::setCurrentIndex );
61 connect( bgGeomType, static_cast < void ( QButtonGroup::* )( int ) > ( &QButtonGroup::buttonClicked ), this, &QgsDelimitedTextSourceSelect::showCrsWidget );
62
63 cmbEncoding->clear();
64 cmbEncoding->addItems( QgsVectorDataProvider::availableEncodings() );
65 cmbEncoding->setCurrentIndex( cmbEncoding->findText( QStringLiteral( "UTF-8" ) ) );
66
67 loadSettings();
68 updateFieldsAndEnable();
69
70 connect( txtLayerName, &QLineEdit::textChanged, this, &QgsDelimitedTextSourceSelect::enableAccept );
71 connect( cmbEncoding, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
72
73 connect( delimiterCSV, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
74 connect( delimiterChars, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
75 connect( delimiterRegexp, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
76
77 connect( cbxDelimComma, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
78 connect( cbxDelimSpace, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
79 connect( cbxDelimTab, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
80 connect( cbxDelimSemicolon, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
81 connect( cbxDelimColon, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
82
83 connect( txtDelimiterOther, &QLineEdit::textChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
84 connect( txtQuoteChars, &QLineEdit::textChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
85 connect( txtEscapeChars, &QLineEdit::textChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
86 connect( txtDelimiterRegexp, &QLineEdit::textChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
87
88 connect( rowCounter, static_cast < void ( QSpinBox::* )( int ) > ( &QSpinBox::valueChanged ), this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
89 connect( cbxUseHeader, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
90 connect( cbxSkipEmptyFields, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
91 connect( cbxTrimFields, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
92
93 connect( cbxPointIsComma, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
94 connect( cbxXyDms, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
95
96 connect( crsGeometry, &QgsProjectionSelectionWidget::crsChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
97
98 const QgsSettings settings;
99 mFileWidget->setDialogTitle( tr( "Choose a Delimited Text File to Open" ) );
100 mFileWidget->setFilter( tr( "Text files" ) + QStringLiteral( " (*.txt *.csv *.dat *.wkt);;" ) + tr( "All files" ) + QStringLiteral( " (* *.*)" ) );
101 mFileWidget->setSelectedFilter( settings.value( mSettingsKey + QStringLiteral( "/file_filter" ), QString() ).toString() );
102 mMaxFields = settings.value( mSettingsKey + QStringLiteral( "/max_fields" ), DEFAULT_MAX_FIELDS ).toInt();
103 connect( mFileWidget, &QgsFileWidget::fileChanged, this, &QgsDelimitedTextSourceSelect::updateFileName );
104 }
105
addButtonClicked()106 void QgsDelimitedTextSourceSelect::addButtonClicked()
107 {
108 // The following conditions should not be hit! OK will not be enabled...
109 if ( txtLayerName->text().isEmpty() )
110 {
111 QMessageBox::warning( this, tr( "No layer name" ), tr( "Please enter a layer name before adding the layer to the map" ) );
112 txtLayerName->setFocus();
113 return;
114 }
115 if ( delimiterChars->isChecked() )
116 {
117 if ( selectedChars().isEmpty() )
118 {
119 QMessageBox::warning( this, tr( "No delimiters set" ), tr( "Use one or more characters as the delimiter, or choose a different delimiter type" ) );
120 txtDelimiterOther->setFocus();
121 return;
122 }
123 }
124 if ( delimiterRegexp->isChecked() )
125 {
126 const QRegularExpression re( txtDelimiterRegexp->text() );
127 if ( ! re.isValid() )
128 {
129 QMessageBox::warning( this, tr( "Invalid regular expression" ), tr( "Please enter a valid regular expression as the delimiter, or choose a different delimiter type" ) );
130 txtDelimiterRegexp->setFocus();
131 return;
132 }
133 }
134 if ( ! mFile->isValid() )
135 {
136 QMessageBox::warning( this, tr( "Invalid delimited text file" ), tr( "Please enter a valid file and delimiter" ) );
137 return;
138 }
139
140 //Build the delimited text URI from the user provided information
141
142 QUrl url = mFile->url();
143 QUrlQuery query( url );
144
145 query.addQueryItem( QStringLiteral( "detectTypes" ), cbxDetectTypes->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
146
147 if ( cbxPointIsComma->isChecked() )
148 {
149 query.addQueryItem( QStringLiteral( "decimalPoint" ), QStringLiteral( "," ) );
150 }
151 if ( cbxXyDms->isChecked() )
152 {
153 query.addQueryItem( QStringLiteral( "xyDms" ), QStringLiteral( "yes" ) );
154 }
155
156 bool haveGeom = true;
157 if ( geomTypeXY->isChecked() )
158 {
159 QString field;
160 if ( !cmbXField->currentText().isEmpty() && !cmbYField->currentText().isEmpty() )
161 {
162 field = cmbXField->currentText();
163 query.addQueryItem( QStringLiteral( "xField" ), field );
164 field = cmbYField->currentText();
165 query.addQueryItem( QStringLiteral( "yField" ), field );
166 }
167 if ( !cmbZField->currentText().isEmpty() )
168 {
169 field = cmbZField->currentText();
170 query.addQueryItem( QStringLiteral( "zField" ), field );
171 }
172 if ( !cmbMField->currentText().isEmpty() )
173 {
174 field = cmbMField->currentText();
175 query.addQueryItem( QStringLiteral( "mField" ), field );
176 }
177 }
178 else if ( geomTypeWKT->isChecked() )
179 {
180 if ( ! cmbWktField->currentText().isEmpty() )
181 {
182 const QString field = cmbWktField->currentText();
183 query.addQueryItem( QStringLiteral( "wktField" ), field );
184 }
185 if ( cmbGeometryType->currentIndex() > 0 )
186 {
187 query.addQueryItem( QStringLiteral( "geomType" ), cmbGeometryType->currentText() );
188 }
189 }
190 else
191 {
192 haveGeom = false;
193 query.addQueryItem( QStringLiteral( "geomType" ), QStringLiteral( "none" ) );
194 }
195 if ( haveGeom )
196 {
197 const QgsCoordinateReferenceSystem crs = crsGeometry->crs();
198 if ( crs.isValid() )
199 {
200 query.addQueryItem( QStringLiteral( "crs" ), crs.authid() );
201 }
202
203 }
204
205 if ( ! geomTypeNone->isChecked() )
206 {
207 query.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
208 }
209
210 query.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
211 query.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
212
213 url.setQuery( query );
214 // store the settings
215 saveSettings();
216 saveSettingsForFile( mFileWidget->filePath() );
217
218
219 // add the layer to the map
220 emit addVectorLayer( QString::fromLatin1( url.toEncoded() ), txtLayerName->text() );
221
222 // clear the file and layer name show something has happened, ready for another file
223
224 mFileWidget->setFilePath( QString() );
225 txtLayerName->setText( QString() );
226
227 if ( widgetMode() == QgsProviderRegistry::WidgetMode::None )
228 {
229 accept();
230 }
231 }
232
233
selectedChars()234 QString QgsDelimitedTextSourceSelect::selectedChars()
235 {
236 QString chars;
237 if ( cbxDelimComma->isChecked() )
238 chars.append( ',' );
239 if ( cbxDelimSpace->isChecked() )
240 chars.append( ' ' );
241 if ( cbxDelimTab->isChecked() )
242 chars.append( '\t' );
243 if ( cbxDelimSemicolon->isChecked() )
244 chars.append( ';' );
245 if ( cbxDelimColon->isChecked() )
246 chars.append( ':' );
247 chars = QgsDelimitedTextFile::encodeChars( chars );
248 chars.append( txtDelimiterOther->text() );
249 return chars;
250 }
setSelectedChars(const QString & delimiters)251 void QgsDelimitedTextSourceSelect::setSelectedChars( const QString &delimiters )
252 {
253 QString chars = QgsDelimitedTextFile::decodeChars( delimiters );
254 cbxDelimComma->setChecked( chars.contains( ',' ) );
255 cbxDelimSpace->setChecked( chars.contains( ' ' ) );
256 cbxDelimTab->setChecked( chars.contains( '\t' ) );
257 cbxDelimColon->setChecked( chars.contains( ':' ) );
258 cbxDelimSemicolon->setChecked( chars.contains( ';' ) );
259 chars = chars.remove( QRegularExpression( QStringLiteral( "[ ,:;\t]" ) ) );
260 chars = QgsDelimitedTextFile::encodeChars( chars );
261 txtDelimiterOther->setText( chars );
262 }
263
loadSettings(const QString & subkey,bool loadGeomSettings)264 void QgsDelimitedTextSourceSelect::loadSettings( const QString &subkey, bool loadGeomSettings )
265 {
266 const QgsSettings settings;
267
268 // at startup, fetch the last used delimiter and directory from
269 // settings
270 QString key = mSettingsKey;
271 if ( ! subkey.isEmpty() ) key.append( '/' ).append( subkey );
272
273 // and how to use the delimiter
274 const QString delimiterType = settings.value( key + "/delimiterType", "" ).toString();
275 if ( delimiterType == QLatin1String( "chars" ) )
276 {
277 delimiterChars->setChecked( true );
278 }
279 else if ( delimiterType == QLatin1String( "regexp" ) )
280 {
281 delimiterRegexp->setChecked( true );
282 }
283 else if ( delimiterType == QLatin1String( "csv" ) )
284 {
285 delimiterCSV->setChecked( true );
286 }
287 swFileFormat->setCurrentIndex( bgFileFormat->checkedId() );
288
289 const QString encoding = settings.value( key + "/encoding", "" ).toString();
290 if ( ! encoding.isEmpty() ) cmbEncoding->setCurrentIndex( cmbEncoding->findText( encoding ) );
291 const QString delimiters = settings.value( key + "/delimiters", "" ).toString();
292 if ( ! delimiters.isEmpty() ) setSelectedChars( delimiters );
293
294 txtQuoteChars->setText( settings.value( key + "/quoteChars", "\"" ).toString() );
295 txtEscapeChars->setText( settings.value( key + "/escapeChars", "\"" ).toString() );
296
297 const QString regexp = settings.value( key + "/delimiterRegexp", "" ).toString();
298 if ( ! regexp.isEmpty() ) txtDelimiterRegexp->setText( regexp );
299
300 rowCounter->setValue( settings.value( key + "/startFrom", 0 ).toInt() );
301 cbxUseHeader->setChecked( settings.value( key + "/useHeader", "true" ) != "false" );
302 cbxDetectTypes->setChecked( settings.value( key + "/detectTypes", "true" ) != "false" );
303 cbxTrimFields->setChecked( settings.value( key + "/trimFields", "false" ) == "true" );
304 cbxSkipEmptyFields->setChecked( settings.value( key + "/skipEmptyFields", "false" ) == "true" );
305 cbxPointIsComma->setChecked( settings.value( key + "/decimalPoint", "." ).toString().contains( ',' ) );
306 cbxSubsetIndex->setChecked( settings.value( key + "/subsetIndex", "false" ) == "true" );
307 cbxSpatialIndex->setChecked( settings.value( key + "/spatialIndex", "false" ) == "true" );
308 cbxWatchFile->setChecked( settings.value( key + "/watchFile", "false" ) == "true" );
309
310 if ( loadGeomSettings )
311 {
312 const QString geomColumnType = settings.value( key + "/geomColumnType", "xy" ).toString();
313 if ( geomColumnType == QLatin1String( "xy" ) ) geomTypeXY->setChecked( true );
314 else if ( geomColumnType == QLatin1String( "wkt" ) ) geomTypeWKT->setChecked( true );
315 else geomTypeNone->setChecked( true );
316 cbxXyDms->setChecked( settings.value( key + "/xyDms", "false" ) == "true" );
317 swGeomType->setCurrentIndex( bgGeomType->checkedId() );
318 const QString authid = settings.value( key + "/crs", "" ).toString();
319 const QgsCoordinateReferenceSystem crs = QgsCoordinateReferenceSystem::fromOgcWmsCrs( authid );
320 if ( crs.isValid() )
321 {
322 crsGeometry->setCrs( crs );
323 }
324 }
325
326 }
327
saveSettings(const QString & subkey,bool saveGeomSettings)328 void QgsDelimitedTextSourceSelect::saveSettings( const QString &subkey, bool saveGeomSettings )
329 {
330 QgsSettings settings;
331 QString key = mSettingsKey;
332 if ( ! subkey.isEmpty() ) key.append( '/' ).append( subkey );
333 settings.setValue( key + "/encoding", cmbEncoding->currentText() );
334 settings.setValue( key + "/geometry", saveGeometry() );
335
336 if ( delimiterCSV->isChecked() )
337 settings.setValue( key + "/delimiterType", "csv" );
338 else if ( delimiterChars->isChecked() )
339 settings.setValue( key + "/delimiterType", "chars" );
340 else
341 settings.setValue( key + "/delimiterType", "regexp" );
342 settings.setValue( key + "/delimiters", selectedChars() );
343 settings.setValue( key + "/quoteChars", txtQuoteChars->text() );
344 settings.setValue( key + "/escapeChars", txtEscapeChars->text() );
345 settings.setValue( key + "/delimiterRegexp", txtDelimiterRegexp->text() );
346 settings.setValue( key + "/startFrom", rowCounter->value() );
347 settings.setValue( key + "/useHeader", cbxUseHeader->isChecked() ? "true" : "false" );
348 settings.setValue( key + "/detectTypes", cbxDetectTypes->isChecked() ? "true" : "false" );
349 settings.setValue( key + "/trimFields", cbxTrimFields->isChecked() ? "true" : "false" );
350 settings.setValue( key + "/skipEmptyFields", cbxSkipEmptyFields->isChecked() ? "true" : "false" );
351 settings.setValue( key + "/decimalPoint", cbxPointIsComma->isChecked() ? "," : "." );
352 settings.setValue( key + "/subsetIndex", cbxSubsetIndex->isChecked() ? "true" : "false" );
353 settings.setValue( key + "/spatialIndex", cbxSpatialIndex->isChecked() ? "true" : "false" );
354 settings.setValue( key + "/watchFile", cbxWatchFile->isChecked() ? "true" : "false" );
355 if ( saveGeomSettings )
356 {
357 QString geomColumnType = QStringLiteral( "none" );
358 if ( geomTypeXY->isChecked() ) geomColumnType = QStringLiteral( "xy" );
359 if ( geomTypeWKT->isChecked() ) geomColumnType = QStringLiteral( "wkt" );
360 settings.setValue( key + "/geomColumnType", geomColumnType );
361 settings.setValue( key + "/xyDms", cbxXyDms->isChecked() ? "true" : "false" );
362 if ( crsGeometry->crs().isValid() )
363 {
364 settings.setValue( key + "/crs", crsGeometry->crs().authid() );
365 }
366 }
367
368 }
369
loadSettingsForFile(const QString & filename)370 void QgsDelimitedTextSourceSelect::loadSettingsForFile( const QString &filename )
371 {
372 if ( filename.isEmpty() ) return;
373 const QFileInfo fi( filename );
374 const QString filetype = fi.suffix();
375 // Don't expect to change settings if not changing file type
376 if ( filetype != mLastFileType ) loadSettings( fi.suffix(), true );
377 mLastFileType = filetype;
378 }
379
saveSettingsForFile(const QString & filename)380 void QgsDelimitedTextSourceSelect::saveSettingsForFile( const QString &filename )
381 {
382 if ( filename.isEmpty() ) return;
383 const QFileInfo fi( filename );
384 saveSettings( fi.suffix(), true );
385 }
386
387
loadDelimitedFileDefinition()388 bool QgsDelimitedTextSourceSelect::loadDelimitedFileDefinition()
389 {
390 mFile->setFileName( mFileWidget->filePath() );
391 mFile->setEncoding( cmbEncoding->currentText() );
392 if ( delimiterChars->isChecked() )
393 {
394 mFile->setTypeCSV( selectedChars(), txtQuoteChars->text(), txtEscapeChars->text() );
395 }
396 else if ( delimiterRegexp->isChecked() )
397 {
398 mFile->setTypeRegexp( txtDelimiterRegexp->text() );
399 }
400 else
401 {
402 mFile->setTypeCSV();
403 }
404 mFile->setSkipLines( rowCounter->value() );
405 mFile->setUseHeader( cbxUseHeader->isChecked() );
406 mFile->setDiscardEmptyFields( cbxSkipEmptyFields->isChecked() );
407 mFile->setTrimFields( cbxTrimFields->isChecked() );
408 mFile->setMaxFields( mMaxFields );
409 return mFile->isValid();
410 }
411
412
updateFieldLists()413 void QgsDelimitedTextSourceSelect::updateFieldLists()
414 {
415 // Update the x and y field drop-down boxes
416 QgsDebugMsgLevel( QStringLiteral( "Updating field lists" ), 3 );
417
418 disconnect( cmbXField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
419 disconnect( cmbYField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
420 disconnect( cmbWktField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
421 disconnect( geomTypeXY, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
422 disconnect( geomTypeWKT, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
423 disconnect( geomTypeNone, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
424
425 const QString columnX = cmbXField->currentText();
426 const QString columnY = cmbYField->currentText();
427 const QString columnZ = cmbZField->currentText();
428 const QString columnM = cmbMField->currentText();
429 const QString columnWkt = cmbWktField->currentText();
430
431 // clear the field lists
432 cmbXField->clear();
433 cmbYField->clear();
434 cmbZField->clear();
435 cmbMField->clear();
436 cmbWktField->clear();
437
438 // clear the sample text box
439 tblSample->clear();
440 tblSample->setColumnCount( 0 );
441 tblSample->setRowCount( 0 );
442
443 if ( ! loadDelimitedFileDefinition() )
444 return;
445
446 // Put a sample set of records into the sample box. Also while scanning assess suitability of
447 // fields for use as coordinate and WKT fields
448
449
450 QList<bool> isValidCoordinate;
451 QList<bool> isValidWkt;
452 QList<bool> isEmpty;
453 int counter = 0;
454 mBadRowCount = 0;
455 QStringList values;
456 const QRegularExpression wktre( "^\\s*(?:MULTI)?(?:POINT|LINESTRING|POLYGON)\\s*Z?\\s*M?\\(", QRegularExpression::CaseInsensitiveOption );
457
458 while ( counter < mExampleRowCount )
459 {
460 const QgsDelimitedTextFile::Status status = mFile->nextRecord( values );
461 if ( status == QgsDelimitedTextFile::RecordEOF ) break;
462 if ( status != QgsDelimitedTextFile::RecordOk ) { mBadRowCount++; continue; }
463 counter++;
464
465 // Look at count of non-blank fields
466
467 int nv = values.size();
468 while ( nv > 0 && values[nv - 1].isEmpty() ) nv--;
469
470 if ( isEmpty.size() < nv )
471 {
472 while ( isEmpty.size() < nv )
473 {
474 isEmpty.append( true );
475 isValidCoordinate.append( false );
476 isValidWkt.append( false );
477 }
478 tblSample->setColumnCount( nv );
479 }
480
481 tblSample->setRowCount( counter );
482
483 const bool xyDms = cbxXyDms->isChecked();
484
485 for ( int i = 0; i < tblSample->columnCount(); i++ )
486 {
487 QString value = i < nv ? values[i] : QString();
488 if ( value.length() > MAX_SAMPLE_LENGTH )
489 value = value.mid( 0, MAX_SAMPLE_LENGTH ) + QChar( 0x2026 );
490 QTableWidgetItem *item = new QTableWidgetItem( value );
491 tblSample->setItem( counter - 1, i, item );
492 if ( ! value.isEmpty() )
493 {
494 if ( isEmpty[i] )
495 {
496 isEmpty[i] = false;
497 isValidCoordinate[i] = true;
498 isValidWkt[i] = true;
499 }
500 if ( isValidCoordinate[i] )
501 {
502 bool ok = true;
503 if ( cbxPointIsComma->isChecked() )
504 {
505 value.replace( ',', '.' );
506 }
507 if ( xyDms )
508 {
509 const QRegularExpressionMatch match = QgsDelimitedTextProvider::sCrdDmsRegexp.match( value );
510 ok = match.capturedStart() == 0;
511 }
512 else
513 {
514 ( void )value.toDouble( &ok );
515 }
516 isValidCoordinate[i] = ok;
517 }
518 if ( isValidWkt[i] )
519 {
520 value.remove( QgsDelimitedTextProvider::sWktPrefixRegexp );
521 isValidWkt[i] = value.contains( wktre );
522 }
523 }
524 }
525 }
526
527 QStringList fieldList = mFile->fieldNames();
528
529 if ( isEmpty.size() < fieldList.size() )
530 {
531 while ( isEmpty.size() < fieldList.size() )
532 {
533 isEmpty.append( true );
534 isValidCoordinate.append( false );
535 isValidWkt.append( false );
536 }
537 tblSample->setColumnCount( fieldList.size() );
538 }
539
540 tblSample->setHorizontalHeaderLabels( fieldList );
541 tblSample->resizeColumnsToContents();
542 tblSample->resizeRowsToContents();
543
544 // We don't know anything about a text based field other
545 // than its name. All fields are assumed to be text
546 // As we ignore blank fields we need to map original index
547 // of selected fields to index in combo box.
548
549 int fieldNo = 0;
550 for ( int i = 0; i < fieldList.size(); i++ )
551 {
552 const QString field = fieldList[i];
553 // skip empty field names
554 if ( field.isEmpty() ) continue;
555 cmbXField->addItem( field );
556 cmbYField->addItem( field );
557 cmbZField->addItem( field );
558 cmbMField->addItem( field );
559 cmbWktField->addItem( field );
560 fieldNo++;
561 }
562
563 // Try resetting current values for column names
564
565 cmbWktField->setCurrentIndex( cmbWktField->findText( columnWkt ) );
566 cmbXField->setCurrentIndex( cmbXField->findText( columnX ) );
567 cmbYField->setCurrentIndex( cmbYField->findText( columnY ) );
568 cmbZField->setCurrentIndex( cmbYField->findText( columnZ ) );
569 cmbMField->setCurrentIndex( cmbYField->findText( columnM ) );
570
571 // Now try setting optional X,Y fields - will only reset the fields if
572 // not already set.
573
574 trySetXYField( fieldList, isValidCoordinate, QStringLiteral( "longitude" ), QStringLiteral( "latitude" ) );
575 trySetXYField( fieldList, isValidCoordinate, QStringLiteral( "lon" ), QStringLiteral( "lat" ) );
576 trySetXYField( fieldList, isValidCoordinate, QStringLiteral( "east" ), QStringLiteral( "north" ) );
577 trySetXYField( fieldList, isValidCoordinate, QStringLiteral( "x" ), QStringLiteral( "y" ) );
578 trySetXYField( fieldList, isValidCoordinate, QStringLiteral( "e" ), QStringLiteral( "n" ) );
579
580 // And also a WKT field if there is one
581
582 if ( cmbWktField->currentIndex() < 0 )
583 {
584 for ( int i = 0; i < fieldList.size(); i++ )
585 {
586 if ( ! isValidWkt[i] ) continue;
587 const int index = cmbWktField->findText( fieldList[i] );
588 if ( index >= 0 )
589 {
590 cmbWktField->setCurrentIndex( index );
591 break;
592 }
593 }
594 }
595
596 const bool haveFields = fieldNo > 0;
597
598 if ( !geomTypeNone->isChecked() )
599 {
600 const bool isXY = cmbWktField->currentIndex() < 0 ||
601 ( geomTypeXY->isChecked() &&
602 ( cmbXField->currentIndex() >= 0 && cmbYField->currentIndex() >= 0 ) );
603 geomTypeXY->setChecked( isXY );
604 geomTypeWKT->setChecked( ! isXY );
605 }
606 swGeomType->setCurrentIndex( bgGeomType->checkedId() );
607
608 if ( haveFields )
609 {
610 connect( cmbXField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
611 connect( cmbYField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
612 connect( cmbWktField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
613 connect( geomTypeXY, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
614 connect( geomTypeWKT, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
615 connect( geomTypeNone, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
616 }
617
618 }
619
trySetXYField(QStringList & fields,QList<bool> & isValidNumber,const QString & xname,const QString & yname)620 bool QgsDelimitedTextSourceSelect::trySetXYField( QStringList &fields, QList<bool> &isValidNumber, const QString &xname, const QString &yname )
621 {
622 // If fields already set, then nothing to do
623 if ( cmbXField->currentIndex() >= 0 && cmbYField->currentIndex() >= 0 ) return true;
624
625 // Try and find a valid field name matching the x field
626 int indexX = -1;
627 int indexY = -1;
628
629 for ( int i = 0; i < fields.size(); i++ )
630 {
631 // Only interested in number fields containing the xname string
632 // that are in the X combo box
633 if ( ! isValidNumber[i] ) continue;
634 if ( ! fields[i].contains( xname, Qt::CaseInsensitive ) ) continue;
635 indexX = cmbXField->findText( fields[i] );
636 if ( indexX < 0 ) continue;
637
638 // Now look for potential y fields, like xname with x replaced with y
639 const QString xfield( fields[i] );
640 int from = 0;
641 while ( true )
642 {
643 const int pos = xfield.indexOf( xname, from, Qt::CaseInsensitive );
644 if ( pos < 0 ) break;
645 from = pos + 1;
646 const QString yfield = xfield.mid( 0, pos ) + yname + xfield.mid( pos + xname.size() );
647 if ( ! fields.contains( yfield, Qt::CaseInsensitive ) ) continue;
648 for ( int iy = 0; iy < fields.size(); iy++ )
649 {
650 if ( ! isValidNumber[iy] ) continue;
651 if ( iy == i ) continue;
652 if ( fields[iy].compare( yfield, Qt::CaseInsensitive ) == 0 )
653 {
654 indexY = cmbYField->findText( fields[iy] );
655 break;
656 }
657 }
658 if ( indexY >= 0 ) break;
659 }
660 if ( indexY >= 0 ) break;
661 }
662 if ( indexY >= 0 )
663 {
664 cmbXField->setCurrentIndex( indexX );
665 cmbYField->setCurrentIndex( indexY );
666 }
667 return indexY >= 0;
668 }
669
updateFileName()670 void QgsDelimitedTextSourceSelect::updateFileName()
671 {
672 QgsSettings settings;
673 settings.setValue( mSettingsKey + "/file_filter", mFileWidget->selectedFilter() );
674
675 // put a default layer name in the text entry
676 const QString filename = mFileWidget->filePath();
677 const QFileInfo finfo( filename );
678 if ( finfo.exists() )
679 {
680 QgsSettings settings;
681 settings.setValue( mSettingsKey + "/text_path", finfo.path() );
682 }
683
684 txtLayerName->setText( finfo.completeBaseName() );
685 loadSettingsForFile( filename );
686 updateFieldsAndEnable();
687 }
688
updateFieldsAndEnable()689 void QgsDelimitedTextSourceSelect::updateFieldsAndEnable()
690 {
691 updateFieldLists();
692 enableAccept();
693 }
694
validate()695 bool QgsDelimitedTextSourceSelect::validate()
696 {
697 // Check that input data is valid - provide a status message if not..
698
699 QString message;
700 bool enabled = false;
701
702 if ( mFileWidget->filePath().trimmed().isEmpty() )
703 {
704 message = tr( "Please select an input file" );
705 }
706 else if ( ! QFileInfo::exists( mFileWidget->filePath() ) )
707 {
708 message = tr( "File %1 does not exist" ).arg( mFileWidget->filePath() );
709 }
710 else if ( txtLayerName->text().isEmpty() )
711 {
712 message = tr( "Please enter a layer name" );
713 }
714 else if ( delimiterChars->isChecked() && selectedChars().isEmpty() )
715 {
716 message = tr( "At least one delimiter character must be specified" );
717 }
718
719 if ( message.isEmpty() && delimiterRegexp->isChecked() )
720 {
721 const QRegularExpression re( txtDelimiterRegexp->text() );
722 if ( ! re.isValid() )
723 {
724 message = tr( "Regular expression is not valid" );
725 }
726 else if ( re.pattern().startsWith( '^' ) && re.captureCount() == 0 )
727 {
728 message = tr( "^.. expression needs capture groups" );
729 }
730 lblRegexpError->setText( message );
731 }
732 if ( ! message.isEmpty() )
733 {
734 // continue...
735 }
736 // Hopefully won't hit this none-specific message, but just in case ...
737 else if ( ! mFile->isValid() )
738 {
739 message = tr( "Definition of filename and delimiters is not valid" );
740 }
741 // Assume that the sample table will have been populated if data was found
742 else if ( tblSample->rowCount() == 0 )
743 {
744 message = tr( "No data found in file" );
745 if ( mBadRowCount > 0 )
746 {
747 message = message + " (" + tr( "%1 badly formatted records discarded" ).arg( mBadRowCount ) + ')';
748 }
749 }
750 else if ( geomTypeXY->isChecked() && ( cmbXField->currentText().isEmpty() || cmbYField->currentText().isEmpty() ) )
751 {
752 message = tr( "X and Y field names must be selected" );
753 }
754 else if ( geomTypeXY->isChecked() && ( cmbXField->currentText() == cmbYField->currentText() ) )
755 {
756 message = tr( "X and Y field names cannot be the same" );
757 }
758 else if ( geomTypeWKT->isChecked() && cmbWktField->currentText().isEmpty() )
759 {
760 message = tr( "The WKT field name must be selected" );
761 }
762 else if ( ! geomTypeNone->isChecked() && ! crsGeometry->crs().isValid() )
763 {
764 message = tr( "The CRS must be selected" );
765 }
766 else
767 {
768 enabled = true;
769 if ( mBadRowCount > 0 )
770 {
771 message = tr( "%1 badly formatted records discarded from sample data" ).arg( mBadRowCount );
772 }
773
774 }
775 lblStatus->setText( message );
776 return enabled;
777 }
778
779
enableAccept()780 void QgsDelimitedTextSourceSelect::enableAccept()
781 {
782 emit enableButtons( validate() );
783 }
784
showHelp()785 void QgsDelimitedTextSourceSelect::showHelp()
786 {
787 QgsHelp::openHelp( QStringLiteral( "managing_data_source/opening_data.html#importing-a-delimited-text-file" ) );
788 }
789
showCrsWidget()790 void QgsDelimitedTextSourceSelect::showCrsWidget()
791 {
792 crsGeometry->setVisible( !geomTypeNone->isChecked() );
793 textLabelCrs->setVisible( !geomTypeNone->isChecked() );
794 }
795