1 /***************************************************************************
2  *   Copyright (C) 2004 by Gary Sherman                                    *
3  *   sherman at mrcc.com                                                   *
4  *                                                                         *
5  *   GUI for loading a delimited text file as a layer in QGIS              *
6  *   This plugin works in conjunction with the delimited text data          *
7  *   provider plugin                                                       *
8  *                                                                         *
9  *   This program is free software; you can redistribute it and/or modify  *
10  *   it under the terms of the GNU General Public License as published by  *
11  *   the Free Software Foundation; either version 2 of the License, or     *
12  *   (at your option) any later version.                                   *
13  ***************************************************************************/
14 #include "qgsdelimitedtextsourceselect.h"
15 
16 #include "qgisinterface.h"
17 #include "qgslogger.h"
18 #include "qgsvectordataprovider.h"
19 #include "qgsdelimitedtextprovider.h"
20 #include "qgsdelimitedtextfile.h"
21 #include "qgssettings.h"
22 #include "qgsproviderregistry.h"
23 #include "qgsgui.h"
24 
25 #include <QButtonGroup>
26 #include <QFile>
27 #include <QFileDialog>
28 #include <QFileInfo>
29 #include <QMessageBox>
30 #include <QRegularExpression>
31 #include <QTextStream>
32 #include <QTextCodec>
33 #include <QUrl>
34 #include <QUrlQuery>
35 
36 const int MAX_SAMPLE_LENGTH = 200;
37 
QgsDelimitedTextSourceSelect(QWidget * parent,Qt::WindowFlags fl,QgsProviderRegistry::WidgetMode theWidgetMode)38 QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget *parent, Qt::WindowFlags fl, QgsProviderRegistry::WidgetMode theWidgetMode )
39   : QgsAbstractDataSourceWidget( parent, fl, theWidgetMode )
40   , mFile( std::make_unique<QgsDelimitedTextFile>() )
41   , mSettingsKey( QStringLiteral( "/Plugin-DelimitedText" ) )
42 {
43 
44   setupUi( this );
45   QgsGui::instance()->enableAutoGeometryRestore( this );
46   setupButtons( buttonBox );
47   connect( buttonBox, &QDialogButtonBox::helpRequested, this, &QgsDelimitedTextSourceSelect::showHelp );
48 
49   bgFileFormat = new QButtonGroup( this );
50   bgFileFormat->addButton( delimiterCSV, swFileFormat->indexOf( swpCSVOptions ) );
51   bgFileFormat->addButton( delimiterChars, swFileFormat->indexOf( swpDelimOptions ) );
52   bgFileFormat->addButton( delimiterRegexp, swFileFormat->indexOf( swpRegexpOptions ) );
53 
54   bgGeomType = new QButtonGroup( this );
55   bgGeomType->addButton( geomTypeXY, swGeomType->indexOf( swpGeomXY ) );
56   bgGeomType->addButton( geomTypeWKT, swGeomType->indexOf( swpGeomWKT ) );
57   bgGeomType->addButton( geomTypeNone, swGeomType->indexOf( swpGeomNone ) );
58 
59   connect( bgFileFormat, static_cast < void ( QButtonGroup::* )( int ) > ( &QButtonGroup::buttonClicked ), swFileFormat, &QStackedWidget::setCurrentIndex );
60   connect( bgGeomType, static_cast < void ( QButtonGroup::* )( int ) > ( &QButtonGroup::buttonClicked ), swGeomType, &QStackedWidget::setCurrentIndex );
61   connect( bgGeomType, static_cast < void ( QButtonGroup::* )( int ) > ( &QButtonGroup::buttonClicked ), this, &QgsDelimitedTextSourceSelect::showCrsWidget );
62 
63   cmbEncoding->clear();
64   cmbEncoding->addItems( QgsVectorDataProvider::availableEncodings() );
65   cmbEncoding->setCurrentIndex( cmbEncoding->findText( QStringLiteral( "UTF-8" ) ) );
66 
67   loadSettings();
68   updateFieldsAndEnable();
69 
70   connect( txtLayerName, &QLineEdit::textChanged, this, &QgsDelimitedTextSourceSelect::enableAccept );
71   connect( cmbEncoding, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
72 
73   connect( delimiterCSV, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
74   connect( delimiterChars, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
75   connect( delimiterRegexp, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
76 
77   connect( cbxDelimComma, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
78   connect( cbxDelimSpace, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
79   connect( cbxDelimTab, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
80   connect( cbxDelimSemicolon, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
81   connect( cbxDelimColon, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
82 
83   connect( txtDelimiterOther, &QLineEdit::textChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
84   connect( txtQuoteChars, &QLineEdit::textChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
85   connect( txtEscapeChars, &QLineEdit::textChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
86   connect( txtDelimiterRegexp, &QLineEdit::textChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
87 
88   connect( rowCounter, static_cast < void ( QSpinBox::* )( int ) > ( &QSpinBox::valueChanged ), this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
89   connect( cbxUseHeader, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
90   connect( cbxSkipEmptyFields, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
91   connect( cbxTrimFields, &QCheckBox::stateChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
92 
93   connect( cbxPointIsComma, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
94   connect( cbxXyDms, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
95 
96   connect( crsGeometry, &QgsProjectionSelectionWidget::crsChanged, this, &QgsDelimitedTextSourceSelect::updateFieldsAndEnable );
97 
98   const QgsSettings settings;
99   mFileWidget->setDialogTitle( tr( "Choose a Delimited Text File to Open" ) );
100   mFileWidget->setFilter( tr( "Text files" ) + QStringLiteral( " (*.txt *.csv *.dat *.wkt);;" ) + tr( "All files" ) + QStringLiteral( " (* *.*)" ) );
101   mFileWidget->setSelectedFilter( settings.value( mSettingsKey + QStringLiteral( "/file_filter" ), QString() ).toString() );
102   mMaxFields = settings.value( mSettingsKey + QStringLiteral( "/max_fields" ), DEFAULT_MAX_FIELDS ).toInt();
103   connect( mFileWidget, &QgsFileWidget::fileChanged, this, &QgsDelimitedTextSourceSelect::updateFileName );
104 }
105 
addButtonClicked()106 void QgsDelimitedTextSourceSelect::addButtonClicked()
107 {
108   // The following conditions should not be hit! OK will not be enabled...
109   if ( txtLayerName->text().isEmpty() )
110   {
111     QMessageBox::warning( this, tr( "No layer name" ), tr( "Please enter a layer name before adding the layer to the map" ) );
112     txtLayerName->setFocus();
113     return;
114   }
115   if ( delimiterChars->isChecked() )
116   {
117     if ( selectedChars().isEmpty() )
118     {
119       QMessageBox::warning( this, tr( "No delimiters set" ), tr( "Use one or more characters as the delimiter, or choose a different delimiter type" ) );
120       txtDelimiterOther->setFocus();
121       return;
122     }
123   }
124   if ( delimiterRegexp->isChecked() )
125   {
126     const QRegularExpression re( txtDelimiterRegexp->text() );
127     if ( ! re.isValid() )
128     {
129       QMessageBox::warning( this, tr( "Invalid regular expression" ), tr( "Please enter a valid regular expression as the delimiter, or choose a different delimiter type" ) );
130       txtDelimiterRegexp->setFocus();
131       return;
132     }
133   }
134   if ( ! mFile->isValid() )
135   {
136     QMessageBox::warning( this, tr( "Invalid delimited text file" ), tr( "Please enter a valid file and delimiter" ) );
137     return;
138   }
139 
140   //Build the delimited text URI from the user provided information
141 
142   QUrl url = mFile->url();
143   QUrlQuery query( url );
144 
145   query.addQueryItem( QStringLiteral( "detectTypes" ), cbxDetectTypes->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
146 
147   if ( cbxPointIsComma->isChecked() )
148   {
149     query.addQueryItem( QStringLiteral( "decimalPoint" ), QStringLiteral( "," ) );
150   }
151   if ( cbxXyDms->isChecked() )
152   {
153     query.addQueryItem( QStringLiteral( "xyDms" ), QStringLiteral( "yes" ) );
154   }
155 
156   bool haveGeom = true;
157   if ( geomTypeXY->isChecked() )
158   {
159     QString field;
160     if ( !cmbXField->currentText().isEmpty() && !cmbYField->currentText().isEmpty() )
161     {
162       field = cmbXField->currentText();
163       query.addQueryItem( QStringLiteral( "xField" ), field );
164       field = cmbYField->currentText();
165       query.addQueryItem( QStringLiteral( "yField" ), field );
166     }
167     if ( !cmbZField->currentText().isEmpty() )
168     {
169       field = cmbZField->currentText();
170       query.addQueryItem( QStringLiteral( "zField" ), field );
171     }
172     if ( !cmbMField->currentText().isEmpty() )
173     {
174       field = cmbMField->currentText();
175       query.addQueryItem( QStringLiteral( "mField" ), field );
176     }
177   }
178   else if ( geomTypeWKT->isChecked() )
179   {
180     if ( ! cmbWktField->currentText().isEmpty() )
181     {
182       const QString field = cmbWktField->currentText();
183       query.addQueryItem( QStringLiteral( "wktField" ), field );
184     }
185     if ( cmbGeometryType->currentIndex() > 0 )
186     {
187       query.addQueryItem( QStringLiteral( "geomType" ), cmbGeometryType->currentText() );
188     }
189   }
190   else
191   {
192     haveGeom = false;
193     query.addQueryItem( QStringLiteral( "geomType" ), QStringLiteral( "none" ) );
194   }
195   if ( haveGeom )
196   {
197     const QgsCoordinateReferenceSystem crs = crsGeometry->crs();
198     if ( crs.isValid() )
199     {
200       query.addQueryItem( QStringLiteral( "crs" ), crs.authid() );
201     }
202 
203   }
204 
205   if ( ! geomTypeNone->isChecked() )
206   {
207     query.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
208   }
209 
210   query.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
211   query.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
212 
213   url.setQuery( query );
214   // store the settings
215   saveSettings();
216   saveSettingsForFile( mFileWidget->filePath() );
217 
218 
219   // add the layer to the map
220   emit addVectorLayer( QString::fromLatin1( url.toEncoded() ), txtLayerName->text() );
221 
222   // clear the file and layer name show something has happened, ready for another file
223 
224   mFileWidget->setFilePath( QString() );
225   txtLayerName->setText( QString() );
226 
227   if ( widgetMode() == QgsProviderRegistry::WidgetMode::None )
228   {
229     accept();
230   }
231 }
232 
233 
selectedChars()234 QString QgsDelimitedTextSourceSelect::selectedChars()
235 {
236   QString chars;
237   if ( cbxDelimComma->isChecked() )
238     chars.append( ',' );
239   if ( cbxDelimSpace->isChecked() )
240     chars.append( ' ' );
241   if ( cbxDelimTab->isChecked() )
242     chars.append( '\t' );
243   if ( cbxDelimSemicolon->isChecked() )
244     chars.append( ';' );
245   if ( cbxDelimColon->isChecked() )
246     chars.append( ':' );
247   chars = QgsDelimitedTextFile::encodeChars( chars );
248   chars.append( txtDelimiterOther->text() );
249   return chars;
250 }
setSelectedChars(const QString & delimiters)251 void QgsDelimitedTextSourceSelect::setSelectedChars( const QString &delimiters )
252 {
253   QString chars = QgsDelimitedTextFile::decodeChars( delimiters );
254   cbxDelimComma->setChecked( chars.contains( ',' ) );
255   cbxDelimSpace->setChecked( chars.contains( ' ' ) );
256   cbxDelimTab->setChecked( chars.contains( '\t' ) );
257   cbxDelimColon->setChecked( chars.contains( ':' ) );
258   cbxDelimSemicolon->setChecked( chars.contains( ';' ) );
259   chars = chars.remove( QRegularExpression( QStringLiteral( "[ ,:;\t]" ) ) );
260   chars = QgsDelimitedTextFile::encodeChars( chars );
261   txtDelimiterOther->setText( chars );
262 }
263 
loadSettings(const QString & subkey,bool loadGeomSettings)264 void QgsDelimitedTextSourceSelect::loadSettings( const QString &subkey, bool loadGeomSettings )
265 {
266   const QgsSettings settings;
267 
268   // at startup, fetch the last used delimiter and directory from
269   // settings
270   QString key = mSettingsKey;
271   if ( ! subkey.isEmpty() ) key.append( '/' ).append( subkey );
272 
273   // and how to use the delimiter
274   const QString delimiterType = settings.value( key + "/delimiterType", "" ).toString();
275   if ( delimiterType == QLatin1String( "chars" ) )
276   {
277     delimiterChars->setChecked( true );
278   }
279   else if ( delimiterType == QLatin1String( "regexp" ) )
280   {
281     delimiterRegexp->setChecked( true );
282   }
283   else if ( delimiterType == QLatin1String( "csv" ) )
284   {
285     delimiterCSV->setChecked( true );
286   }
287   swFileFormat->setCurrentIndex( bgFileFormat->checkedId() );
288 
289   const QString encoding = settings.value( key + "/encoding", "" ).toString();
290   if ( ! encoding.isEmpty() ) cmbEncoding->setCurrentIndex( cmbEncoding->findText( encoding ) );
291   const QString delimiters = settings.value( key + "/delimiters", "" ).toString();
292   if ( ! delimiters.isEmpty() ) setSelectedChars( delimiters );
293 
294   txtQuoteChars->setText( settings.value( key + "/quoteChars", "\"" ).toString() );
295   txtEscapeChars->setText( settings.value( key + "/escapeChars", "\"" ).toString() );
296 
297   const QString regexp = settings.value( key + "/delimiterRegexp", "" ).toString();
298   if ( ! regexp.isEmpty() ) txtDelimiterRegexp->setText( regexp );
299 
300   rowCounter->setValue( settings.value( key + "/startFrom", 0 ).toInt() );
301   cbxUseHeader->setChecked( settings.value( key + "/useHeader", "true" ) != "false" );
302   cbxDetectTypes->setChecked( settings.value( key + "/detectTypes", "true" ) != "false" );
303   cbxTrimFields->setChecked( settings.value( key + "/trimFields", "false" ) == "true" );
304   cbxSkipEmptyFields->setChecked( settings.value( key + "/skipEmptyFields", "false" ) == "true" );
305   cbxPointIsComma->setChecked( settings.value( key + "/decimalPoint", "." ).toString().contains( ',' ) );
306   cbxSubsetIndex->setChecked( settings.value( key + "/subsetIndex", "false" ) == "true" );
307   cbxSpatialIndex->setChecked( settings.value( key + "/spatialIndex", "false" ) == "true" );
308   cbxWatchFile->setChecked( settings.value( key + "/watchFile", "false" ) == "true" );
309 
310   if ( loadGeomSettings )
311   {
312     const QString geomColumnType = settings.value( key + "/geomColumnType", "xy" ).toString();
313     if ( geomColumnType == QLatin1String( "xy" ) ) geomTypeXY->setChecked( true );
314     else if ( geomColumnType == QLatin1String( "wkt" ) ) geomTypeWKT->setChecked( true );
315     else geomTypeNone->setChecked( true );
316     cbxXyDms->setChecked( settings.value( key + "/xyDms", "false" ) == "true" );
317     swGeomType->setCurrentIndex( bgGeomType->checkedId() );
318     const QString authid = settings.value( key + "/crs", "" ).toString();
319     const QgsCoordinateReferenceSystem crs = QgsCoordinateReferenceSystem::fromOgcWmsCrs( authid );
320     if ( crs.isValid() )
321     {
322       crsGeometry->setCrs( crs );
323     }
324   }
325 
326 }
327 
saveSettings(const QString & subkey,bool saveGeomSettings)328 void QgsDelimitedTextSourceSelect::saveSettings( const QString &subkey, bool saveGeomSettings )
329 {
330   QgsSettings settings;
331   QString key = mSettingsKey;
332   if ( ! subkey.isEmpty() ) key.append( '/' ).append( subkey );
333   settings.setValue( key + "/encoding", cmbEncoding->currentText() );
334   settings.setValue( key + "/geometry", saveGeometry() );
335 
336   if ( delimiterCSV->isChecked() )
337     settings.setValue( key + "/delimiterType", "csv" );
338   else if ( delimiterChars->isChecked() )
339     settings.setValue( key + "/delimiterType", "chars" );
340   else
341     settings.setValue( key + "/delimiterType", "regexp" );
342   settings.setValue( key + "/delimiters", selectedChars() );
343   settings.setValue( key + "/quoteChars", txtQuoteChars->text() );
344   settings.setValue( key + "/escapeChars", txtEscapeChars->text() );
345   settings.setValue( key + "/delimiterRegexp", txtDelimiterRegexp->text() );
346   settings.setValue( key + "/startFrom", rowCounter->value() );
347   settings.setValue( key + "/useHeader", cbxUseHeader->isChecked() ? "true" : "false" );
348   settings.setValue( key + "/detectTypes", cbxDetectTypes->isChecked() ? "true" : "false" );
349   settings.setValue( key + "/trimFields", cbxTrimFields->isChecked() ? "true" : "false" );
350   settings.setValue( key + "/skipEmptyFields", cbxSkipEmptyFields->isChecked() ? "true" : "false" );
351   settings.setValue( key + "/decimalPoint", cbxPointIsComma->isChecked() ? "," : "." );
352   settings.setValue( key + "/subsetIndex", cbxSubsetIndex->isChecked() ? "true" : "false" );
353   settings.setValue( key + "/spatialIndex", cbxSpatialIndex->isChecked() ? "true" : "false" );
354   settings.setValue( key + "/watchFile", cbxWatchFile->isChecked() ? "true" : "false" );
355   if ( saveGeomSettings )
356   {
357     QString geomColumnType = QStringLiteral( "none" );
358     if ( geomTypeXY->isChecked() ) geomColumnType = QStringLiteral( "xy" );
359     if ( geomTypeWKT->isChecked() ) geomColumnType = QStringLiteral( "wkt" );
360     settings.setValue( key + "/geomColumnType", geomColumnType );
361     settings.setValue( key + "/xyDms", cbxXyDms->isChecked() ? "true" : "false" );
362     if ( crsGeometry->crs().isValid() )
363     {
364       settings.setValue( key + "/crs", crsGeometry->crs().authid() );
365     }
366   }
367 
368 }
369 
loadSettingsForFile(const QString & filename)370 void QgsDelimitedTextSourceSelect::loadSettingsForFile( const QString &filename )
371 {
372   if ( filename.isEmpty() ) return;
373   const QFileInfo fi( filename );
374   const QString filetype = fi.suffix();
375   // Don't expect to change settings if not changing file type
376   if ( filetype != mLastFileType ) loadSettings( fi.suffix(), true );
377   mLastFileType = filetype;
378 }
379 
saveSettingsForFile(const QString & filename)380 void QgsDelimitedTextSourceSelect::saveSettingsForFile( const QString &filename )
381 {
382   if ( filename.isEmpty() ) return;
383   const QFileInfo fi( filename );
384   saveSettings( fi.suffix(), true );
385 }
386 
387 
loadDelimitedFileDefinition()388 bool QgsDelimitedTextSourceSelect::loadDelimitedFileDefinition()
389 {
390   mFile->setFileName( mFileWidget->filePath() );
391   mFile->setEncoding( cmbEncoding->currentText() );
392   if ( delimiterChars->isChecked() )
393   {
394     mFile->setTypeCSV( selectedChars(), txtQuoteChars->text(), txtEscapeChars->text() );
395   }
396   else if ( delimiterRegexp->isChecked() )
397   {
398     mFile->setTypeRegexp( txtDelimiterRegexp->text() );
399   }
400   else
401   {
402     mFile->setTypeCSV();
403   }
404   mFile->setSkipLines( rowCounter->value() );
405   mFile->setUseHeader( cbxUseHeader->isChecked() );
406   mFile->setDiscardEmptyFields( cbxSkipEmptyFields->isChecked() );
407   mFile->setTrimFields( cbxTrimFields->isChecked() );
408   mFile->setMaxFields( mMaxFields );
409   return mFile->isValid();
410 }
411 
412 
updateFieldLists()413 void QgsDelimitedTextSourceSelect::updateFieldLists()
414 {
415   // Update the x and y field drop-down boxes
416   QgsDebugMsgLevel( QStringLiteral( "Updating field lists" ), 3 );
417 
418   disconnect( cmbXField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
419   disconnect( cmbYField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
420   disconnect( cmbWktField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
421   disconnect( geomTypeXY, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
422   disconnect( geomTypeWKT, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
423   disconnect( geomTypeNone, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
424 
425   const QString columnX = cmbXField->currentText();
426   const QString columnY = cmbYField->currentText();
427   const QString columnZ = cmbZField->currentText();
428   const QString columnM = cmbMField->currentText();
429   const QString columnWkt = cmbWktField->currentText();
430 
431   // clear the field lists
432   cmbXField->clear();
433   cmbYField->clear();
434   cmbZField->clear();
435   cmbMField->clear();
436   cmbWktField->clear();
437 
438   // clear the sample text box
439   tblSample->clear();
440   tblSample->setColumnCount( 0 );
441   tblSample->setRowCount( 0 );
442 
443   if ( ! loadDelimitedFileDefinition() )
444     return;
445 
446   // Put a sample set of records into the sample box.  Also while scanning assess suitability of
447   // fields for use as coordinate and WKT fields
448 
449 
450   QList<bool> isValidCoordinate;
451   QList<bool> isValidWkt;
452   QList<bool> isEmpty;
453   int counter = 0;
454   mBadRowCount = 0;
455   QStringList values;
456   const QRegularExpression wktre( "^\\s*(?:MULTI)?(?:POINT|LINESTRING|POLYGON)\\s*Z?\\s*M?\\(", QRegularExpression::CaseInsensitiveOption );
457 
458   while ( counter < mExampleRowCount )
459   {
460     const QgsDelimitedTextFile::Status status = mFile->nextRecord( values );
461     if ( status == QgsDelimitedTextFile::RecordEOF ) break;
462     if ( status != QgsDelimitedTextFile::RecordOk ) { mBadRowCount++; continue; }
463     counter++;
464 
465     // Look at count of non-blank fields
466 
467     int nv = values.size();
468     while ( nv > 0 && values[nv - 1].isEmpty() ) nv--;
469 
470     if ( isEmpty.size() < nv )
471     {
472       while ( isEmpty.size() < nv )
473       {
474         isEmpty.append( true );
475         isValidCoordinate.append( false );
476         isValidWkt.append( false );
477       }
478       tblSample->setColumnCount( nv );
479     }
480 
481     tblSample->setRowCount( counter );
482 
483     const bool xyDms = cbxXyDms->isChecked();
484 
485     for ( int i = 0; i < tblSample->columnCount(); i++ )
486     {
487       QString value = i < nv ? values[i] : QString();
488       if ( value.length() > MAX_SAMPLE_LENGTH )
489         value = value.mid( 0, MAX_SAMPLE_LENGTH ) + QChar( 0x2026 );
490       QTableWidgetItem *item = new QTableWidgetItem( value );
491       tblSample->setItem( counter - 1, i, item );
492       if ( ! value.isEmpty() )
493       {
494         if ( isEmpty[i] )
495         {
496           isEmpty[i] = false;
497           isValidCoordinate[i] = true;
498           isValidWkt[i] = true;
499         }
500         if ( isValidCoordinate[i] )
501         {
502           bool ok = true;
503           if ( cbxPointIsComma->isChecked() )
504           {
505             value.replace( ',', '.' );
506           }
507           if ( xyDms )
508           {
509             const QRegularExpressionMatch match = QgsDelimitedTextProvider::sCrdDmsRegexp.match( value );
510             ok = match.capturedStart() == 0;
511           }
512           else
513           {
514             ( void )value.toDouble( &ok );
515           }
516           isValidCoordinate[i] = ok;
517         }
518         if ( isValidWkt[i] )
519         {
520           value.remove( QgsDelimitedTextProvider::sWktPrefixRegexp );
521           isValidWkt[i] = value.contains( wktre );
522         }
523       }
524     }
525   }
526 
527   QStringList fieldList = mFile->fieldNames();
528 
529   if ( isEmpty.size() < fieldList.size() )
530   {
531     while ( isEmpty.size() < fieldList.size() )
532     {
533       isEmpty.append( true );
534       isValidCoordinate.append( false );
535       isValidWkt.append( false );
536     }
537     tblSample->setColumnCount( fieldList.size() );
538   }
539 
540   tblSample->setHorizontalHeaderLabels( fieldList );
541   tblSample->resizeColumnsToContents();
542   tblSample->resizeRowsToContents();
543 
544   // We don't know anything about a text based field other
545   // than its name. All fields are assumed to be text
546   // As we ignore blank fields we need to map original index
547   // of selected fields to index in combo box.
548 
549   int fieldNo = 0;
550   for ( int i = 0; i < fieldList.size(); i++ )
551   {
552     const QString field = fieldList[i];
553     // skip empty field names
554     if ( field.isEmpty() ) continue;
555     cmbXField->addItem( field );
556     cmbYField->addItem( field );
557     cmbZField->addItem( field );
558     cmbMField->addItem( field );
559     cmbWktField->addItem( field );
560     fieldNo++;
561   }
562 
563   // Try resetting current values for column names
564 
565   cmbWktField->setCurrentIndex( cmbWktField->findText( columnWkt ) );
566   cmbXField->setCurrentIndex( cmbXField->findText( columnX ) );
567   cmbYField->setCurrentIndex( cmbYField->findText( columnY ) );
568   cmbZField->setCurrentIndex( cmbYField->findText( columnZ ) );
569   cmbMField->setCurrentIndex( cmbYField->findText( columnM ) );
570 
571   // Now try setting optional X,Y fields - will only reset the fields if
572   // not already set.
573 
574   trySetXYField( fieldList, isValidCoordinate, QStringLiteral( "longitude" ), QStringLiteral( "latitude" ) );
575   trySetXYField( fieldList, isValidCoordinate, QStringLiteral( "lon" ), QStringLiteral( "lat" ) );
576   trySetXYField( fieldList, isValidCoordinate, QStringLiteral( "east" ), QStringLiteral( "north" ) );
577   trySetXYField( fieldList, isValidCoordinate, QStringLiteral( "x" ), QStringLiteral( "y" ) );
578   trySetXYField( fieldList, isValidCoordinate, QStringLiteral( "e" ), QStringLiteral( "n" ) );
579 
580   // And also a WKT field if there is one
581 
582   if ( cmbWktField->currentIndex() < 0 )
583   {
584     for ( int i = 0; i < fieldList.size(); i++ )
585     {
586       if ( ! isValidWkt[i] ) continue;
587       const int index = cmbWktField->findText( fieldList[i] );
588       if ( index >= 0 )
589       {
590         cmbWktField->setCurrentIndex( index );
591         break;
592       }
593     }
594   }
595 
596   const bool haveFields = fieldNo > 0;
597 
598   if ( !geomTypeNone->isChecked() )
599   {
600     const bool isXY = cmbWktField->currentIndex() < 0 ||
601                       ( geomTypeXY->isChecked() &&
602                         ( cmbXField->currentIndex() >= 0 && cmbYField->currentIndex() >= 0 ) );
603     geomTypeXY->setChecked( isXY );
604     geomTypeWKT->setChecked( ! isXY );
605   }
606   swGeomType->setCurrentIndex( bgGeomType->checkedId() );
607 
608   if ( haveFields )
609   {
610     connect( cmbXField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
611     connect( cmbYField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
612     connect( cmbWktField, static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ), this, &QgsDelimitedTextSourceSelect::enableAccept );
613     connect( geomTypeXY, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
614     connect( geomTypeWKT, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
615     connect( geomTypeNone, &QAbstractButton::toggled, this, &QgsDelimitedTextSourceSelect::enableAccept );
616   }
617 
618 }
619 
trySetXYField(QStringList & fields,QList<bool> & isValidNumber,const QString & xname,const QString & yname)620 bool QgsDelimitedTextSourceSelect::trySetXYField( QStringList &fields, QList<bool> &isValidNumber, const QString &xname, const QString &yname )
621 {
622   // If fields already set, then nothing to do
623   if ( cmbXField->currentIndex() >= 0 && cmbYField->currentIndex() >= 0 ) return true;
624 
625   // Try and find a valid field name matching the x field
626   int indexX = -1;
627   int indexY = -1;
628 
629   for ( int i = 0; i < fields.size(); i++ )
630   {
631     // Only interested in number fields containing the xname string
632     // that are in the X combo box
633     if ( ! isValidNumber[i] ) continue;
634     if ( ! fields[i].contains( xname, Qt::CaseInsensitive ) ) continue;
635     indexX = cmbXField->findText( fields[i] );
636     if ( indexX < 0 ) continue;
637 
638     // Now look for potential y fields, like xname with x replaced with y
639     const QString xfield( fields[i] );
640     int from = 0;
641     while ( true )
642     {
643       const int pos = xfield.indexOf( xname, from, Qt::CaseInsensitive );
644       if ( pos < 0 ) break;
645       from = pos + 1;
646       const QString yfield = xfield.mid( 0, pos ) + yname + xfield.mid( pos + xname.size() );
647       if ( ! fields.contains( yfield, Qt::CaseInsensitive ) ) continue;
648       for ( int iy = 0; iy < fields.size(); iy++ )
649       {
650         if ( ! isValidNumber[iy] ) continue;
651         if ( iy == i ) continue;
652         if ( fields[iy].compare( yfield, Qt::CaseInsensitive ) == 0 )
653         {
654           indexY = cmbYField->findText( fields[iy] );
655           break;
656         }
657       }
658       if ( indexY >= 0 ) break;
659     }
660     if ( indexY >= 0 ) break;
661   }
662   if ( indexY >= 0 )
663   {
664     cmbXField->setCurrentIndex( indexX );
665     cmbYField->setCurrentIndex( indexY );
666   }
667   return indexY >= 0;
668 }
669 
updateFileName()670 void QgsDelimitedTextSourceSelect::updateFileName()
671 {
672   QgsSettings settings;
673   settings.setValue( mSettingsKey + "/file_filter", mFileWidget->selectedFilter() );
674 
675   // put a default layer name in the text entry
676   const QString filename = mFileWidget->filePath();
677   const QFileInfo finfo( filename );
678   if ( finfo.exists() )
679   {
680     QgsSettings settings;
681     settings.setValue( mSettingsKey + "/text_path", finfo.path() );
682   }
683 
684   txtLayerName->setText( finfo.completeBaseName() );
685   loadSettingsForFile( filename );
686   updateFieldsAndEnable();
687 }
688 
updateFieldsAndEnable()689 void QgsDelimitedTextSourceSelect::updateFieldsAndEnable()
690 {
691   updateFieldLists();
692   enableAccept();
693 }
694 
validate()695 bool QgsDelimitedTextSourceSelect::validate()
696 {
697   // Check that input data is valid - provide a status message if not..
698 
699   QString message;
700   bool enabled = false;
701 
702   if ( mFileWidget->filePath().trimmed().isEmpty() )
703   {
704     message = tr( "Please select an input file" );
705   }
706   else if ( ! QFileInfo::exists( mFileWidget->filePath() ) )
707   {
708     message = tr( "File %1 does not exist" ).arg( mFileWidget->filePath() );
709   }
710   else if ( txtLayerName->text().isEmpty() )
711   {
712     message = tr( "Please enter a layer name" );
713   }
714   else if ( delimiterChars->isChecked() && selectedChars().isEmpty() )
715   {
716     message = tr( "At least one delimiter character must be specified" );
717   }
718 
719   if ( message.isEmpty() && delimiterRegexp->isChecked() )
720   {
721     const QRegularExpression re( txtDelimiterRegexp->text() );
722     if ( ! re.isValid() )
723     {
724       message = tr( "Regular expression is not valid" );
725     }
726     else if ( re.pattern().startsWith( '^' ) && re.captureCount() == 0 )
727     {
728       message = tr( "^.. expression needs capture groups" );
729     }
730     lblRegexpError->setText( message );
731   }
732   if ( ! message.isEmpty() )
733   {
734     // continue...
735   }
736   // Hopefully won't hit this none-specific message, but just in case ...
737   else if ( ! mFile->isValid() )
738   {
739     message = tr( "Definition of filename and delimiters is not valid" );
740   }
741   // Assume that the sample table will have been populated if data was found
742   else if ( tblSample->rowCount() == 0 )
743   {
744     message = tr( "No data found in file" );
745     if ( mBadRowCount > 0 )
746     {
747       message = message + " (" + tr( "%1 badly formatted records discarded" ).arg( mBadRowCount ) + ')';
748     }
749   }
750   else if ( geomTypeXY->isChecked() && ( cmbXField->currentText().isEmpty()  || cmbYField->currentText().isEmpty() ) )
751   {
752     message = tr( "X and Y field names must be selected" );
753   }
754   else if ( geomTypeXY->isChecked() && ( cmbXField->currentText() == cmbYField->currentText() ) )
755   {
756     message = tr( "X and Y field names cannot be the same" );
757   }
758   else if ( geomTypeWKT->isChecked() && cmbWktField->currentText().isEmpty() )
759   {
760     message = tr( "The WKT field name must be selected" );
761   }
762   else if ( ! geomTypeNone->isChecked() && ! crsGeometry->crs().isValid() )
763   {
764     message = tr( "The CRS must be selected" );
765   }
766   else
767   {
768     enabled = true;
769     if ( mBadRowCount > 0 )
770     {
771       message = tr( "%1 badly formatted records discarded from sample data" ).arg( mBadRowCount );
772     }
773 
774   }
775   lblStatus->setText( message );
776   return enabled;
777 }
778 
779 
enableAccept()780 void QgsDelimitedTextSourceSelect::enableAccept()
781 {
782   emit enableButtons( validate() );
783 }
784 
showHelp()785 void QgsDelimitedTextSourceSelect::showHelp()
786 {
787   QgsHelp::openHelp( QStringLiteral( "managing_data_source/opening_data.html#importing-a-delimited-text-file" ) );
788 }
789 
showCrsWidget()790 void QgsDelimitedTextSourceSelect::showCrsWidget()
791 {
792   crsGeometry->setVisible( !geomTypeNone->isChecked() );
793   textLabelCrs->setVisible( !geomTypeNone->isChecked() );
794 }
795