1 /******************************************************************************
2 *
3 * Project: Elasticsearch Translator
4 * Purpose:
5 * Author:
6 *
7 ******************************************************************************
8 * Copyright (c) 2011, Adam Estrada
9 * Copyright (c) 2012-2016, Even Rouault <even dot rouault at spatialys.com>
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a
12 * copy of this software and associated documentation files (the "Software"),
13 * to deal in the Software without restriction, including without limitation
14 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 * and/or sell copies of the Software, and to permit persons to whom the
16 * Software is furnished to do so, subject to the following conditions:
17 *
18 * The above copyright notice and this permission notice shall be included
19 * in all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 * DEALINGS IN THE SOFTWARE.
28 ****************************************************************************/
29
30 #include "ogr_elastic.h"
31 #include "cpl_conv.h"
32 #include "cpl_minixml.h"
33 #include "cpl_http.h"
34 #include "ogr_api.h"
35 #include "ogr_p.h"
36 #include "ogr_swq.h"
37 #include "../geojson/ogrgeojsonwriter.h"
OGRElasticsearchDriverIdentify(GDALOpenInfo * poOpenInfo)38 #include "../geojson/ogrgeojsonreader.h"
39 #include "../geojson/ogrgeojsonutils.h"
40 #include "ogr_geo_utils.h"
41
42 #include <cstdlib>
43 #include <set>
44
45
46 /************************************************************************/
47 /* CPLGettimeofday() */
48 /************************************************************************/
49
50 #if defined(_WIN32) && !defined(__CYGWIN__)
51 # include <sys/timeb.h>
52
53 struct CPLTimeVal
54 {
55 time_t tv_sec; /* seconds */
56 long tv_usec; /* and microseconds */
57 };
58
59 static void CPLGettimeofday(struct CPLTimeVal* tp, void* /* timezonep*/ )
60 {
61 struct _timeb theTime;
62
63 _ftime(&theTime);
64 tp->tv_sec = static_cast<time_t>(theTime.time);
65 tp->tv_usec = theTime.millitm * 1000;
OGRElasticsearchDriverCreate(const char * pszName,CPL_UNUSED int nXSize,CPL_UNUSED int nYSize,CPL_UNUSED int nBands,CPL_UNUSED GDALDataType eDT,char ** papszOptions)66 }
67 #else
68 # include <sys/time.h> /* for gettimeofday() */
69 # define CPLTimeVal timeval
70 # define CPLGettimeofday(t,u) gettimeofday(t,u)
71 #endif
72
73 static double GetTimestamp()
74 {
75 struct CPLTimeVal tv;
76 CPLGettimeofday(&tv, nullptr);
77 return tv.tv_sec + tv.tv_usec * 1e-6;
78 }
79
80 CPL_CVSID("$Id: ogrelasticlayer.cpp e243cb96b4330dcbba3f089219968107aa9b1ef6 2021-02-04 17:23:54 +0100 Even Rouault $")
81
82 /************************************************************************/
83 /* OGRElasticLayer() */
84 /************************************************************************/
85
86 OGRElasticLayer::OGRElasticLayer( const char* pszLayerName,
RegisterOGRElastic()87 const char* pszIndexName,
88 const char* pszMappingName,
89 OGRElasticDataSource* poDS,
90 char** papszOptions,
91 const char* pszESSearch ) :
92
93 m_poDS(poDS),
94 m_osIndexName(pszIndexName ? pszIndexName : ""),
95 // Types are no longer supported in Elasticsearch 7+.
96 m_osMappingName(poDS->m_nMajorVersion < 7
97 ? pszMappingName ? pszMappingName : ""
98 : ""),
99 m_poFeatureDefn(new OGRFeatureDefn(pszLayerName)),
100 m_bFeatureDefnFinalized(false),
101 m_bManualMapping(false),
102 m_bSerializeMapping(false),
103 m_osWriteMapFilename(
104 CSLFetchNameValueDef(papszOptions, "WRITE_MAPPING",
105 poDS->m_pszWriteMap ? poDS->m_pszWriteMap : "")),
106 m_bStoreFields(CPLFetchBool(papszOptions, "STORE_FIELDS", false)),
107 m_papszStoredFields(nullptr),
108 m_papszNotAnalyzedFields(nullptr),
109 m_papszNotIndexedFields(nullptr),
110 m_papszFieldsWithRawValue(nullptr),
111 m_osESSearch(pszESSearch ? pszESSearch : ""),
112 m_nBulkUpload(poDS->m_nBulkUpload),
113 m_eGeomTypeMapping(ES_GEOMTYPE_AUTO),
114 m_osPrecision(CSLFetchNameValueDef(papszOptions, "GEOM_PRECISION", "")),
115 m_iCurID(0),
116 m_nNextFID(-1),
117 m_iCurFeatureInPage(0),
118 m_bEOF(false),
119 m_poSpatialFilter(nullptr),
120 m_bFilterMustBeClientSideEvaluated(false),
121 m_poJSONFilter(nullptr),
122 m_bIgnoreSourceID(false),
123 m_bDotAsNestedField(true),
124 // Undocumented. Only useful for developers.
125 m_bAddPretty(CPLTestBool(CPLGetConfigOption("ES_ADD_PRETTY", "FALSE"))),
126 m_bGeoShapeAsGeoJSON(EQUAL(CSLFetchNameValueDef(papszOptions, "GEO_SHAPE_ENCODING", "GeoJSON"), "GeoJSON"))
127 {
128 const char* pszESGeomType = CSLFetchNameValue(papszOptions, "GEOM_MAPPING_TYPE");
129 if( pszESGeomType != nullptr )
130 {
131 if( EQUAL(pszESGeomType, "GEO_POINT") )
132 m_eGeomTypeMapping = ES_GEOMTYPE_GEO_POINT;
133 else if( EQUAL(pszESGeomType, "GEO_SHAPE") )
134 m_eGeomTypeMapping = ES_GEOMTYPE_GEO_SHAPE;
135 }
136
137 if( CPLFetchBool(papszOptions, "BULK_INSERT", true) )
138 {
139 m_nBulkUpload = atoi(CSLFetchNameValueDef(papszOptions, "BULK_SIZE", "1000000"));
140 }
141
142 const char* pszStoredFields = CSLFetchNameValue(papszOptions, "STORED_FIELDS");
143 if( pszStoredFields )
144 m_papszStoredFields = CSLTokenizeString2(pszStoredFields, ",", 0);
145
146 const char* pszNotAnalyzedFields = CSLFetchNameValue(papszOptions, "NOT_ANALYZED_FIELDS");
147 if( pszNotAnalyzedFields )
148 m_papszNotAnalyzedFields = CSLTokenizeString2(pszNotAnalyzedFields, ",", 0);
149
150 const char* pszNotIndexedFields = CSLFetchNameValue(papszOptions, "NOT_INDEXED_FIELDS");
151 if( pszNotIndexedFields )
152 m_papszNotIndexedFields = CSLTokenizeString2(pszNotIndexedFields, ",", 0);
153
154 const char* pszFieldsWithRawValue = CSLFetchNameValue(papszOptions,
155 "FIELDS_WITH_RAW_VALUE");
156 if( pszFieldsWithRawValue )
157 m_papszFieldsWithRawValue = CSLTokenizeString2(
158 pszFieldsWithRawValue, ",", 0);
159
160 const char* pszSingleQueryTimeout = CSLFetchNameValue(papszOptions, "SINGLE_QUERY_TIMEOUT");
161 if( pszSingleQueryTimeout )
162 {
163 m_dfSingleQueryTimeout = CPLAtof(pszSingleQueryTimeout);
164 if( m_dfSingleQueryTimeout < 1 && m_dfSingleQueryTimeout >= 1e-3 )
165 m_osSingleQueryTimeout = CPLSPrintf("%dms", static_cast<int>(m_dfSingleQueryTimeout * 1000));
166 else if( m_dfSingleQueryTimeout >= 1 )
167 m_osSingleQueryTimeout = CPLSPrintf("%ds", static_cast<int>(m_dfSingleQueryTimeout));
168 }
169
170 m_osSingleQueryTerminateAfter = CSLFetchNameValueDef(
171 papszOptions, "SINGLE_QUERY_TERMINATE_AFTER", "");
172 m_nSingleQueryTerminateAfter = CPLAtoGIntBig(m_osSingleQueryTerminateAfter);
173
174 const char* pszFeatureIterationTimeout = CSLFetchNameValue(papszOptions, "FEATURE_ITERATION_TIMEOUT");
175 if( pszFeatureIterationTimeout )
176 {
177 m_dfFeatureIterationTimeout = CPLAtof(pszFeatureIterationTimeout);
178 }
179 m_nFeatureIterationTerminateAfter = CPLAtoGIntBig(CSLFetchNameValueDef(
180 papszOptions, "FEATURE_ITERATION_TERMINATE_AFTER", ""));
181
182 SetDescription( m_poFeatureDefn->GetName() );
183 m_poFeatureDefn->Reference();
184 m_poFeatureDefn->SetGeomType(wkbNone);
185
186 AddFieldDefn("_id", OFTString, std::vector<CPLString>());
187
188 if( !m_osESSearch.empty() )
189 {
190 AddFieldDefn("_index", OFTString, std::vector<CPLString>());
191 AddFieldDefn("_type", OFTString, std::vector<CPLString>());
192 }
193
194 OGRElasticLayer::ResetReading();
195 }
196
197 /************************************************************************/
198 /* Clone() */
199 /************************************************************************/
200
201 OGRElasticLayer* OGRElasticLayer::Clone() const
202 {
203 OGRElasticLayer* poNew = new OGRElasticLayer(m_poFeatureDefn->GetName(),
204 m_osIndexName,
205 m_osMappingName,
206 m_poDS,
207 nullptr);
208 poNew->m_poFeatureDefn->Release();
209 poNew->m_poFeatureDefn =
210 const_cast<OGRElasticLayer*>(this)->GetLayerDefn()->Clone();
211 poNew->m_poFeatureDefn->Reference();
212 poNew->m_bFeatureDefnFinalized = true;
213 poNew->m_osBulkContent = m_osBulkContent;
214 poNew->m_nBulkUpload = m_nBulkUpload;
215 poNew->m_osFID = m_osFID;
216 poNew->m_aaosFieldPaths = m_aaosFieldPaths;
217 poNew->m_aosMapToFieldIndex = m_aosMapToFieldIndex;
218 poNew->m_aaosGeomFieldPaths = m_aaosGeomFieldPaths;
219 poNew->m_aosMapToGeomFieldIndex = m_aosMapToGeomFieldIndex;
220 poNew->m_abIsGeoPoint = m_abIsGeoPoint;
221 poNew->m_eGeomTypeMapping = m_eGeomTypeMapping;
222 poNew->m_osPrecision = m_osPrecision;
223 poNew->m_papszNotAnalyzedFields = CSLDuplicate(m_papszNotAnalyzedFields);
224 poNew->m_papszNotIndexedFields = CSLDuplicate(m_papszNotIndexedFields);
225 poNew->m_papszFieldsWithRawValue = CSLDuplicate(m_papszFieldsWithRawValue);
226 poNew->m_bGeoShapeAsGeoJSON = m_bGeoShapeAsGeoJSON;
227 poNew->m_osSingleQueryTimeout = m_osSingleQueryTimeout;
228 poNew->m_dfSingleQueryTimeout = m_dfSingleQueryTimeout;
229 poNew->m_dfFeatureIterationTimeout = m_dfFeatureIterationTimeout;
230 poNew->m_nSingleQueryTerminateAfter = m_nSingleQueryTerminateAfter;
231 poNew->m_nFeatureIterationTerminateAfter = m_nFeatureIterationTerminateAfter;
232 poNew->m_osSingleQueryTerminateAfter = m_osSingleQueryTerminateAfter;
233 return poNew;
234 }
235
236 /************************************************************************/
237 /* ~OGRElasticLayer() */
238 /************************************************************************/
239
240 OGRElasticLayer::~OGRElasticLayer() {
241 OGRElasticLayer::SyncToDisk();
242
243 OGRElasticLayer::ResetReading();
244
245 json_object_put(m_poSpatialFilter);
246 json_object_put(m_poJSONFilter);
247
248 for(int i=0;i<(int)m_apoCT.size();i++)
249 delete m_apoCT[i];
250
251 m_poFeatureDefn->Release();
252
253 CSLDestroy(m_papszStoredFields);
254 CSLDestroy(m_papszNotAnalyzedFields);
255 CSLDestroy(m_papszNotIndexedFields);
256 CSLDestroy(m_papszFieldsWithRawValue);
257 }
258
259 /************************************************************************/
260 /* AddFieldDefn() */
261 /************************************************************************/
262
263 void OGRElasticLayer::AddFieldDefn( const char* pszName,
264 OGRFieldType eType,
265 const std::vector<CPLString>& aosPath,
266 OGRFieldSubType eSubType )
267 {
268 OGRFieldDefn oFieldDefn(pszName, eType);
269 oFieldDefn.SetSubType(eSubType);
270 if( eSubType == OFSTBoolean )
271 oFieldDefn.SetWidth(1);
272 m_aaosFieldPaths.push_back(aosPath);
273 if( !aosPath.empty() )
274 m_aosMapToFieldIndex[ BuildPathFromArray(aosPath) ] = m_poFeatureDefn->GetFieldCount();
275 m_poFeatureDefn->AddFieldDefn(&oFieldDefn);
276 }
277
278 /************************************************************************/
279 /* AddGeomFieldDefn() */
280 /************************************************************************/
281
282 void OGRElasticLayer::AddGeomFieldDefn( const char* pszName,
283 OGRwkbGeometryType eType,
284 const std::vector<CPLString>& aosPath,
285 int bIsGeoPoint )
286 {
287 OGRGeomFieldDefn oFieldDefn(pszName, eType);
288 m_aaosGeomFieldPaths.push_back(aosPath);
289 m_aosMapToGeomFieldIndex[ BuildPathFromArray(aosPath) ] = m_poFeatureDefn->GetGeomFieldCount();
290 m_abIsGeoPoint.push_back(bIsGeoPoint);
291
292 OGRSpatialReference* poSRS_WGS84 = new OGRSpatialReference();
293 poSRS_WGS84->SetFromUserInput(SRS_WKT_WGS84_LAT_LONG);
294 poSRS_WGS84->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
295 oFieldDefn.SetSpatialRef(poSRS_WGS84);
296 poSRS_WGS84->Dereference();
297
298 m_poFeatureDefn->AddGeomFieldDefn(&oFieldDefn);
299
300 m_apoCT.push_back(nullptr);
301 }
302
303 /************************************************************************/
304 /* InitFeatureDefnFromMapping() */
305 /************************************************************************/
306
307 void OGRElasticLayer::InitFeatureDefnFromMapping(json_object* poSchema,
308 const char* pszPrefix,
309 const std::vector<CPLString>& aosPath)
310 {
311 json_object* poTopProperties = CPL_json_object_object_get(poSchema, "properties");
312 if( poTopProperties == nullptr || json_object_get_type(poTopProperties) != json_type_object )
313 return;
314 json_object_iter it;
315 it.key = nullptr;
316 it.val = nullptr;
317 it.entry = nullptr;
318 json_object_object_foreachC( poTopProperties, it )
319 {
320 json_object* poProperties = CPL_json_object_object_get(it.val, "properties");
321 if( poProperties && json_object_get_type(poProperties) == json_type_object )
322 {
323 json_object* poType = json_ex_get_object_by_path(poProperties, "coordinates.type");
324 if( poType && json_object_get_type(poType) == json_type_string &&
325 strcmp(json_object_get_string(poType), "geo_point") == 0 )
326 {
327 CPLString osFieldName;
328 if( pszPrefix[0] )
329 {
330 osFieldName = pszPrefix;
331 osFieldName += ".";
332 }
333 osFieldName += it.key;
334
335 if( m_poFeatureDefn->GetGeomFieldIndex(osFieldName) < 0 )
336 {
337 std::vector<CPLString> aosNewPaths = aosPath;
338 aosNewPaths.push_back(osFieldName);
339 aosNewPaths.push_back("coordinates");
340
341 AddGeomFieldDefn(osFieldName, wkbPoint, aosNewPaths, TRUE);
342 }
343
344 continue;
345 }
346
347 if( aosPath.empty() && m_osMappingName == "FeatureCollection" && strcmp(it.key, "properties") == 0 )
348 {
349 std::vector<CPLString> aosNewPaths = aosPath;
350 aosNewPaths.push_back(it.key);
351
352 InitFeatureDefnFromMapping(it.val, pszPrefix, aosNewPaths);
353
354 continue;
355 }
356 else if( m_poDS->m_bFlattenNestedAttributes )
357 {
358 std::vector<CPLString> aosNewPaths = aosPath;
359 aosNewPaths.push_back(it.key);
360
361 CPLString osPrefix;
362 if( pszPrefix[0] )
363 {
364 osPrefix = pszPrefix;
365 osPrefix += ".";
366 }
367 osPrefix += it.key;
368
369 InitFeatureDefnFromMapping(it.val, osPrefix, aosNewPaths);
370
371 continue;
372 }
373 }
374
375 if( aosPath.empty() && EQUAL(it.key, m_poDS->GetFID()) )
376 {
377 m_osFID = it.key;
378 }
379 else
380 {
381 CreateFieldFromSchema(it.key, pszPrefix, aosPath, it.val);
382 }
383 }
384
385 if( aosPath.empty() )
386 {
387 json_object* poMeta = CPL_json_object_object_get(poSchema, "_meta");
388 if( poMeta && json_object_get_type(poMeta) == json_type_object )
389 {
390 json_object* poFID = CPL_json_object_object_get(poMeta, "fid");
391 if( poFID && json_object_get_type(poFID) == json_type_string )
392 m_osFID = json_object_get_string(poFID);
393
394 json_object* poGeomFields = CPL_json_object_object_get(poMeta, "geomfields");
395 if( poGeomFields && json_object_get_type(poGeomFields) == json_type_object )
396 {
397 for( int i=0; i< m_poFeatureDefn->GetGeomFieldCount(); i++ )
398 {
399 json_object* poObj = CPL_json_object_object_get(poGeomFields,
400 m_poFeatureDefn->GetGeomFieldDefn(i)->GetNameRef());
401 if( poObj && json_object_get_type(poObj) == json_type_string )
402 {
403 OGRwkbGeometryType eType = OGRFromOGCGeomType(json_object_get_string(poObj));
404 if( eType != wkbUnknown )
405 m_poFeatureDefn->GetGeomFieldDefn(i)->SetType(eType);
406 }
407 }
408 }
409
410 json_object* poFields = CPL_json_object_object_get(poMeta, "fields");
411 if( poFields && json_object_get_type(poFields) == json_type_object )
412 {
413 for( int i=0; i< m_poFeatureDefn->GetFieldCount(); i++ )
414 {
415 json_object* poObj = CPL_json_object_object_get(poFields,
416 m_poFeatureDefn->GetFieldDefn(i)->GetNameRef());
417 if( poObj && json_object_get_type(poObj) == json_type_string )
418 {
419 for(int j=0; j<=OFTMaxType;j++)
420 {
421 if( EQUAL(OGR_GetFieldTypeName((OGRFieldType)j),
422 json_object_get_string(poObj)) )
423 {
424 m_poFeatureDefn->GetFieldDefn(i)->SetType((OGRFieldType)j);
425 break;
426 }
427 }
428 }
429 }
430 }
431 }
432 }
433 }
434
435 /************************************************************************/
436 /* CreateFieldFromSchema() */
437 /************************************************************************/
438
439 void OGRElasticLayer::CreateFieldFromSchema(const char* pszName,
440 const char* pszPrefix,
441 std::vector<CPLString> aosPath,
442 json_object* poObj)
443 {
444 const char* pszType = "";
445 json_object* poType = CPL_json_object_object_get(poObj, "type");
446 if( poType && json_object_get_type(poType) == json_type_string )
447 {
448 pszType = json_object_get_string(poType);
449 }
450
451 CPLString osFieldName;
452 if( pszPrefix[0] )
453 {
454 osFieldName = pszPrefix;
455 osFieldName += ".";
456 }
457 osFieldName += pszName;
458
459 if( EQUAL(pszType, "geo_point") || EQUAL(pszType, "geo_shape") )
460 {
461 if( m_poFeatureDefn->GetGeomFieldIndex(osFieldName) >= 0 )
462 return;
463
464 aosPath.push_back(pszName);
465 AddGeomFieldDefn(osFieldName,
466 EQUAL(pszType, "geo_point") ? wkbPoint : wkbUnknown,
467 aosPath, EQUAL(pszType, "geo_point"));
468 }
469 else if( !( aosPath.empty() && m_osMappingName == "FeatureCollection" ) )
470 {
471 if( m_poFeatureDefn->GetFieldIndex(osFieldName) >= 0 )
472 return;
473
474 OGRFieldType eType = OFTString;
475 OGRFieldSubType eSubType = OFSTNone;
476 if( EQUAL(pszType, "integer") )
477 eType = OFTInteger;
478 else if( EQUAL(pszType, "boolean") )
479 {
480 eType = OFTInteger;
481 eSubType = OFSTBoolean;
482 }
483 else if( EQUAL(pszType, "long") )
484 eType = OFTInteger64;
485 else if( EQUAL(pszType, "float") )
486 eType = OFTReal;
487 else if( EQUAL(pszType, "double") )
488 eType = OFTReal;
489 else if( EQUAL(pszType, "date") )
490 {
491 eType = OFTDateTime;
492 json_object* poFormat = CPL_json_object_object_get(poObj, "format");
493 if( poFormat && json_object_get_type(poFormat) == json_type_string )
494 {
495 const char* pszFormat = json_object_get_string(poFormat);
496 if( EQUAL(pszFormat, "HH:mm:ss.SSS") || EQUAL(pszFormat, "time") )
497 eType = OFTTime;
498 else if( EQUAL(pszFormat, "yyyy/MM/dd") || EQUAL(pszFormat, "date") )
499 eType = OFTDate;
500 }
501 }
502 else if( EQUAL(pszType, "binary") )
503 eType = OFTBinary;
504 else if( EQUAL(pszType, "string") ) // ES < 5.0
505 {
506 json_object* poIndex = CPL_json_object_object_get(poObj, "index");
507 if( poIndex && json_object_get_type(poIndex) == json_type_string )
508 {
509 const char* pszIndex = json_object_get_string(poIndex);
510 if( EQUAL(pszIndex, "not_analyzed") )
511 {
512 m_papszNotAnalyzedFields = CSLAddString(
513 m_papszNotAnalyzedFields, osFieldName);
514 }
515 }
516 }
517 else if( EQUAL(pszType, "keyword") ) // ES >= 5.0
518 {
519 m_papszNotAnalyzedFields = CSLAddString(
520 m_papszNotAnalyzedFields, osFieldName);
521 }
522
523 aosPath.push_back( pszName );
524 AddFieldDefn(osFieldName, eType, aosPath, eSubType);
525
526 json_object* poFields = CPL_json_object_object_get(poObj, "fields");
527 if( poFields && json_object_get_type(poFields) == json_type_object )
528 {
529 json_object* poRaw = CPL_json_object_object_get(poFields, "raw");
530 if( poRaw && json_object_get_type(poRaw) == json_type_object )
531 {
532 json_object* poRawType = CPL_json_object_object_get(poRaw,
533 "type");
534 if( poRawType && json_object_get_type(poRawType) ==
535 json_type_string )
536 {
537 const char* pszRawType = json_object_get_string(poRawType);
538 if( EQUAL(pszRawType, "keyword") ) // ES >= 5.0
539 {
540 m_papszFieldsWithRawValue = CSLAddString(
541 m_papszFieldsWithRawValue, osFieldName);
542 }
543 else if( EQUAL(pszRawType, "string") ) // ES < 5.0
544 {
545 json_object* poRawIndex = CPL_json_object_object_get(
546 poRaw, "index");
547 if( poRawIndex && json_object_get_type(poRawIndex) ==
548 json_type_string )
549 {
550 const char* pszRawIndex =
551 json_object_get_string(poRawIndex);
552 if( EQUAL(pszRawIndex, "not_analyzed") )
553 {
554 m_papszFieldsWithRawValue = CSLAddString(
555 m_papszFieldsWithRawValue, osFieldName);
556 }
557 }
558 }
559 }
560 }
561 }
562 }
563 }
564
565 /************************************************************************/
566 /* FinalizeFeatureDefn() */
567 /************************************************************************/
568
569 void OGRElasticLayer::FinalizeFeatureDefn(bool bReadFeatures)
570 {
571 if( m_bFeatureDefnFinalized )
572 return;
573
574 m_bFeatureDefnFinalized = true;
575
576 int nFeatureCountToEstablishFeatureDefn = m_poDS->m_nFeatureCountToEstablishFeatureDefn;
577 if( !m_osESSearch.empty() && nFeatureCountToEstablishFeatureDefn <= 0 )
578 nFeatureCountToEstablishFeatureDefn = 1;
579 std::set< std::pair<CPLString, CPLString> > oVisited;
580
581 if( bReadFeatures && nFeatureCountToEstablishFeatureDefn != 0 )
582 {
583 //CPLDebug("ES", "Try to get %d features to establish feature definition",
584 // FeatureCountToEstablishFeatureDefn);
585 bool bFirst = true;
586 int nAlreadyQueried = 0;
587 while( true )
588 {
589 CPLString osRequest;
590 CPLString osPostData;
591 if( bFirst )
592 {
593 bFirst = false;
594 if( !m_osESSearch.empty() )
595 {
596 osRequest = CPLSPrintf("%s/_search?scroll=1m&size=%d",
597 m_poDS->GetURL(), m_poDS->m_nBatchSize);
598 osPostData = m_osESSearch;
599 }
600 else
601 {
602 osRequest = BuildMappingURL(false);
603 osRequest += CPLSPrintf("/_search?scroll=1m&size=%d", m_poDS->m_nBatchSize);
604 }
605 }
606 else
607 {
608 if( m_osScrollID.empty() )
609 break;
610 osRequest = CPLSPrintf("%s/_search/scroll?scroll=1m&scroll_id=%s",
611 m_poDS->GetURL(), m_osScrollID.c_str());
612 }
613
614 if( m_bAddPretty )
615 osRequest += "&pretty";
616 json_object* poResponse = m_poDS->RunRequest(osRequest, osPostData);
617 if( poResponse == nullptr )
618 {
619 break;
620 }
621 json_object* poScrollID = CPL_json_object_object_get(poResponse, "_scroll_id");
622 if( poScrollID )
623 {
624 const char* pszScrollID = json_object_get_string(poScrollID);
625 if( pszScrollID )
626 m_osScrollID = pszScrollID;
627 }
628
629 json_object* poHits = json_ex_get_object_by_path(poResponse, "hits.hits");
630 if( poHits == nullptr || json_object_get_type(poHits) != json_type_array )
631 {
632 json_object_put(poResponse);
633 break;
634 }
635 const auto nHits = json_object_array_length(poHits);
636 if( nHits == 0 )
637 {
638 m_osScrollID = "";
639 json_object_put(poResponse);
640 break;
641 }
642 for(auto i=decltype(nHits){0};i<nHits;i++)
643 {
644 json_object* poHit = json_object_array_get_idx(poHits, i);
645 if( poHit == nullptr || json_object_get_type(poHit) != json_type_object )
646 {
647 continue;
648 }
649 json_object* poSource = CPL_json_object_object_get(poHit, "_source");
650 if( poSource == nullptr || json_object_get_type(poSource) != json_type_object )
651 {
652 continue;
653 }
654
655 if( !m_osESSearch.empty() )
656 {
657 json_object* poIndex = CPL_json_object_object_get(poHit, "_index");
658 if( poIndex == nullptr || json_object_get_type(poIndex) != json_type_string )
659 break;
660 if (m_poDS->m_nMajorVersion < 7)
661 {
662 json_object* poType = CPL_json_object_object_get(poHit, "_type");
663 if( poType == nullptr || json_object_get_type(poType) != json_type_string )
664 break;
665 m_osMappingName = json_object_get_string(poType);
666 }
667 CPLString osIndex(json_object_get_string(poIndex));
668
669 if( oVisited.find( std::pair<CPLString,CPLString>(osIndex, m_osMappingName) ) == oVisited.end() )
670 {
671 oVisited.insert( std::pair<CPLString,CPLString>(osIndex, m_osMappingName) );
672
673 CPLString osURL = CPLSPrintf("%s/%s/_mapping", m_poDS->GetURL(), osIndex.c_str());
674 if (m_poDS->m_nMajorVersion < 7)
675 osURL += CPLSPrintf("/%s", m_osMappingName.c_str());
676 osURL += "?pretty";
677
678 json_object* poMappingRes = m_poDS->RunRequest(osURL);
679 if( poMappingRes )
680 {
681 json_object* poLayerObj = CPL_json_object_object_get(poMappingRes, osIndex);
682 json_object* poMappings = nullptr;
683 if( poLayerObj && json_object_get_type(poLayerObj) == json_type_object )
684 poMappings = CPL_json_object_object_get(poLayerObj, "mappings");
685 if( poMappings && json_object_get_type(poMappings) == json_type_object )
686 {
687 json_object* poMapping = m_poDS->m_nMajorVersion < 7
688 ? CPL_json_object_object_get(poMappings, m_osMappingName)
689 : poMappings;
690 if( poMapping)
691 {
692 InitFeatureDefnFromMapping(poMapping, "", std::vector<CPLString>());
693 }
694 }
695 json_object_put(poMappingRes);
696 }
697 }
698 }
699
700 json_object_iter it;
701 it.key = nullptr;
702 it.val = nullptr;
703 it.entry = nullptr;
704 json_object_object_foreachC( poSource, it )
705 {
706 if( !m_osFID.empty() )
707 {
708 if( EQUAL(it.key, m_osFID) )
709 continue;
710 }
711 else if( EQUAL(it.key, m_poDS->GetFID()) )
712 {
713 m_osFID = it.key;
714 continue;
715 }
716
717 if( m_osMappingName == "FeatureCollection" )
718 {
719 if( strcmp(it.key, "properties") == 0 &&
720 json_object_get_type(it.val) == json_type_object )
721 {
722 json_object_iter it2;
723 it2.key = nullptr;
724 it2.val = nullptr;
725 it2.entry = nullptr;
726 json_object_object_foreachC( it.val, it2 )
727 {
728 std::vector<CPLString> aosPath;
729 aosPath.push_back("properties");
730 AddOrUpdateField(it2.key, it2.key, it2.val, '.', aosPath);
731 }
732 }
733 }
734 else
735 {
736 std::vector<CPLString> aosPath;
737 AddOrUpdateField(it.key, it.key, it.val, '.', aosPath);
738 }
739 }
740
741 nAlreadyQueried ++;
742 if( nFeatureCountToEstablishFeatureDefn > 0 &&
743 nAlreadyQueried >= nFeatureCountToEstablishFeatureDefn )
744 {
745 break;
746 }
747 }
748
749 json_object_put(poResponse);
750
751 if( nFeatureCountToEstablishFeatureDefn > 0 &&
752 nAlreadyQueried >= nFeatureCountToEstablishFeatureDefn )
753 {
754 break;
755 }
756 }
757
758 ResetReading();
759 }
760
761 if( m_poDS->m_bJSonField )
762 {
763 AddFieldDefn("_json", OFTString, std::vector<CPLString>() );
764 }
765 }
766
767 /************************************************************************/
768 /* BuildPathFromArray() */
769 /************************************************************************/
770
771 CPLString OGRElasticLayer::BuildPathFromArray(const std::vector<CPLString>& aosPath)
772 {
773 CPLString osPath(aosPath[0]);
774 for(size_t i=1;i<aosPath.size();i++)
775 {
776 osPath += ".";
777 osPath += aosPath[i];
778 }
779 return osPath;
780 }
781
782 /************************************************************************/
783 /* GetOGRGeomTypeFromES() */
784 /************************************************************************/
785
786 static OGRwkbGeometryType GetOGRGeomTypeFromES(const char* pszType)
787 {
788 if( EQUAL( pszType, "envelope") )
789 return wkbPolygon;
790 if( EQUAL( pszType, "circle") )
791 return wkbPolygon;
792 return OGRFromOGCGeomType(pszType);
793 }
794
795 /************************************************************************/
796 /* AddOrUpdateField() */
797 /************************************************************************/
798
799 void OGRElasticLayer::AddOrUpdateField(const char* pszAttrName,
800 const char* pszKey,
801 json_object* poObj,
802 char chNestedAttributeSeparator,
803 std::vector<CPLString>& aosPath)
804 {
805 json_type eJSONType = json_object_get_type(poObj);
806 if( eJSONType == json_type_null )
807 return;
808
809 if( eJSONType == json_type_object )
810 {
811 json_object* poType = CPL_json_object_object_get(poObj, "type");
812 OGRwkbGeometryType eGeomType;
813 if( poType && json_object_get_type(poType) == json_type_string &&
814 (eGeomType = GetOGRGeomTypeFromES(json_object_get_string(poType))) != wkbUnknown &&
815 CPL_json_object_object_get(poObj, (eGeomType == wkbGeometryCollection) ? "geometries" : "coordinates") )
816 {
817 int nIndex = m_poFeatureDefn->GetGeomFieldIndex(pszAttrName);
818 if( nIndex < 0 )
819 {
820 aosPath.push_back(pszKey);
821 AddGeomFieldDefn( pszAttrName, eGeomType, aosPath, FALSE );
822 }
823 else
824 {
825 OGRGeomFieldDefn* poFDefn = m_poFeatureDefn->GetGeomFieldDefn(nIndex);
826 if( poFDefn->GetType() != eGeomType )
827 poFDefn->SetType(wkbUnknown);
828 }
829 }
830 else if( m_poDS->m_bFlattenNestedAttributes )
831 {
832 if( m_poFeatureDefn->GetGeomFieldIndex(pszAttrName) >= 0 )
833 return;
834 aosPath.push_back(pszKey);
835
836 json_object_iter it;
837 it.key = nullptr;
838 it.val = nullptr;
839 it.entry = nullptr;
840 json_object_object_foreachC( poObj, it )
841 {
842 char szSeparator[2];
843 szSeparator[0] = chNestedAttributeSeparator;
844 szSeparator[1] = 0;
845 CPLString osAttrName(CPLSPrintf("%s%s%s", pszAttrName, szSeparator,
846 it.key));
847
848 std::vector<CPLString> aosNewPaths(aosPath);
849 AddOrUpdateField(osAttrName, it.key, it.val, chNestedAttributeSeparator,
850 aosNewPaths);
851 }
852 return;
853 }
854 }
855 /*else if( eJSONType == json_type_array )
856 {
857 if( m_poFeatureDefn->GetGeomFieldIndex(pszAttrName) >= 0 )
858 return;
859 }*/
860
861 if( m_poFeatureDefn->GetGeomFieldIndex(pszAttrName) >= 0 )
862 return;
863
864 OGRFieldSubType eNewSubType;
865 OGRFieldType eNewType = GeoJSONPropertyToFieldType( poObj, eNewSubType );
866
867 int nIndex = m_poFeatureDefn->GetFieldIndex(pszAttrName);
868 OGRFieldDefn* poFDefn = nullptr;
869 if( nIndex >= 0 )
870 poFDefn = m_poFeatureDefn->GetFieldDefn(nIndex);
871 if( (poFDefn == nullptr && eNewType == OFTString) ||
872 (poFDefn != nullptr &&
873 (poFDefn->GetType() == OFTDate || poFDefn->GetType() == OFTDateTime || poFDefn->GetType() == OFTTime) ) )
874 {
875 int nYear = 0;
876 int nMonth = 0;
877 int nDay = 0;
878 int nHour = 0;
879 int nMinute = 0;
880 float fSecond = 0.0f;
881 if( sscanf(json_object_get_string(poObj),
882 "%04d/%02d/%02d %02d:%02d",
883 &nYear, &nMonth, &nDay, &nHour, &nMinute) == 5 ||
884 sscanf(json_object_get_string(poObj),
885 "%04d-%02d-%02dT%02d:%02d",
886 &nYear, &nMonth, &nDay, &nHour, &nMinute) == 5 )
887 {
888 eNewType = OFTDateTime;
889 }
890 else if( sscanf(json_object_get_string(poObj),
891 "%04d/%02d/%02d",
892 &nYear, &nMonth, &nDay) == 3 ||
893 sscanf(json_object_get_string(poObj),
894 "%04d-%02d-%02d",
895 &nYear, &nMonth, &nDay) == 3)
896 {
897 eNewType = OFTDate;
898 }
899 else if( sscanf(json_object_get_string(poObj),
900 "%02d:%02d:%f",
901 &nHour, &nMinute, &fSecond) == 3 )
902 {
903 eNewType = OFTTime;
904 }
905 }
906
907 if( poFDefn == nullptr )
908 {
909 aosPath.push_back(pszKey);
910 AddFieldDefn( pszAttrName, eNewType, aosPath, eNewSubType );
911 }
912 else
913 {
914 OGRUpdateFieldType(poFDefn, eNewType, eNewSubType);
915 }
916 }
917
918 /************************************************************************/
919 /* SyncToDisk() */
920 /************************************************************************/
921
922 OGRErr OGRElasticLayer::SyncToDisk()
923 {
924 if( WriteMapIfNecessary() != OGRERR_NONE )
925 return OGRERR_FAILURE;
926
927 if( !PushIndex() )
928 return OGRERR_FAILURE;
929
930 return OGRERR_NONE;
931 }
932
933 /************************************************************************/
934 /* GetLayerDefn() */
935 /************************************************************************/
936
937 OGRFeatureDefn * OGRElasticLayer::GetLayerDefn() {
938
939 FinalizeFeatureDefn();
940
941 return m_poFeatureDefn;
942 }
943
944 /************************************************************************/
945 /* GetFIDColumn() */
946 /************************************************************************/
947
948 const char* OGRElasticLayer::GetFIDColumn()
949 {
950 GetLayerDefn();
951 return m_osFID.c_str();
952 }
953
954 /************************************************************************/
955 /* ResetReading() */
956 /************************************************************************/
957
958 void OGRElasticLayer::ResetReading()
959 {
960 if( !m_osScrollID.empty() )
961 {
962 char** papszOptions = CSLAddNameValue(nullptr, "CUSTOMREQUEST", "DELETE");
963 CPLHTTPResult* psResult = m_poDS->HTTPFetch((m_poDS->GetURL() + CPLString("/_search/scroll?scroll_id=") + m_osScrollID).c_str(), papszOptions);
964 CSLDestroy(papszOptions);
965 CPLHTTPDestroyResult(psResult);
966
967 m_osScrollID = "";
968 }
969 for(int i=0;i<(int)m_apoCachedFeatures.size();i++)
970 delete m_apoCachedFeatures[i];
971 m_apoCachedFeatures.resize(0);
972 m_iCurID = 0;
973 m_iCurFeatureInPage = 0;
974 m_bEOF = false;
975
976 m_nReadFeaturesSinceResetReading = 0;
977 m_dfEndTimeStamp = 0;
978 const double dfTimeout =
979 m_bUseSingleQueryParams ? m_dfSingleQueryTimeout : m_dfFeatureIterationTimeout;
980 if( dfTimeout > 0 )
981 m_dfEndTimeStamp = GetTimestamp() + dfTimeout;
982 }
983
984 /************************************************************************/
985 /* GetNextFeature() */
986 /************************************************************************/
987
988 OGRFeature *OGRElasticLayer::GetNextFeature()
989
990 {
991 FinalizeFeatureDefn();
992
993 while( true )
994 {
995 OGRFeature *poFeature = GetNextRawFeature();
996 if( poFeature == nullptr )
997 return nullptr;
998
999 if( (m_poFilterGeom == nullptr
1000 || FilterGeometry( poFeature->GetGeomFieldRef(m_iGeomFieldFilter) ) )
1001 && (m_poAttrQuery == nullptr
1002 || m_poAttrQuery->Evaluate( poFeature )) )
1003 return poFeature;
1004
1005 delete poFeature;
1006 }
1007 }
1008
1009 /************************************************************************/
1010 /* BuildSort() */
1011 /************************************************************************/
1012
1013 json_object* OGRElasticLayer::BuildSort()
1014 {
1015 json_object* poRet = json_object_new_array();
1016 for( size_t i=0; i<m_aoSortColumns.size(); ++i)
1017 {
1018 const int nIdx =
1019 m_poFeatureDefn->GetFieldIndex(m_aoSortColumns[i].osColumn);
1020 CPLString osFieldName( nIdx == 0 ? "_uid" :
1021 BuildPathFromArray(m_aaosFieldPaths[ nIdx ]));
1022 if( CSLFindString(m_papszFieldsWithRawValue,
1023 m_aoSortColumns[i].osColumn) >= 0 )
1024 {
1025 osFieldName += ".raw";
1026 }
1027 json_object* poSortCol = json_object_new_object();
1028 json_object* poSortProp = json_object_new_object();
1029 json_object_array_add(poRet, poSortCol);
1030 json_object_object_add(poSortProp, "order",
1031 json_object_new_string(m_aoSortColumns[i].bAsc ? "asc" : "desc"));
1032 json_object_object_add(poSortCol, osFieldName, poSortProp);
1033 }
1034 return poRet;
1035 }
1036
1037 /************************************************************************/
1038 /* BuildQuery() */
1039 /************************************************************************/
1040
1041 CPLString OGRElasticLayer::BuildQuery(bool bCountOnly)
1042 {
1043 CPLString osRet = "{ ";
1044 if( bCountOnly &&
1045 (m_poDS->m_nMajorVersion < 5 || !m_osSingleQueryTimeout.empty()) )
1046 {
1047 osRet += "\"size\": 0, ";
1048 }
1049 if( m_poSpatialFilter && m_poJSONFilter)
1050 {
1051 osRet += CPLSPrintf(
1052 "\"query\": { \"constant_score\" : { \"filter\": "
1053 "{ \"bool\" : { \"must\" : [%s, %s] } } } }",
1054 json_object_to_json_string( m_poSpatialFilter ),
1055 json_object_to_json_string( m_poJSONFilter ));
1056 }
1057 else
1058 {
1059 osRet += CPLSPrintf(
1060 "\"query\": { \"constant_score\" : { \"filter\": %s } }",
1061 json_object_to_json_string( m_poSpatialFilter ?
1062 m_poSpatialFilter : m_poJSONFilter ));
1063 }
1064 if( !bCountOnly && !m_aoSortColumns.empty() )
1065 {
1066 json_object* poSort = BuildSort();
1067 osRet += CPLSPrintf(", \"sort\" : %s",
1068 json_object_to_json_string(poSort));
1069 json_object_put(poSort);
1070 }
1071 osRet += " }";
1072 return osRet;
1073 }
1074
1075 /************************************************************************/
1076 /* GetNextRawFeature() */
1077 /************************************************************************/
1078
1079 OGRFeature *OGRElasticLayer::GetNextRawFeature()
1080 {
1081 json_object* poResponse = nullptr;
1082
1083 if( m_dfEndTimeStamp > 0 && GetTimestamp() >= m_dfEndTimeStamp )
1084 {
1085 CPLDebug("ES", "Terminating request due to timeout");
1086 return nullptr;
1087 }
1088 const auto nTerminateAfter =
1089 m_bUseSingleQueryParams ? m_nSingleQueryTerminateAfter : m_nFeatureIterationTerminateAfter;
1090 if( nTerminateAfter > 0 && m_nReadFeaturesSinceResetReading >= nTerminateAfter )
1091 {
1092 CPLDebug("ES", "Terminating request due to terminate_after reached");
1093 return nullptr;
1094 }
1095
1096 if( m_bEOF )
1097 return nullptr;
1098
1099 if( m_iCurFeatureInPage < (int)m_apoCachedFeatures.size() )
1100 {
1101 OGRFeature* poRet = m_apoCachedFeatures[m_iCurFeatureInPage];
1102 m_apoCachedFeatures[m_iCurFeatureInPage] = nullptr;
1103 m_iCurFeatureInPage ++;
1104 m_nReadFeaturesSinceResetReading ++;
1105 return poRet;
1106 }
1107
1108 for(int i=0;i<(int)m_apoCachedFeatures.size();i++)
1109 delete m_apoCachedFeatures[i];
1110 m_apoCachedFeatures.resize(0);
1111 m_iCurFeatureInPage = 0;
1112
1113 CPLString osRequest, osPostData;
1114 if( m_nReadFeaturesSinceResetReading == 0 )
1115 {
1116 if( !m_osESSearch.empty() )
1117 {
1118 osRequest = CPLSPrintf("%s/_search?scroll=1m&size=%d",
1119 m_poDS->GetURL(), m_poDS->m_nBatchSize);
1120 osPostData = m_osESSearch;
1121 }
1122 else if( (m_poSpatialFilter && m_osJSONFilter.empty()) || m_poJSONFilter )
1123 {
1124 osPostData = BuildQuery(false);
1125 osRequest = BuildMappingURL(false);
1126 osRequest += CPLSPrintf("/_search?scroll=1m&size=%d", m_poDS->m_nBatchSize);
1127 }
1128 else if( !m_aoSortColumns.empty() && m_osJSONFilter.empty() )
1129 {
1130 osRequest = BuildMappingURL(false);
1131 osRequest += CPLSPrintf("/_search?scroll=1m&size=%d", m_poDS->m_nBatchSize);
1132 json_object* poSort = BuildSort();
1133 osPostData = CPLSPrintf(
1134 "{ \"sort\": %s }",
1135 json_object_to_json_string(poSort));
1136 json_object_put(poSort);
1137 }
1138 else
1139 {
1140 osRequest = BuildMappingURL(false);
1141 osRequest += CPLSPrintf("/_search?scroll=1m&size=%d", m_poDS->m_nBatchSize);
1142 osPostData = m_osJSONFilter;
1143 }
1144 }
1145 else
1146 {
1147 if( m_osScrollID.empty() )
1148 {
1149 m_bEOF = true;
1150 return nullptr;
1151 }
1152 osRequest =
1153 CPLSPrintf("%s/_search/scroll?scroll=1m&scroll_id=%s",
1154 m_poDS->GetURL(), m_osScrollID.c_str());
1155 }
1156
1157 if( m_bAddPretty )
1158 osRequest += "&pretty";
1159 poResponse = m_poDS->RunRequest(osRequest, osPostData);
1160 if( poResponse == nullptr )
1161 {
1162 m_bEOF = true;
1163 return nullptr;
1164 }
1165 m_osScrollID.clear();
1166 json_object* poScrollID = CPL_json_object_object_get(poResponse, "_scroll_id");
1167 if( poScrollID )
1168 {
1169 const char* pszScrollID = json_object_get_string(poScrollID);
1170 if( pszScrollID )
1171 m_osScrollID = pszScrollID;
1172 }
1173
1174 json_object* poHits = CPL_json_object_object_get(poResponse, "hits");
1175 if( poHits == nullptr || json_object_get_type(poHits) != json_type_object )
1176 {
1177 m_bEOF = true;
1178 json_object_put(poResponse);
1179 return nullptr;
1180 }
1181 poHits = CPL_json_object_object_get(poHits, "hits");
1182 if( poHits == nullptr || json_object_get_type(poHits) != json_type_array )
1183 {
1184 m_bEOF = true;
1185 json_object_put(poResponse);
1186 return nullptr;
1187 }
1188 const auto nHits = json_object_array_length(poHits);
1189 if( nHits == 0 )
1190 {
1191 m_osScrollID = "";
1192 m_bEOF = true;
1193 json_object_put(poResponse);
1194 return nullptr;
1195 }
1196 for(auto i=decltype(nHits){0};i<nHits;i++)
1197 {
1198 json_object* poHit = json_object_array_get_idx(poHits, i);
1199 if( poHit == nullptr || json_object_get_type(poHit) != json_type_object )
1200 {
1201 continue;
1202 }
1203 json_object* poSource = CPL_json_object_object_get(poHit, "_source");
1204 if( poSource == nullptr || json_object_get_type(poSource) != json_type_object )
1205 {
1206 continue;
1207 }
1208
1209 const char* pszId = nullptr;
1210 json_object* poId = CPL_json_object_object_get(poHit, "_id");
1211 if( poId != nullptr && json_object_get_type(poId) == json_type_string )
1212 pszId = json_object_get_string(poId);
1213
1214 OGRFeature* poFeature = new OGRFeature(m_poFeatureDefn);
1215 if( pszId )
1216 poFeature->SetField("_id", pszId);
1217
1218 if( !m_osESSearch.empty() )
1219 {
1220 json_object* poIndex = CPL_json_object_object_get(poHit, "_index");
1221 if( poIndex != nullptr && json_object_get_type(poIndex) == json_type_string )
1222 poFeature->SetField("_index", json_object_get_string(poIndex));
1223
1224 json_object* poType = CPL_json_object_object_get(poHit, "_type");
1225 if( poType != nullptr && json_object_get_type(poType) == json_type_string )
1226 poFeature->SetField("_type", json_object_get_string(poType));
1227 }
1228
1229 if( m_poDS->m_bJSonField )
1230 poFeature->SetField("_json", json_object_to_json_string(poSource));
1231
1232 BuildFeature(poFeature, poSource, CPLString());
1233 if( poFeature->GetFID() < 0 )
1234 poFeature->SetFID( ++m_iCurID );
1235 m_apoCachedFeatures.push_back(poFeature);
1236 }
1237
1238 json_object_put(poResponse);
1239 if( !m_apoCachedFeatures.empty() )
1240 {
1241 OGRFeature* poRet = m_apoCachedFeatures[ 0 ];
1242 m_apoCachedFeatures[ 0 ] = nullptr;
1243 m_iCurFeatureInPage ++;
1244 m_nReadFeaturesSinceResetReading ++;
1245 return poRet;
1246 }
1247 return nullptr;
1248 }
1249
1250 /************************************************************************/
1251 /* decode_geohash_bbox() */
1252 /************************************************************************/
1253
1254 /* Derived from routine from https://github.com/davetroy/geohash/blob/master/ext/geohash_native.c */
1255 /* (c) 2008-2010 David Troy, davetroy@gmail.com, (The MIT License) */
1256
1257 static const char BASE32[] = "0123456789bcdefghjkmnpqrstuvwxyz";
1258
1259 static void decode_geohash_bbox( const char *geohash, double lat[2],
1260 double lon[2] )
1261 {
1262 int i;
1263 int j;
1264 int hashlen;
1265 char c;
1266 char cd;
1267 char mask;
1268 char is_even = 1;
1269 static const char bits[] = {16,8,4,2,1};
1270 lat[0] = -90.0;
1271 lat[1] = 90.0;
1272 lon[0] = -180.0;
1273 lon[1] = 180.0;
1274 hashlen = static_cast<int>(strlen(geohash));
1275 for( i = 0; i<hashlen; i++ )
1276 {
1277 c = static_cast<char>(tolower(geohash[i]));
1278 cd = static_cast<char>(strchr(BASE32, c)-BASE32);
1279 for (j=0; j<5; j++) {
1280 mask = bits[j];
1281 if (is_even) {
1282 lon[!(cd&mask)] = (lon[0] + lon[1])/2;
1283 } else {
1284 lat[!(cd&mask)] = (lat[0] + lat[1])/2;
1285 }
1286 is_even = !is_even;
1287 }
1288 }
1289 }
1290
1291 /************************************************************************/
1292 /* BuildFeature() */
1293 /************************************************************************/
1294
1295 void OGRElasticLayer::BuildFeature(OGRFeature* poFeature, json_object* poSource,
1296 CPLString osPath)
1297 {
1298 json_object_iter it;
1299 it.key = nullptr;
1300 it.val = nullptr;
1301 it.entry = nullptr;
1302 CPLString osCurPath;
1303 json_object_object_foreachC( poSource, it )
1304 {
1305 if( osPath.empty() &&
1306 !m_osFID.empty() && EQUAL(m_osFID, it.key) )
1307 {
1308 json_type eJSONType = json_object_get_type(it.val);
1309 if( eJSONType == json_type_int )
1310 {
1311 poFeature->SetFID((GIntBig)json_object_get_int64(it.val));
1312 continue;
1313 }
1314 }
1315
1316 if( !osPath.empty() )
1317 osCurPath = osPath + "." + it.key;
1318 else
1319 osCurPath = it.key;
1320 std::map<CPLString,int>::iterator oIter = m_aosMapToFieldIndex.find(osCurPath);
1321 if( oIter != m_aosMapToFieldIndex.end() )
1322 {
1323 switch( json_object_get_type(it.val) )
1324 {
1325 case json_type_null:
1326 poFeature->SetFieldNull( oIter->second );
1327 break;
1328 case json_type_boolean:
1329 poFeature->SetField( oIter->second, json_object_get_boolean(it.val));
1330 break;
1331 case json_type_int:
1332 poFeature->SetField( oIter->second, (GIntBig)json_object_get_int64(it.val));
1333 break;
1334 case json_type_double:
1335 poFeature->SetField( oIter->second, json_object_get_double(it.val));
1336 break;
1337 case json_type_array:
1338 {
1339 if( m_poFeatureDefn->GetFieldDefn(oIter->second)->GetType() == OFTIntegerList )
1340 {
1341 std::vector<int> anValues;
1342 const auto nLength = json_object_array_length(it.val);
1343 for(auto i=decltype(nLength){0};i<nLength;i++)
1344 {
1345 anValues.push_back( json_object_get_int( json_object_array_get_idx( it.val, i ) ) );
1346 }
1347 if( nLength )
1348 poFeature->SetField( oIter->second,
1349 static_cast<int>(nLength),
1350 &anValues[0] );
1351 }
1352 else if( m_poFeatureDefn->GetFieldDefn(oIter->second)->GetType() == OFTInteger64List )
1353 {
1354 std::vector<GIntBig> anValues;
1355 const auto nLength = json_object_array_length(it.val);
1356 for(auto i=decltype(nLength){0};i<nLength;i++)
1357 {
1358 anValues.push_back( json_object_get_int64( json_object_array_get_idx( it.val, i ) ) );
1359 }
1360 if( nLength )
1361 poFeature->SetField( oIter->second,
1362 static_cast<int>(nLength),
1363 &anValues[0] );
1364 }
1365 else if( m_poFeatureDefn->GetFieldDefn(oIter->second)->GetType() == OFTRealList )
1366 {
1367 std::vector<double> adfValues;
1368 const auto nLength = json_object_array_length(it.val);
1369 for(auto i=decltype(nLength){0};i<nLength;i++)
1370 {
1371 adfValues.push_back( json_object_get_double( json_object_array_get_idx( it.val, i ) ) );
1372 }
1373 if( nLength )
1374 poFeature->SetField( oIter->second,
1375 static_cast<int>(nLength),
1376 &adfValues[0] );
1377 }
1378 else if( m_poFeatureDefn->GetFieldDefn(oIter->second)->GetType() == OFTStringList )
1379 {
1380 std::vector<char*> apszValues;
1381 const auto nLength = json_object_array_length(it.val);
1382 for(auto i=decltype(nLength){0};i<nLength;i++)
1383 {
1384 apszValues.push_back( CPLStrdup(json_object_get_string( json_object_array_get_idx( it.val, i ) )) );
1385 }
1386 apszValues.push_back( nullptr);
1387 poFeature->SetField( oIter->second, &apszValues[0] );
1388 for(auto i=decltype(nLength){0};i<nLength;i++)
1389 {
1390 CPLFree(apszValues[i]);
1391 }
1392 }
1393 break;
1394 }
1395 default:
1396 {
1397 if( m_poFeatureDefn->GetFieldDefn(oIter->second)->GetType() == OFTBinary )
1398 {
1399 GByte* pabyBase64 = (GByte*) CPLStrdup( json_object_get_string(it.val) );
1400 int nBytes = CPLBase64DecodeInPlace( pabyBase64 );
1401 poFeature->SetField( oIter->second, nBytes, pabyBase64 );
1402 CPLFree(pabyBase64);
1403 }
1404 else
1405 {
1406 poFeature->SetField( oIter->second, json_object_get_string(it.val));
1407 }
1408 break;
1409 }
1410 }
1411 }
1412 else if( ( oIter = m_aosMapToGeomFieldIndex.find(osCurPath) ) != m_aosMapToGeomFieldIndex.end() )
1413 {
1414 OGRGeometry* poGeom = nullptr;
1415 if( m_abIsGeoPoint[oIter->second] )
1416 {
1417 json_type eJSONType = json_object_get_type(it.val);
1418 if( eJSONType == json_type_array &&
1419 json_object_array_length(it.val) == 2 )
1420 {
1421 json_object* poX = json_object_array_get_idx(it.val, 0);
1422 json_object* poY = json_object_array_get_idx(it.val, 1);
1423 if( poX != nullptr && poY != nullptr )
1424 {
1425 poGeom = new OGRPoint( json_object_get_double(poX),
1426 json_object_get_double(poY) );
1427 }
1428 }
1429 else if( eJSONType == json_type_object )
1430 {
1431 json_object* poX = CPL_json_object_object_get(it.val, "lon");
1432 json_object* poY = CPL_json_object_object_get(it.val, "lat");
1433 if( poX != nullptr && poY != nullptr )
1434 {
1435 poGeom = new OGRPoint( json_object_get_double(poX),
1436 json_object_get_double(poY) );
1437 }
1438 }
1439 else if( eJSONType == json_type_string )
1440 {
1441 const char* pszLatLon = json_object_get_string(it.val);
1442 char** papszTokens = CSLTokenizeString2(pszLatLon, ",", 0);
1443 if( CSLCount(papszTokens) == 2 )
1444 {
1445 poGeom = new OGRPoint( CPLAtof(papszTokens[1]),
1446 CPLAtof(papszTokens[0]) );
1447 }
1448 else
1449 {
1450 double lat[2] = { 0.0, 0.0 };
1451 double lon[2] = { 0.0, 0.0 };
1452 decode_geohash_bbox(pszLatLon, lat, lon);
1453 poGeom = new OGRPoint( (lon[0] + lon[1]) / 2,
1454 (lat[0] + lat[1]) / 2 );
1455 }
1456
1457 CSLDestroy(papszTokens);
1458 }
1459 }
1460 else if( json_object_get_type(it.val) == json_type_object )
1461 {
1462 json_object* poType = CPL_json_object_object_get(it.val, "type");
1463 json_object* poRadius = CPL_json_object_object_get(it.val, "radius");
1464 json_object* poCoordinates = CPL_json_object_object_get(it.val, "coordinates");
1465 if( poType && poRadius && poCoordinates &&
1466 json_object_get_type(poType) == json_type_string &&
1467 EQUAL( json_object_get_string(poType), "circle" ) &&
1468 (json_object_get_type(poRadius) == json_type_string ||
1469 json_object_get_type(poRadius) == json_type_double ||
1470 json_object_get_type(poRadius) == json_type_int ) &&
1471 json_object_get_type(poCoordinates) == json_type_array &&
1472 json_object_array_length(poCoordinates) == 2 )
1473 {
1474 const char* pszRadius = json_object_get_string(poRadius);
1475 const double dfX = json_object_get_double(json_object_array_get_idx(poCoordinates, 0));
1476 const double dfY = json_object_get_double(json_object_array_get_idx(poCoordinates, 1));
1477 const int nRadiusLength = (int)strlen(pszRadius);
1478 double dfRadius = CPLAtof(pszRadius);
1479 double dfUnit = 0.0;
1480 if( nRadiusLength >= 1 && pszRadius[nRadiusLength-1] == 'm' )
1481 {
1482 if( nRadiusLength >= 2 && pszRadius[nRadiusLength-2] == 'k' )
1483 dfUnit = 1000;
1484 else if( nRadiusLength >= 2 &&
1485 pszRadius[nRadiusLength-2] >= '0' &&
1486 pszRadius[nRadiusLength-2] <= '9' )
1487 dfUnit = 1;
1488 }
1489 else if ( nRadiusLength >= 1 &&
1490 pszRadius[nRadiusLength-1] >= '0' &&
1491 pszRadius[nRadiusLength-1] <= '9' )
1492 {
1493 dfUnit = 1;
1494 }
1495
1496 if( dfRadius == 0 )
1497 CPLError(CE_Warning, CPLE_AppDefined, "Unknown unit in %s", pszRadius);
1498 else
1499 {
1500 dfRadius *= dfUnit;
1501 OGRLinearRing* poRing = new OGRLinearRing();
1502 for(double dfStep = 0; dfStep <= 360; dfStep += 4 )
1503 {
1504 double dfLat = 0.0;
1505 double dfLon = 0.0;
1506 OGR_GreatCircle_ExtendPosition(dfY, dfX, dfRadius,
1507 dfStep, &dfLat, &dfLon);
1508 poRing->addPoint(dfLon, dfLat);
1509 }
1510 OGRPolygon* poPoly = new OGRPolygon();
1511 poPoly->addRingDirectly(poRing);
1512 poGeom = poPoly;
1513 }
1514 }
1515 else if( poType && poCoordinates &&
1516 json_object_get_type(poType) == json_type_string &&
1517 EQUAL( json_object_get_string(poType), "envelope" ) &&
1518 json_object_get_type(poCoordinates) == json_type_array &&
1519 json_object_array_length(poCoordinates) == 2 )
1520 {
1521 json_object* poCorner1 = json_object_array_get_idx(poCoordinates, 0);
1522 json_object* poCorner2 = json_object_array_get_idx(poCoordinates, 1);
1523 if( poCorner1 && poCorner2 &&
1524 json_object_get_type(poCorner1) == json_type_array &&
1525 json_object_array_length(poCorner1) == 2 &&
1526 json_object_get_type(poCorner2) == json_type_array &&
1527 json_object_array_length(poCorner2) == 2 )
1528 {
1529 const double dfX1 = json_object_get_double(json_object_array_get_idx(poCorner1, 0));
1530 const double dfY1 = json_object_get_double(json_object_array_get_idx(poCorner1, 1));
1531 const double dfX2 = json_object_get_double(json_object_array_get_idx(poCorner2, 0));
1532 const double dfY2 = json_object_get_double(json_object_array_get_idx(poCorner2, 1));
1533 OGRLinearRing* poRing = new OGRLinearRing();
1534 poRing->addPoint(dfX1, dfY1);
1535 poRing->addPoint(dfX2, dfY1);
1536 poRing->addPoint(dfX2, dfY2);
1537 poRing->addPoint(dfX1, dfY2);
1538 poRing->addPoint(dfX1, dfY1);
1539 OGRPolygon* poPoly = new OGRPolygon();
1540 poPoly->addRingDirectly(poRing);
1541 poGeom = poPoly;
1542 }
1543 }
1544 else
1545 {
1546 poGeom = OGRGeoJSONReadGeometry( it.val );
1547 }
1548 }
1549 else if( json_object_get_type(it.val) == json_type_string )
1550 {
1551 // Assume this is WKT
1552 OGRGeometryFactory::createFromWkt(
1553 json_object_get_string(it.val), nullptr, &poGeom);
1554 }
1555
1556 if( poGeom != nullptr )
1557 {
1558 poGeom->assignSpatialReference( m_poFeatureDefn->GetGeomFieldDefn(oIter->second)->GetSpatialRef() );
1559 poFeature->SetGeomFieldDirectly( oIter->second, poGeom );
1560 }
1561 }
1562 else if( json_object_get_type(it.val) == json_type_object &&
1563 (m_poDS->m_bFlattenNestedAttributes ||
1564 (osPath.empty() && m_osMappingName == "FeatureCollection" && strcmp(it.key, "properties") == 0)) )
1565 {
1566 BuildFeature(poFeature, it.val, osCurPath);
1567 }
1568 else if( json_object_get_type(it.val) == json_type_object &&
1569 !m_poDS->m_bFlattenNestedAttributes )
1570 {
1571 if( m_aosMapToGeomFieldIndex.find(osCurPath + ".coordinates")
1572 != m_aosMapToGeomFieldIndex.end() )
1573 {
1574 BuildFeature(poFeature, it.val, osCurPath);
1575 }
1576 }
1577 }
1578 }
1579
1580 /************************************************************************/
1581 /* AppendGroup() */
1582 /************************************************************************/
1583
1584 static
1585 json_object *AppendGroup(json_object *parent, const CPLString &name) {
1586 json_object *obj = json_object_new_object();
1587 json_object *properties = json_object_new_object();
1588 json_object_object_add(parent, name, obj);
1589 json_object_object_add(obj, "properties", properties);
1590 return properties;
1591 }
1592
1593 /************************************************************************/
1594 /* AddPropertyMap() */
1595 /************************************************************************/
1596
1597 static json_object* AddPropertyMap(const CPLString &type) {
1598 json_object *obj = json_object_new_object();
1599 json_object_object_add(obj, "type", json_object_new_string(type.c_str()));
1600 return obj;
1601 }
1602
1603 /************************************************************************/
1604 /* GetContainerForMapping() */
1605 /************************************************************************/
1606
1607 static json_object* GetContainerForMapping( json_object* poContainer,
1608 const std::vector<CPLString>& aosPath,
1609 std::map< std::vector<CPLString>, json_object* >& oMap )
1610 {
1611 std::vector<CPLString> aosSubPath;
1612 for(int j=0;j<(int)aosPath.size()-1;j++)
1613 {
1614 aosSubPath.push_back(aosPath[j]);
1615 std::map< std::vector<CPLString>, json_object* >::iterator oIter = oMap.find(aosSubPath);
1616 if( oIter == oMap.end() )
1617 {
1618 json_object* poNewContainer = json_object_new_object();
1619 json_object* poProperties = json_object_new_object();
1620 json_object_object_add(poContainer, aosPath[j], poNewContainer);
1621 json_object_object_add(poNewContainer, "properties", poProperties);
1622 oMap[aosSubPath] = poProperties;
1623 poContainer = poProperties;
1624 }
1625 else
1626 {
1627 poContainer = oIter->second;
1628 }
1629 }
1630 return poContainer;
1631 }
1632
1633 /************************************************************************/
1634 /* BuildMap() */
1635 /************************************************************************/
1636
1637 CPLString OGRElasticLayer::BuildMap() {
1638 json_object *map = json_object_new_object();
1639
1640 std::map< std::vector<CPLString>, json_object* > oMap;
1641
1642 json_object *poMapping;
1643 json_object *poMappingProperties = json_object_new_object();
1644 if (m_poDS->m_nMajorVersion < 7)
1645 {
1646 poMapping = json_object_new_object();
1647 json_object_object_add(map, m_osMappingName, poMapping);
1648 }
1649 else
1650 {
1651 poMapping = map;
1652 }
1653 json_object_object_add(poMapping, "properties", poMappingProperties);
1654
1655 if( m_poDS->m_nMajorVersion < 7 && m_osMappingName == "FeatureCollection" )
1656 {
1657 json_object_object_add(poMappingProperties, "type", AddPropertyMap(
1658 m_poDS->m_nMajorVersion >= 5 ? "text" : "string"));
1659
1660 std::vector<CPLString> aosPath;
1661 aosPath.push_back("properties");
1662 aosPath.push_back("dummy");
1663 GetContainerForMapping(poMappingProperties, aosPath, oMap);
1664 }
1665
1666 /* skip _id field */
1667 for(int i=1;i<m_poFeatureDefn->GetFieldCount();i++)
1668 {
1669 OGRFieldDefn* poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1670
1671 json_object* poContainer = GetContainerForMapping(poMappingProperties,
1672 m_aaosFieldPaths[i],
1673 oMap);
1674 const char* pszLastComponent = m_aaosFieldPaths[i].back();
1675
1676 const char* pszType = "string";
1677 const char* pszFormat = nullptr;
1678
1679 switch (poFieldDefn->GetType())
1680 {
1681 case OFTInteger:
1682 case OFTIntegerList:
1683 {
1684 if( poFieldDefn->GetSubType() == OFSTBoolean )
1685 pszType = "boolean";
1686 else
1687 pszType = "integer";
1688 break;
1689 }
1690 case OFTInteger64:
1691 case OFTInteger64List:
1692 pszType = "long";
1693 break;
1694 case OFTReal:
1695 case OFTRealList:
1696 pszType = "double";
1697 break;
1698 case OFTDateTime:
1699 case OFTDate:
1700 pszType = "date";
1701 pszFormat = "yyyy/MM/dd HH:mm:ss.SSSZZ||yyyy/MM/dd HH:mm:ss.SSS||yyyy/MM/dd";
1702 break;
1703 case OFTTime:
1704 pszType = "date";
1705 pszFormat = "HH:mm:ss.SSS";
1706 break;
1707 case OFTBinary:
1708 pszType = "binary";
1709 break;
1710 default:
1711 break;
1712 }
1713
1714 bool bAnalyzed = EQUAL(pszType, "string");
1715 json_object* poPropertyMap = json_object_new_object();
1716 if( m_poDS->m_nMajorVersion >= 5 && EQUAL(pszType, "string") )
1717 {
1718 if( CSLFindString(m_papszNotAnalyzedFields,
1719 poFieldDefn->GetNameRef()) >= 0 ||
1720 (CSLCount(m_papszNotAnalyzedFields) == 1 &&
1721 EQUAL(m_papszNotAnalyzedFields[0], "{ALL}")) )
1722 {
1723 bAnalyzed = false;
1724 pszType = "keyword";
1725 }
1726 else
1727 pszType = "text";
1728 }
1729 json_object_object_add(poPropertyMap, "type", json_object_new_string(pszType));
1730 if( pszFormat )
1731 json_object_object_add(poPropertyMap, "format", json_object_new_string(pszFormat));
1732 if( m_bStoreFields || CSLFindString(m_papszStoredFields, poFieldDefn->GetNameRef()) >= 0 )
1733 json_object_object_add(poPropertyMap, "store", json_object_new_string("yes"));
1734 if( m_poDS->m_nMajorVersion < 5 &&
1735 (CSLFindString(m_papszNotAnalyzedFields,
1736 poFieldDefn->GetNameRef()) >= 0 ||
1737 (CSLCount(m_papszNotAnalyzedFields) == 1 &&
1738 EQUAL(m_papszNotAnalyzedFields[0], "{ALL}"))) )
1739 {
1740 bAnalyzed = false;
1741 json_object_object_add(poPropertyMap, "index", json_object_new_string("not_analyzed"));
1742 }
1743 else if( CSLFindString(m_papszNotIndexedFields, poFieldDefn->GetNameRef()) >= 0 )
1744 json_object_object_add(poPropertyMap, "index", json_object_new_string("no"));
1745
1746 if( bAnalyzed && (CSLFindString(m_papszFieldsWithRawValue,
1747 poFieldDefn->GetNameRef()) >= 0 ||
1748 (CSLCount(m_papszFieldsWithRawValue) == 1 &&
1749 EQUAL(m_papszFieldsWithRawValue[0], "{ALL}"))) )
1750 {
1751 json_object* poFields = json_object_new_object();
1752 json_object* poRaw = json_object_new_object();
1753 json_object_object_add(poFields, "raw", poRaw);
1754 if( m_poDS->m_nMajorVersion >= 5 )
1755 {
1756 json_object_object_add(poRaw, "type",
1757 json_object_new_string("keyword"));
1758 }
1759 else
1760 {
1761 json_object_object_add(poRaw, "type",
1762 json_object_new_string("string"));
1763 json_object_object_add(poRaw, "index",
1764 json_object_new_string("not_analyzed"));
1765 }
1766 json_object_object_add(poPropertyMap, "fields", poFields);
1767 }
1768
1769 json_object_object_add(poContainer, pszLastComponent, poPropertyMap);
1770 }
1771
1772 for(int i=0;i<m_poFeatureDefn->GetGeomFieldCount();i++)
1773 {
1774 std::vector<CPLString> aosPath = m_aaosGeomFieldPaths[i];
1775 bool bAddGeoJSONType = false;
1776 if( m_abIsGeoPoint[i] &&
1777 aosPath.size() >= 2 &&
1778 aosPath.back() == "coordinates" )
1779 {
1780 bAddGeoJSONType = true;
1781 aosPath.resize( (int)aosPath.size() - 1 );
1782 }
1783
1784 json_object* poContainer = GetContainerForMapping(poMappingProperties,
1785 aosPath,
1786 oMap);
1787 const char* pszLastComponent = aosPath.back();
1788
1789 if( m_abIsGeoPoint[i] )
1790 {
1791 json_object* geo_point = AddPropertyMap("geo_point");
1792 if( bAddGeoJSONType )
1793 {
1794 json_object *geometry = AppendGroup(poContainer, pszLastComponent);
1795 json_object_object_add(geometry, "type", AddPropertyMap(
1796 m_poDS->m_nMajorVersion >= 5 ? "text" : "string"));
1797 json_object_object_add(geometry, "coordinates", geo_point);
1798 }
1799 else
1800 {
1801 json_object_object_add(poContainer, pszLastComponent, geo_point);
1802 }
1803 if( !m_osPrecision.empty() )
1804 {
1805 json_object* field_data = json_object_new_object();
1806 json_object_object_add(geo_point, "fielddata", field_data);
1807 json_object_object_add(field_data, "format", json_object_new_string("compressed"));
1808 json_object_object_add(field_data, "precision", json_object_new_string(m_osPrecision.c_str()));
1809 }
1810 }
1811 else
1812 {
1813 json_object *geometry = json_object_new_object();
1814 json_object_object_add(poContainer,
1815 pszLastComponent,
1816 geometry);
1817 json_object_object_add(geometry, "type", json_object_new_string("geo_shape"));
1818 if( !m_osPrecision.empty() )
1819 json_object_object_add(geometry, "precision", json_object_new_string(m_osPrecision.c_str()));
1820 }
1821 }
1822
1823 json_object* poMeta = nullptr;
1824 json_object* poGeomFields = nullptr;
1825 json_object* poFields = nullptr;
1826 if( !m_osFID.empty() )
1827 {
1828 poMeta = json_object_new_object();
1829 json_object_object_add(poMeta, "fid",
1830 json_object_new_string(m_osFID.c_str()));
1831 }
1832 for( int i=0; i < m_poFeatureDefn->GetGeomFieldCount(); i++ )
1833 {
1834 OGRGeomFieldDefn* poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
1835 if( !m_abIsGeoPoint[i] &&
1836 poGeomFieldDefn->GetType() != wkbUnknown )
1837 {
1838 if( poMeta == nullptr )
1839 poMeta = json_object_new_object();
1840 if( poGeomFields == nullptr )
1841 {
1842 poGeomFields = json_object_new_object();
1843 json_object_object_add(poMeta, "geomfields", poGeomFields);
1844 }
1845 json_object_object_add(poGeomFields,
1846 poGeomFieldDefn->GetNameRef(),
1847 json_object_new_string(OGRToOGCGeomType(poGeomFieldDefn->GetType())));
1848 }
1849 }
1850 for( int i=0; i < m_poFeatureDefn->GetFieldCount(); i++ )
1851 {
1852 OGRFieldDefn* poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1853 OGRFieldType eType = poFieldDefn->GetType();
1854 if( eType == OFTIntegerList || eType == OFTInteger64List ||
1855 eType == OFTRealList || eType == OFTStringList )
1856 {
1857 if( poMeta == nullptr )
1858 poMeta = json_object_new_object();
1859 if( poFields == nullptr )
1860 {
1861 poFields = json_object_new_object();
1862 json_object_object_add(poMeta, "fields", poFields);
1863 }
1864 json_object_object_add(poFields,
1865 poFieldDefn->GetNameRef(),
1866 json_object_new_string(OGR_GetFieldTypeName(eType)));
1867 }
1868 }
1869 if( poMeta )
1870 json_object_object_add(poMapping, "_meta", poMeta );
1871
1872 CPLString jsonMap(json_object_to_json_string(map));
1873 json_object_put(map);
1874
1875 // Got personally caught by that...
1876 if( CSLCount(m_papszStoredFields) == 1 &&
1877 (EQUAL(m_papszStoredFields[0], "YES") || EQUAL(m_papszStoredFields[0], "TRUE")) &&
1878 m_poFeatureDefn->GetFieldIndex(m_papszStoredFields[0]) < 0 )
1879 {
1880 CPLError(CE_Warning, CPLE_AppDefined,
1881 "STORED_FIELDS=%s was specified. Perhaps you meant STORE_FIELDS=%s instead?",
1882 m_papszStoredFields[0], m_papszStoredFields[0]);
1883 }
1884
1885 return jsonMap;
1886 }
1887
1888 /************************************************************************/
1889 /* BuildGeoJSONGeometry() */
1890 /************************************************************************/
1891
1892 static void BuildGeoJSONGeometry(json_object* geometry, const OGRGeometry* poGeom)
1893 {
1894 const int nPrecision = 10;
1895 double dfEps = pow(10.0, -(double)nPrecision);
1896 const char* pszGeomType = "";
1897 switch( wkbFlatten(poGeom->getGeometryType()) )
1898 {
1899 case wkbPoint: pszGeomType = "point"; break;
1900 case wkbLineString: pszGeomType = "linestring"; break;
1901 case wkbPolygon: pszGeomType = "polygon"; break;
1902 case wkbMultiPoint: pszGeomType = "multipoint"; break;
1903 case wkbMultiLineString: pszGeomType = "multilinestring"; break;
1904 case wkbMultiPolygon: pszGeomType = "multipolygon"; break;
1905 case wkbGeometryCollection: pszGeomType = "geometrycollection"; break;
1906 default: break;
1907 }
1908 json_object_object_add(geometry, "type", json_object_new_string(pszGeomType));
1909
1910 switch( wkbFlatten(poGeom->getGeometryType()) )
1911 {
1912 case wkbPoint:
1913 {
1914 const OGRPoint* poPoint = poGeom->toPoint();
1915 json_object *coordinates = json_object_new_array();
1916 json_object_object_add(geometry, "coordinates", coordinates);
1917 json_object_array_add(coordinates, json_object_new_double_with_precision(poPoint->getX(), nPrecision));
1918 json_object_array_add(coordinates, json_object_new_double_with_precision(poPoint->getY(), nPrecision));
1919 break;
1920 }
1921
1922 case wkbLineString:
1923 {
1924 const OGRLineString* poLS = poGeom->toLineString();
1925 json_object *coordinates = json_object_new_array();
1926 json_object_object_add(geometry, "coordinates", coordinates);
1927 for(int i=0;i<poLS->getNumPoints();i++)
1928 {
1929 json_object *point = json_object_new_array();
1930 json_object_array_add(coordinates, point);
1931 json_object_array_add(point, json_object_new_double_with_precision(poLS->getX(i), nPrecision));
1932 json_object_array_add(point, json_object_new_double_with_precision(poLS->getY(i), nPrecision));
1933 }
1934 break;
1935 }
1936
1937 case wkbPolygon:
1938 {
1939 const OGRPolygon* poPoly = poGeom->toPolygon();
1940 json_object *coordinates = json_object_new_array();
1941 json_object_object_add(geometry, "coordinates", coordinates);
1942 for( auto&& poLS: *poPoly )
1943 {
1944 json_object *ring = json_object_new_array();
1945 json_object_array_add(coordinates, ring);
1946 for(int j=0;j<poLS->getNumPoints();j++)
1947 {
1948 if( j > 0 && fabs(poLS->getX(j) - poLS->getX(j-1)) < dfEps &&
1949 fabs(poLS->getY(j) - poLS->getY(j-1)) < dfEps )
1950 continue;
1951 json_object *point = json_object_new_array();
1952 json_object_array_add(ring, point);
1953 json_object_array_add(point, json_object_new_double_with_precision(poLS->getX(j), nPrecision));
1954 json_object_array_add(point, json_object_new_double_with_precision(poLS->getY(j), nPrecision));
1955 }
1956 }
1957 break;
1958 }
1959
1960 case wkbMultiPoint:
1961 {
1962 const OGRMultiPoint* poMP = poGeom->toMultiPoint();
1963 json_object *coordinates = json_object_new_array();
1964 json_object_object_add(geometry, "coordinates", coordinates);
1965 for( auto&& poPoint: *poMP )
1966 {
1967 json_object *point = json_object_new_array();
1968 json_object_array_add(coordinates, point);
1969 json_object_array_add(point, json_object_new_double_with_precision(poPoint->getX(), nPrecision));
1970 json_object_array_add(point, json_object_new_double_with_precision(poPoint->getY(), nPrecision));
1971 }
1972 break;
1973 }
1974
1975 case wkbMultiLineString:
1976 {
1977 const OGRMultiLineString* poMLS = poGeom->toMultiLineString();
1978 json_object *coordinates = json_object_new_array();
1979 json_object_object_add(geometry, "coordinates", coordinates);
1980 for( auto&& poLS: *poMLS )
1981 {
1982 json_object *ls = json_object_new_array();
1983 json_object_array_add(coordinates, ls);
1984 for( auto&& oPoint: *poLS )
1985 {
1986 json_object *point = json_object_new_array();
1987 json_object_array_add(ls, point);
1988 json_object_array_add(point, json_object_new_double_with_precision(oPoint.getX(), nPrecision));
1989 json_object_array_add(point, json_object_new_double_with_precision(oPoint.getY(), nPrecision));
1990 }
1991 }
1992 break;
1993 }
1994
1995 case wkbMultiPolygon:
1996 {
1997 const OGRMultiPolygon* poMP = poGeom->toMultiPolygon();
1998 json_object *coordinates = json_object_new_array();
1999 json_object_object_add(geometry, "coordinates", coordinates);
2000 for( auto&& poPoly: *poMP )
2001 {
2002 json_object *poly = json_object_new_array();
2003 json_object_array_add(coordinates, poly);
2004 for( auto&& poLS: *poPoly )
2005 {
2006 json_object *ring = json_object_new_array();
2007 json_object_array_add(poly, ring);
2008 for(int k=0;k<poLS->getNumPoints();k++)
2009 {
2010 if( k > 0 && fabs(poLS->getX(k)- poLS->getX(k-1)) < dfEps &&
2011 fabs(poLS->getY(k) - poLS->getY(k-1)) < dfEps )
2012 continue;
2013 json_object *point = json_object_new_array();
2014 json_object_array_add(ring, point);
2015 json_object_array_add(point, json_object_new_double_with_precision(poLS->getX(k), nPrecision));
2016 json_object_array_add(point, json_object_new_double_with_precision(poLS->getY(k), nPrecision));
2017 }
2018 }
2019 }
2020 break;
2021 }
2022
2023 case wkbGeometryCollection:
2024 {
2025 const OGRGeometryCollection* poGC = poGeom->toGeometryCollection();
2026 json_object *geometries = json_object_new_array();
2027 json_object_object_add(geometry, "geometries", geometries);
2028 for( auto&& poSubGeom: *poGC )
2029 {
2030 json_object *subgeom = json_object_new_object();
2031 json_object_array_add(geometries, subgeom);
2032 BuildGeoJSONGeometry(subgeom, poSubGeom);
2033 }
2034 break;
2035 }
2036
2037 default:
2038 break;
2039 }
2040 }
2041
2042 /************************************************************************/
2043 /* WriteMapIfNecessary() */
2044 /************************************************************************/
2045
2046 OGRErr OGRElasticLayer::WriteMapIfNecessary()
2047 {
2048 if( m_bManualMapping )
2049 return OGRERR_NONE;
2050
2051 // Check to see if the user has elected to only write out the mapping file
2052 // This method will only write out one layer from the vector file in cases where there are multiple layers
2053 if (!m_osWriteMapFilename.empty() ) {
2054 if( m_bSerializeMapping )
2055 {
2056 m_bSerializeMapping = false;
2057 CPLString map = BuildMap();
2058
2059 // Write the map to a file
2060 VSILFILE *f = VSIFOpenL(m_osWriteMapFilename, "wb");
2061 if (f) {
2062 VSIFWriteL(map.c_str(), 1, map.length(), f);
2063 VSIFCloseL(f);
2064 }
2065 }
2066 return OGRERR_NONE;
2067 }
2068
2069 // Check to see if we have any fields to upload to this index
2070 if( m_osWriteMapFilename.empty() && m_bSerializeMapping )
2071 {
2072 m_bSerializeMapping = false;
2073 CPLString osURL = BuildMappingURL(true);
2074 if( !m_poDS->UploadFile(osURL.c_str(), BuildMap()) )
2075 {
2076 return OGRERR_FAILURE;
2077 }
2078 }
2079
2080 return OGRERR_NONE;
2081 }
2082
2083 /************************************************************************/
2084 /* GetContainerForFeature() */
2085 /************************************************************************/
2086
2087 static json_object* GetContainerForFeature( json_object* poContainer,
2088 const std::vector<CPLString>& aosPath,
2089 std::map< std::vector<CPLString>, json_object* >& oMap )
2090 {
2091 std::vector<CPLString> aosSubPath;
2092 for(int j=0;j<(int)aosPath.size()-1;j++)
2093 {
2094 aosSubPath.push_back(aosPath[j]);
2095 std::map< std::vector<CPLString>, json_object* >::iterator oIter = oMap.find(aosSubPath);
2096 if( oIter == oMap.end() )
2097 {
2098 json_object* poNewContainer = json_object_new_object();
2099 json_object_object_add(poContainer, aosPath[j], poNewContainer);
2100 oMap[aosSubPath] = poNewContainer;
2101 poContainer = poNewContainer;
2102 }
2103 else
2104 {
2105 poContainer = oIter->second;
2106 }
2107 }
2108 return poContainer;
2109 }
2110
2111 /************************************************************************/
2112 /* BuildMappingURL() */
2113 /************************************************************************/
2114 CPLString OGRElasticLayer::BuildMappingURL(bool bMappingApi)
2115 {
2116 CPLString osURL = CPLSPrintf("%s/%s", m_poDS->GetURL(), m_osIndexName.c_str());
2117 if (bMappingApi)
2118 osURL += "/_mapping";
2119 if (m_poDS->m_nMajorVersion < 7)
2120 osURL += CPLSPrintf("/%s", m_osMappingName.c_str());
2121 return osURL;
2122 }
2123
2124 /************************************************************************/
2125 /* BuildJSonFromFeature() */
2126 /************************************************************************/
2127
2128 CPLString OGRElasticLayer::BuildJSonFromFeature(OGRFeature *poFeature)
2129 {
2130
2131 CPLString fields;
2132 int nJSonFieldIndex = m_poFeatureDefn->GetFieldIndex("_json");
2133 if( nJSonFieldIndex >= 0 && poFeature->IsFieldSetAndNotNull(nJSonFieldIndex) )
2134 {
2135 fields = poFeature->GetFieldAsString(nJSonFieldIndex);
2136 }
2137 else
2138 {
2139 json_object *fieldObject = json_object_new_object();
2140
2141 if( poFeature->GetFID() >= 0 && !m_osFID.empty() )
2142 {
2143 json_object_object_add(fieldObject,
2144 m_osFID.c_str(),
2145 json_object_new_int64(poFeature->GetFID()) );
2146 }
2147
2148 std::map< std::vector<CPLString>, json_object* > oMap;
2149
2150 for(int i=0;i<poFeature->GetGeomFieldCount();i++)
2151 {
2152 OGRGeometry* poGeom = poFeature->GetGeomFieldRef(i);
2153 if( poGeom != nullptr && !poGeom->IsEmpty() )
2154 {
2155 OGREnvelope env;
2156 poGeom->getEnvelope(&env);
2157
2158 if( m_apoCT[i] != nullptr )
2159 poGeom->transform( m_apoCT[i] );
2160 else if( env.MinX < -180 || env.MinY < -90 ||
2161 env.MaxX > 180 || env.MaxY > 90 )
2162 {
2163 static bool bHasWarned = false;
2164 if( !bHasWarned )
2165 {
2166 bHasWarned = true;
2167 CPLError(CE_Warning, CPLE_AppDefined,
2168 "At least one geometry has a bounding box outside "
2169 "of [-180,180] longitude range and/or [-90,90] latitude range. Undefined behavior");
2170 }
2171 }
2172
2173 std::vector<CPLString> aosPath = m_aaosGeomFieldPaths[i];
2174 bool bAddGeoJSONType = false;
2175 if( m_abIsGeoPoint[i] &&
2176 aosPath.size() >= 2 &&
2177 aosPath.back() == "coordinates" )
2178 {
2179 bAddGeoJSONType = true;
2180 aosPath.resize( (int)aosPath.size() - 1 );
2181 }
2182
2183 json_object* poContainer = GetContainerForFeature(fieldObject, aosPath, oMap);
2184 const char* pszLastComponent = aosPath.back();
2185
2186 if( m_abIsGeoPoint[i] )
2187 {
2188 json_object *coordinates = json_object_new_array();
2189 const int nPrecision = 10;
2190 json_object_array_add(coordinates, json_object_new_double_with_precision((env.MaxX + env.MinX)*0.5, nPrecision));
2191 json_object_array_add(coordinates, json_object_new_double_with_precision((env.MaxY + env.MinY)*0.5, nPrecision));
2192
2193 if( bAddGeoJSONType )
2194 {
2195 json_object *geometry = json_object_new_object();
2196 json_object_object_add(poContainer, pszLastComponent, geometry);
2197 json_object_object_add(geometry, "type", json_object_new_string("Point"));
2198 json_object_object_add(geometry, "coordinates", coordinates);
2199 }
2200 else
2201 {
2202 json_object_object_add(poContainer, pszLastComponent, coordinates);
2203 }
2204 }
2205 else
2206 {
2207 if( m_bGeoShapeAsGeoJSON )
2208 {
2209 json_object *geometry = json_object_new_object();
2210 json_object_object_add(poContainer, pszLastComponent, geometry);
2211 BuildGeoJSONGeometry(geometry, poGeom);
2212 }
2213 else
2214 {
2215 char* pszWKT = nullptr;
2216 poGeom->exportToWkt(&pszWKT);
2217 json_object_object_add(poContainer, pszLastComponent,
2218 json_object_new_string(pszWKT));
2219 CPLFree(pszWKT);
2220 }
2221 }
2222 }
2223 }
2224
2225 if( m_osMappingName == "FeatureCollection" )
2226 {
2227 if( poFeature->GetGeomFieldCount() == 1 &&
2228 poFeature->GetGeomFieldRef(0) )
2229 {
2230 json_object_object_add(fieldObject, "type", json_object_new_string("Feature"));
2231 }
2232
2233 std::vector<CPLString> aosPath;
2234 aosPath.push_back("properties");
2235 aosPath.push_back("dummy");
2236 GetContainerForFeature(fieldObject, aosPath, oMap);
2237 }
2238
2239 // For every field (except _id)
2240 int fieldCount = m_poFeatureDefn->GetFieldCount();
2241 for (int i = 1; i < fieldCount; i++)
2242 {
2243 if(!poFeature->IsFieldSet( i ) ) {
2244 continue;
2245 }
2246
2247 json_object* poContainer = GetContainerForFeature(fieldObject, m_aaosFieldPaths[i], oMap);
2248 const char* pszLastComponent = m_aaosFieldPaths[i].back();
2249
2250 if( poFeature->IsFieldNull(i) )
2251 {
2252 json_object_object_add(poContainer,
2253 pszLastComponent, nullptr);
2254 continue;
2255 }
2256
2257 switch (m_poFeatureDefn->GetFieldDefn(i)->GetType()) {
2258 case OFTInteger:
2259 if( m_poFeatureDefn->GetFieldDefn(i)->GetSubType() == OFSTBoolean )
2260 json_object_object_add(poContainer,
2261 pszLastComponent,
2262 json_object_new_boolean(poFeature->GetFieldAsInteger(i)));
2263 else
2264 json_object_object_add(poContainer,
2265 pszLastComponent,
2266 json_object_new_int(poFeature->GetFieldAsInteger(i)));
2267 break;
2268 case OFTInteger64:
2269 json_object_object_add(poContainer,
2270 pszLastComponent,
2271 json_object_new_int64(poFeature->GetFieldAsInteger64(i)));
2272 break;
2273 case OFTReal:
2274 json_object_object_add(poContainer,
2275 pszLastComponent,
2276 json_object_new_double_with_significant_figures(poFeature->GetFieldAsDouble(i), -1));
2277 break;
2278 case OFTIntegerList:
2279 {
2280 int nCount = 0;
2281 const int* panValues = poFeature->GetFieldAsIntegerList(i, &nCount);
2282 json_object* poArray = json_object_new_array();
2283 for( int j = 0; j < nCount; j++ )
2284 json_object_array_add(poArray, json_object_new_int(panValues[j]));
2285 json_object_object_add(poContainer,
2286 pszLastComponent, poArray);
2287 break;
2288 }
2289 case OFTInteger64List:
2290 {
2291 int nCount = 0;
2292 const GIntBig* panValues = poFeature->GetFieldAsInteger64List(i, &nCount);
2293 json_object* poArray = json_object_new_array();
2294 for(int j=0;j<nCount;j++)
2295 json_object_array_add(poArray, json_object_new_int64(panValues[j]));
2296 json_object_object_add(poContainer,
2297 pszLastComponent, poArray);
2298 break;
2299 }
2300 case OFTRealList:
2301 {
2302 int nCount = 0;
2303 const double* padfValues = poFeature->GetFieldAsDoubleList(i, &nCount);
2304 json_object* poArray = json_object_new_array();
2305 for(int j=0;j<nCount;j++)
2306 json_object_array_add(poArray, json_object_new_double_with_significant_figures(padfValues[j], -1));
2307 json_object_object_add(poContainer,
2308 pszLastComponent, poArray);
2309 break;
2310 }
2311 case OFTStringList:
2312 {
2313 char** papszValues = poFeature->GetFieldAsStringList(i);
2314 json_object* poArray = json_object_new_array();
2315 for(int j=0;papszValues[j]!= nullptr;j++)
2316 json_object_array_add(poArray, json_object_new_string(papszValues[j]));
2317 json_object_object_add(poContainer,
2318 pszLastComponent, poArray);
2319 break;
2320 }
2321 case OFTBinary:
2322 {
2323 int nCount = 0;
2324 GByte* pabyVal = poFeature->GetFieldAsBinary(i, &nCount);
2325 char* pszVal = CPLBase64Encode(nCount, pabyVal);
2326 json_object_object_add(poContainer,
2327 pszLastComponent,
2328 json_object_new_string(pszVal));
2329 CPLFree(pszVal);
2330 break;
2331 }
2332 case OFTDateTime:
2333 {
2334 int nYear = 0;
2335 int nMonth = 0;
2336 int nDay = 0;
2337 int nHour = 0;
2338 int nMin = 0;
2339 int nTZ = 0;
2340 float fSec = 0.0f;
2341 poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay,
2342 &nHour, &nMin, &fSec, &nTZ);
2343 if( nTZ == 0 )
2344 {
2345 json_object_object_add(poContainer,
2346 pszLastComponent,
2347 json_object_new_string(
2348 CPLSPrintf("%04d/%02d/%02d %02d:%02d:%06.3f",
2349 nYear, nMonth, nDay, nHour, nMin, fSec)));
2350 }
2351 else
2352 {
2353 const int TZOffset = std::abs(nTZ - 100) * 15;
2354 const int TZHour = TZOffset / 60;
2355 const int TZMinute = TZOffset - TZHour * 60;
2356 json_object_object_add(poContainer,
2357 pszLastComponent,
2358 json_object_new_string(
2359 CPLSPrintf("%04d/%02d/%02d %02d:%02d:%06.3f%c%02d:%02d",
2360 nYear, nMonth, nDay, nHour, nMin, fSec,
2361 (nTZ >= 100) ? '+' : '-', TZHour, TZMinute)));
2362 }
2363 break;
2364 }
2365 default:
2366 {
2367 const char* pszVal = poFeature->GetFieldAsString(i);
2368 json_object_object_add(poContainer,
2369 pszLastComponent,
2370 json_object_new_string(pszVal));
2371 }
2372 }
2373 }
2374
2375 // Build the field string
2376 fields = json_object_to_json_string(fieldObject);
2377 json_object_put(fieldObject);
2378 }
2379
2380 return fields;
2381 }
2382
2383 /************************************************************************/
2384 /* ICreateFeature() */
2385 /************************************************************************/
2386
2387 OGRErr OGRElasticLayer::ICreateFeature(OGRFeature *poFeature)
2388 {
2389 if( m_poDS->GetAccess() != GA_Update )
2390 {
2391 CPLError(CE_Failure, CPLE_AppDefined, "Dataset opened in read-only mode");
2392 return OGRERR_FAILURE;
2393 }
2394
2395 FinalizeFeatureDefn();
2396
2397 if( WriteMapIfNecessary() != OGRERR_NONE )
2398 return OGRERR_FAILURE;
2399
2400 if (!m_osWriteMapFilename.empty() )
2401 return OGRERR_NONE;
2402
2403 if( poFeature->GetFID() < 0 )
2404 {
2405 if( m_nNextFID < 0 )
2406 m_nNextFID = GetFeatureCount(FALSE);
2407 poFeature->SetFID(++m_nNextFID);
2408 }
2409
2410 CPLString osFields(BuildJSonFromFeature(poFeature));
2411
2412 const char* pszId = nullptr;
2413 if( poFeature->IsFieldSetAndNotNull(0) && !m_bIgnoreSourceID )
2414 pszId = poFeature->GetFieldAsString(0);
2415
2416 // Check to see if we're using bulk uploading
2417 if (m_nBulkUpload > 0) {
2418 m_osBulkContent += CPLSPrintf("{\"index\" :{\"_index\":\"%s\"", m_osIndexName.c_str());
2419 if(m_poDS->m_nMajorVersion < 7)
2420 m_osBulkContent += CPLSPrintf(", \"_type\":\"%s\"", m_osMappingName.c_str());
2421 if( pszId )
2422 m_osBulkContent += CPLSPrintf(",\"_id\":\"%s\"", pszId);
2423 m_osBulkContent += "}}\n" + osFields + "\n\n";
2424
2425 // Only push the data if we are over our bulk upload limit
2426 if ((int) m_osBulkContent.length() > m_nBulkUpload) {
2427 if( !PushIndex() )
2428 {
2429 return OGRERR_FAILURE;
2430 }
2431 }
2432 }
2433 else
2434 {
2435 // Fall back to using single item upload for every feature.
2436 CPLString osURL(BuildMappingURL(false));
2437 if ( pszId )
2438 osURL += CPLSPrintf("/%s", pszId);
2439 json_object* poRes = m_poDS->RunRequest(osURL, osFields);
2440 if( poRes == nullptr )
2441 {
2442 return OGRERR_FAILURE;
2443 }
2444 if( pszId == nullptr )
2445 {
2446 json_object* poId = CPL_json_object_object_get(poRes, "_id");
2447 if( poId != nullptr && json_object_get_type(poId) == json_type_string )
2448 {
2449 pszId = json_object_get_string(poId);
2450 poFeature->SetField(0, pszId);
2451 }
2452 }
2453 json_object_put(poRes);
2454 }
2455
2456 return OGRERR_NONE;
2457 }
2458
2459 /************************************************************************/
2460 /* ISetFeature() */
2461 /************************************************************************/
2462
2463 OGRErr OGRElasticLayer::ISetFeature(OGRFeature *poFeature)
2464 {
2465 if( m_poDS->GetAccess() != GA_Update )
2466 {
2467 CPLError(CE_Failure, CPLE_AppDefined, "Dataset opened in read-only mode");
2468 return OGRERR_FAILURE;
2469 }
2470
2471 FinalizeFeatureDefn();
2472
2473 if( !poFeature->IsFieldSetAndNotNull(0) )
2474 {
2475 CPLError(CE_Failure, CPLE_AppDefined, "_id field not set");
2476 return OGRERR_FAILURE;
2477 }
2478 if( poFeature->GetFID() < 0 && !m_osFID.empty() )
2479 {
2480 CPLError(CE_Failure, CPLE_AppDefined, "Invalid FID");
2481 return OGRERR_FAILURE;
2482 }
2483
2484 if( WriteMapIfNecessary() != OGRERR_NONE )
2485 return OGRERR_FAILURE;
2486 PushIndex();
2487
2488 CPLString osFields(BuildJSonFromFeature(poFeature));
2489
2490 // TODO? we should theoretically detect if the provided _id doesn't exist
2491 CPLString osURL(CPLSPrintf("%s/%s",
2492 m_poDS->GetURL(), m_osIndexName.c_str()));
2493 if(m_poDS->m_nMajorVersion < 7)
2494 osURL += CPLSPrintf("/%s", m_osMappingName.c_str());
2495 osURL += CPLSPrintf("/%s", poFeature->GetFieldAsString(0));
2496 json_object* poRes = m_poDS->RunRequest(osURL, osFields);
2497 if( poRes == nullptr )
2498 {
2499 return OGRERR_FAILURE;
2500 }
2501 //CPLDebug("ES", "SetFeature(): %s", json_object_to_json_string(poRes));
2502 json_object_put(poRes);
2503
2504 return OGRERR_NONE;
2505 }
2506
2507 /************************************************************************/
2508 /* PushIndex() */
2509 /************************************************************************/
2510
2511 bool OGRElasticLayer::PushIndex()
2512 {
2513 if( m_osBulkContent.empty() )
2514 {
2515 return true;
2516 }
2517
2518 const bool bRet =
2519 m_poDS->UploadFile(CPLSPrintf("%s/_bulk", m_poDS->GetURL()),
2520 m_osBulkContent);
2521 m_osBulkContent.clear();
2522
2523 return bRet;
2524 }
2525
2526 /************************************************************************/
2527 /* CreateField() */
2528 /************************************************************************/
2529
2530 OGRErr OGRElasticLayer::CreateField(OGRFieldDefn *poFieldDefn,
2531 int /*bApproxOK*/)
2532 {
2533 if( m_poDS->GetAccess() != GA_Update )
2534 {
2535 CPLError(CE_Failure, CPLE_AppDefined, "Dataset opened in read-only mode");
2536 return OGRERR_FAILURE;
2537 }
2538
2539 FinalizeFeatureDefn();
2540 ResetReading();
2541
2542 if( m_poFeatureDefn->GetFieldIndex(poFieldDefn->GetNameRef()) >= 0 )
2543 {
2544 if( !EQUAL(poFieldDefn->GetNameRef(), "_id") &&
2545 !EQUAL(poFieldDefn->GetNameRef(), "_json") )
2546 {
2547 CPLError(CE_Failure, CPLE_AppDefined,
2548 "CreateField() called with an already existing field name: %s",
2549 poFieldDefn->GetNameRef());
2550 }
2551 return OGRERR_FAILURE;
2552 }
2553
2554 std::vector<CPLString> aosPath;
2555 if( m_osMappingName == "FeatureCollection" )
2556 aosPath.push_back("properties");
2557
2558 if( m_bDotAsNestedField )
2559 {
2560 char** papszTokens = CSLTokenizeString2(poFieldDefn->GetNameRef(), ".", 0);
2561 for(int i=0; papszTokens[i]; i++ )
2562 aosPath.push_back(papszTokens[i]);
2563 CSLDestroy(papszTokens);
2564 }
2565 else
2566 aosPath.push_back(poFieldDefn->GetNameRef());
2567
2568 AddFieldDefn( poFieldDefn->GetNameRef(),
2569 poFieldDefn->GetType(),
2570 aosPath,
2571 poFieldDefn->GetSubType() );
2572
2573 m_bSerializeMapping = true;
2574
2575 return OGRERR_NONE;
2576 }
2577
2578 /************************************************************************/
2579 /* CreateGeomField() */
2580 /************************************************************************/
2581
2582 OGRErr OGRElasticLayer::CreateGeomField( OGRGeomFieldDefn *poFieldIn,
2583 int /*bApproxOK*/ )
2584
2585 {
2586 if( m_poDS->GetAccess() != GA_Update )
2587 {
2588 CPLError(CE_Failure, CPLE_AppDefined, "Dataset opened in read-only mode");
2589 return OGRERR_FAILURE;
2590 }
2591
2592 FinalizeFeatureDefn();
2593 ResetReading();
2594
2595 if( m_poFeatureDefn->GetGeomFieldIndex(poFieldIn->GetNameRef()) >= 0 )
2596 {
2597 CPLError(CE_Failure, CPLE_AppDefined,
2598 "CreateGeomField() called with an already existing field name: %s",
2599 poFieldIn->GetNameRef());
2600 return OGRERR_FAILURE;
2601 }
2602
2603 OGRGeomFieldDefn oFieldDefn(poFieldIn);
2604 if( oFieldDefn.GetSpatialRef() )
2605 {
2606 oFieldDefn.GetSpatialRef()->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
2607 }
2608 if( EQUAL(oFieldDefn.GetNameRef(), "") )
2609 oFieldDefn.SetName("geometry");
2610
2611 std::vector<CPLString> aosPath;
2612 if( m_bDotAsNestedField )
2613 {
2614 char** papszTokens = CSLTokenizeString2(oFieldDefn.GetNameRef(), ".", 0);
2615 for(int i=0; papszTokens[i]; i++ )
2616 aosPath.push_back(papszTokens[i]);
2617 CSLDestroy(papszTokens);
2618 }
2619 else
2620 aosPath.push_back(oFieldDefn.GetNameRef());
2621
2622 if( m_eGeomTypeMapping == ES_GEOMTYPE_GEO_SHAPE ||
2623 (m_eGeomTypeMapping == ES_GEOMTYPE_AUTO &&
2624 poFieldIn->GetType() != wkbPoint))
2625 {
2626 m_abIsGeoPoint.push_back(FALSE);
2627 }
2628 else
2629 {
2630 m_abIsGeoPoint.push_back(TRUE);
2631 aosPath.push_back("coordinates");
2632 }
2633
2634 m_aaosGeomFieldPaths.push_back(aosPath);
2635
2636 m_aosMapToGeomFieldIndex[ BuildPathFromArray(aosPath) ] = m_poFeatureDefn->GetGeomFieldCount();
2637
2638 m_poFeatureDefn->AddGeomFieldDefn( &oFieldDefn );
2639
2640 OGRCoordinateTransformation* poCT = nullptr;
2641 if( oFieldDefn.GetSpatialRef() != nullptr )
2642 {
2643 OGRSpatialReference oSRS_WGS84;
2644 oSRS_WGS84.SetFromUserInput(SRS_WKT_WGS84_LAT_LONG);
2645 oSRS_WGS84.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
2646 if( !oSRS_WGS84.IsSame(oFieldDefn.GetSpatialRef()) )
2647 {
2648 poCT = OGRCreateCoordinateTransformation( oFieldDefn.GetSpatialRef(), &oSRS_WGS84 );
2649 if( poCT == nullptr )
2650 {
2651 CPLError( CE_Warning, CPLE_AppDefined,
2652 "On-the-fly reprojection to WGS84 long/lat would be "
2653 "needed, but instantiation of transformer failed");
2654 }
2655 }
2656 }
2657 else
2658 {
2659 CPLError( CE_Warning, CPLE_AppDefined,
2660 "No SRS given for geometry column %s. SRS is assumed to "
2661 "be EPSG:4326 (WGS84 long/lat)",
2662 oFieldDefn.GetNameRef());
2663 }
2664
2665 m_apoCT.push_back(poCT);
2666
2667 m_bSerializeMapping = true;
2668
2669 return OGRERR_NONE;
2670 }
2671
2672 /************************************************************************/
2673 /* TestCapability() */
2674 /************************************************************************/
2675
2676 int OGRElasticLayer::TestCapability(const char * pszCap) {
2677 if (EQUAL(pszCap, OLCFastFeatureCount))
2678 return m_poAttrQuery == nullptr && m_poFilterGeom == nullptr;
2679
2680 else if (EQUAL(pszCap, OLCStringsAsUTF8))
2681 return TRUE;
2682
2683 else if (EQUAL(pszCap, OLCSequentialWrite) ||
2684 EQUAL(pszCap, OLCRandomWrite) )
2685 return m_poDS->GetAccess() == GA_Update;
2686 else if (EQUAL(pszCap, OLCCreateField) ||
2687 EQUAL(pszCap, OLCCreateGeomField) )
2688 return m_poDS->GetAccess() == GA_Update;
2689 else
2690 return FALSE;
2691 }
2692
2693 /************************************************************************/
2694 /* AddTimeoutTerminateAfterToURL() */
2695 /************************************************************************/
2696
2697 void OGRElasticLayer::AddTimeoutTerminateAfterToURL( CPLString& osURL )
2698 {
2699 if( !m_osSingleQueryTimeout.empty() )
2700 osURL += "&timeout=" + m_osSingleQueryTimeout;
2701 if( !m_osSingleQueryTerminateAfter.empty() )
2702 osURL += "&terminate_after=" + m_osSingleQueryTerminateAfter;
2703 }
2704
2705 /************************************************************************/
2706 /* GetFeatureCount() */
2707 /************************************************************************/
2708
2709 GIntBig OGRElasticLayer::GetFeatureCount( int bForce )
2710 {
2711 if( m_bFilterMustBeClientSideEvaluated )
2712 {
2713 m_bUseSingleQueryParams = true;
2714 const auto nRet = OGRLayer::GetFeatureCount(bForce);
2715 m_bUseSingleQueryParams = false;
2716 return nRet;
2717 }
2718
2719 json_object* poResponse = nullptr;
2720 CPLString osURL(CPLSPrintf("%s", m_poDS->GetURL()));
2721 CPLString osFilter = "";
2722 if( !m_osESSearch.empty() )
2723 {
2724 if( m_osESSearch[0] != '{' )
2725 return OGRLayer::GetFeatureCount(bForce);
2726 osURL += "/_search?pretty";
2727 osFilter = "{ \"size\": 0 ";
2728 if( m_osESSearch == "{}" )
2729 osFilter += '}';
2730 else
2731 osFilter += ", " + m_osESSearch.substr(1);
2732 }
2733 else if( (m_poSpatialFilter && m_osJSONFilter.empty()) || m_poJSONFilter )
2734 {
2735 osFilter = BuildQuery(true);
2736 osURL += CPLSPrintf("/%s", m_osIndexName.c_str());
2737 if (m_poDS->m_nMajorVersion < 7)
2738 osURL += CPLSPrintf("/%s", m_osMappingName.c_str());
2739 if( m_poDS->m_nMajorVersion >= 5 && m_osSingleQueryTimeout.empty() )
2740 {
2741 osURL += "/_count?pretty";
2742 }
2743 else
2744 {
2745 osURL += "/_search?pretty";
2746 }
2747 }
2748 else if( !m_osJSONFilter.empty() )
2749 {
2750 osURL += CPLSPrintf("/%s", m_osIndexName.c_str());
2751 if (m_poDS->m_nMajorVersion < 7)
2752 osURL += CPLSPrintf("/%s", m_osMappingName.c_str());
2753 osURL += "/_search?pretty";
2754 osFilter = ("{ \"size\": 0, " + m_osJSONFilter.substr(1));
2755 }
2756 else
2757 {
2758 osURL += CPLSPrintf("/%s", m_osIndexName.c_str());
2759 if (m_poDS->m_nMajorVersion < 7)
2760 osURL += CPLSPrintf("/%s", m_osMappingName.c_str());
2761 if( m_osSingleQueryTimeout.empty() )
2762 {
2763 osURL += "/_count?pretty";
2764 }
2765 else
2766 {
2767 osFilter = "{ \"size\": 0 }";
2768 osURL += CPLSPrintf("/_search?pretty");
2769 }
2770 }
2771 AddTimeoutTerminateAfterToURL(osURL);
2772
2773 poResponse = m_poDS->RunRequest(osURL.c_str(), osFilter.c_str());
2774
2775 json_object* poCount = json_ex_get_object_by_path(poResponse, "hits.count");
2776 if( poCount == nullptr )
2777 {
2778 // For _search request
2779 poCount = json_ex_get_object_by_path(poResponse, "hits.total");
2780 if( poCount && json_object_get_type(poCount) == json_type_object )
2781 {
2782 // Since ES 7.0
2783 poCount = json_ex_get_object_by_path(poCount, "value");
2784 }
2785 }
2786 if( poCount == nullptr )
2787 {
2788 // For _count request
2789 poCount = json_ex_get_object_by_path(poResponse, "count");
2790 }
2791 if( poCount == nullptr || json_object_get_type(poCount) != json_type_int )
2792 {
2793 json_object_put(poResponse);
2794 CPLDebug("ES",
2795 "Cannot find hits in GetFeatureCount() response. "
2796 "Falling back to slow implementation");
2797 m_bUseSingleQueryParams = true;
2798 const auto nRet = OGRLayer::GetFeatureCount(bForce);
2799 m_bUseSingleQueryParams = false;
2800 return nRet;
2801 }
2802
2803 GIntBig nCount = json_object_get_int64(poCount);
2804 json_object_put(poResponse);
2805 return nCount;
2806 }
2807
2808 /************************************************************************/
2809 /* GetValue() */
2810 /************************************************************************/
2811
2812 json_object* OGRElasticLayer::GetValue( int nFieldIdx,
2813 swq_expr_node* poValNode )
2814 {
2815 json_object* poVal = nullptr;
2816 if (poValNode->field_type == SWQ_FLOAT)
2817 poVal = json_object_new_double(poValNode->float_value);
2818 else if (poValNode->field_type == SWQ_INTEGER ||
2819 poValNode->field_type == SWQ_INTEGER64)
2820 poVal = json_object_new_int64(poValNode->int_value);
2821 else if (poValNode->field_type == SWQ_STRING)
2822 poVal = json_object_new_string(poValNode->string_value);
2823 else if (poValNode->field_type == SWQ_TIMESTAMP)
2824 {
2825 int nYear = 0;
2826 int nMonth = 0;
2827 int nDay = 0;
2828 int nHour = 0;
2829 int nMinute = 0;
2830 float fSecond = 0;
2831 if( sscanf(poValNode->string_value,"%04d/%02d/%02d %02d:%02d:%f",
2832 &nYear, &nMonth, &nDay, &nHour, &nMinute, &fSecond) >= 3 ||
2833 sscanf(poValNode->string_value,"%04d-%02d-%02dT%02d:%02d:%f",
2834 &nYear, &nMonth, &nDay, &nHour, &nMinute, &fSecond) >= 3 )
2835 {
2836 OGRFieldType eType(m_poFeatureDefn->GetFieldDefn(
2837 nFieldIdx)->GetType());
2838 if( eType == OFTDateTime )
2839 poVal = json_object_new_string(
2840 CPLSPrintf("%04d/%02d/%02d %02d:%02d:%02.03f",
2841 nYear, nMonth, nDay, nHour, nMinute, fSecond));
2842 else if( eType == OFTDate)
2843 poVal = json_object_new_string(
2844 CPLSPrintf("%04d/%02d/%02d",
2845 nYear, nMonth, nDay));
2846 else
2847 poVal = json_object_new_string(
2848 CPLSPrintf("%02d:%02d:%02.03f",
2849 nHour, nMinute, fSecond));
2850 }
2851 else
2852 {
2853 return nullptr;
2854 }
2855 }
2856 else
2857 {
2858 CPLError(CE_Failure, CPLE_NotSupported, "Unhandled type: %d",
2859 poValNode->field_type);
2860 }
2861 return poVal;
2862 }
2863
2864 /************************************************************************/
2865 /* OGRESGetFieldIndexFromSQL() */
2866 /************************************************************************/
2867
2868 static int OGRESGetFieldIndexFromSQL(const swq_expr_node* poNode)
2869 {
2870 if( poNode->eNodeType == SNT_COLUMN )
2871 return poNode->field_index;
2872
2873 if( poNode->eNodeType == SNT_OPERATION &&
2874 poNode->nOperation == SWQ_CAST &&
2875 poNode->nSubExprCount >= 1 &&
2876 poNode->papoSubExpr[0]->eNodeType == SNT_COLUMN )
2877 return poNode->papoSubExpr[0]->field_index;
2878
2879 return -1;
2880 }
2881
2882 /************************************************************************/
2883 /* TranslateSQLToFilter() */
2884 /************************************************************************/
2885
2886 json_object* OGRElasticLayer::TranslateSQLToFilter(swq_expr_node* poNode)
2887 {
2888 if( poNode->eNodeType == SNT_OPERATION )
2889 {
2890 int nFieldIdx;
2891 if( poNode->nOperation == SWQ_AND && poNode->nSubExprCount == 2 )
2892 {
2893 // For AND, we can deal with a failure in one of the branch
2894 // since client-side will do that extra filtering
2895 json_object* poFilter1 = TranslateSQLToFilter(poNode->papoSubExpr[0]);
2896 json_object* poFilter2 = TranslateSQLToFilter(poNode->papoSubExpr[1]);
2897 if( poFilter1 && poFilter2 )
2898 {
2899 json_object* poRet = json_object_new_object();
2900 json_object* poBool = json_object_new_object();
2901 json_object_object_add(poRet, "bool", poBool);
2902 json_object* poMust = json_object_new_array();
2903 json_object_object_add(poBool, "must", poMust);
2904 json_object_array_add(poMust, poFilter1);
2905 json_object_array_add(poMust, poFilter2);
2906 return poRet;
2907 }
2908 else if( poFilter1 )
2909 return poFilter1;
2910 else
2911 return poFilter2;
2912 }
2913 else if( poNode->nOperation == SWQ_OR && poNode->nSubExprCount == 2 )
2914 {
2915 json_object* poFilter1 = TranslateSQLToFilter(poNode->papoSubExpr[0]);
2916 json_object* poFilter2 = TranslateSQLToFilter(poNode->papoSubExpr[1]);
2917 if( poFilter1 && poFilter2 )
2918 {
2919 json_object* poRet = json_object_new_object();
2920 json_object* poBool = json_object_new_object();
2921 json_object_object_add(poRet, "bool", poBool);
2922 json_object* poShould = json_object_new_array();
2923 json_object_object_add(poBool, "should", poShould);
2924 json_object_array_add(poShould, poFilter1);
2925 json_object_array_add(poShould, poFilter2);
2926 return poRet;
2927 }
2928 else
2929 {
2930 json_object_put(poFilter1);
2931 json_object_put(poFilter2);
2932 return nullptr;
2933 }
2934 }
2935 else if( poNode->nOperation == SWQ_NOT && poNode->nSubExprCount == 1 )
2936 {
2937 if( poNode->papoSubExpr[0]->eNodeType == SNT_OPERATION &&
2938 poNode->papoSubExpr[0]->nOperation == SWQ_ISNULL &&
2939 poNode->papoSubExpr[0]->nSubExprCount == 1 &&
2940 poNode->papoSubExpr[0]->papoSubExpr[0]->field_index != 0 &&
2941 poNode->papoSubExpr[0]->papoSubExpr[0]->field_index <
2942 m_poFeatureDefn->GetFieldCount() )
2943 {
2944 json_object* poRet = json_object_new_object();
2945 json_object* poExists = json_object_new_object();
2946 CPLString osFieldName(BuildPathFromArray(
2947 m_aaosFieldPaths[
2948 poNode->papoSubExpr[0]->papoSubExpr[0]->field_index]));
2949 json_object_object_add(poExists, "field",
2950 json_object_new_string(osFieldName));
2951 json_object_object_add(poRet, "exists", poExists);
2952 return poRet;
2953 }
2954 else
2955 {
2956 json_object* poFilter = TranslateSQLToFilter(
2957 poNode->papoSubExpr[0]);
2958 if( poFilter )
2959 {
2960 json_object* poRet = json_object_new_object();
2961 json_object* poBool = json_object_new_object();
2962 json_object_object_add(poRet, "bool", poBool);
2963 json_object_object_add(poBool, "must_not", poFilter);
2964 return poRet;
2965 }
2966 else
2967 {
2968 return nullptr;
2969 }
2970 }
2971 }
2972 else if( poNode->nOperation == SWQ_ISNULL &&
2973 poNode->nSubExprCount == 1 &&
2974 (nFieldIdx = OGRESGetFieldIndexFromSQL(poNode->papoSubExpr[0])) > 0 &&
2975 nFieldIdx < m_poFeatureDefn->GetFieldCount() )
2976 {
2977 json_object* poRet = json_object_new_object();
2978 json_object* poExists = json_object_new_object();
2979 CPLString osFieldName(BuildPathFromArray(
2980 m_aaosFieldPaths[nFieldIdx]));
2981 json_object_object_add(poExists, "field",
2982 json_object_new_string(osFieldName));
2983 json_object* poBool = json_object_new_object();
2984 json_object_object_add(poRet, "bool", poBool);
2985 json_object* poMustNot = json_object_new_object();
2986 json_object_object_add(poMustNot, "exists", poExists);
2987 json_object_object_add(poBool, "must_not", poMustNot);
2988 return poRet;
2989 }
2990 else if( poNode->nOperation == SWQ_NE )
2991 {
2992 poNode->nOperation = SWQ_EQ;
2993 json_object* poFilter = TranslateSQLToFilter(poNode);
2994 poNode->nOperation = SWQ_NE;
2995 if( poFilter )
2996 {
2997 json_object* poRet = json_object_new_object();
2998 json_object* poBool = json_object_new_object();
2999 json_object_object_add(poRet, "bool", poBool);
3000 json_object_object_add(poBool, "must_not", poFilter);
3001 return poRet;
3002 }
3003 else
3004 {
3005 return nullptr;
3006 }
3007 }
3008 else if( poNode->nOperation == SWQ_EQ && poNode->nSubExprCount == 2 &&
3009 poNode->papoSubExpr[1]->eNodeType == SNT_CONSTANT &&
3010 (nFieldIdx = OGRESGetFieldIndexFromSQL(poNode->papoSubExpr[0])) >= 0 &&
3011 nFieldIdx < m_poFeatureDefn->GetFieldCount() )
3012 {
3013 json_object* poVal = GetValue(nFieldIdx,
3014 poNode->papoSubExpr[1]);
3015 if( poVal == nullptr )
3016 {
3017 return nullptr;
3018 }
3019 json_object* poRet = json_object_new_object();
3020 if( nFieldIdx == 0 )
3021 {
3022 json_object* poIds = json_object_new_object();
3023 json_object* poValues = json_object_new_array();
3024 json_object_object_add(poIds, "values", poValues);
3025 json_object_array_add(poValues, poVal);
3026 json_object_object_add(poRet, "ids", poIds);
3027 }
3028 else
3029 {
3030 json_object* poTerm = json_object_new_object();
3031 CPLString osPath(BuildPathFromArray(
3032 m_aaosFieldPaths[ nFieldIdx]));
3033 bool bNotAnalyzed = true;
3034 if( poNode->papoSubExpr[1]->field_type == SWQ_STRING )
3035 {
3036 const char* pszFieldName =
3037 m_poFeatureDefn->GetFieldDefn(
3038 nFieldIdx)->GetNameRef();
3039 bNotAnalyzed = CSLFindString(m_papszNotAnalyzedFields,
3040 pszFieldName) >= 0;
3041 if( !bNotAnalyzed )
3042 {
3043 if( CSLFindString(m_papszFieldsWithRawValue,
3044 pszFieldName) >= 0 )
3045 {
3046 osPath += ".raw";
3047 bNotAnalyzed = true;
3048 }
3049 else if( !m_bFilterMustBeClientSideEvaluated )
3050 {
3051 m_bFilterMustBeClientSideEvaluated = true;
3052 CPLDebug("ES",
3053 "Part or full filter will have to be evaluated on "
3054 "client side (equality test on a analyzed "
3055 "field).");
3056 }
3057 }
3058 }
3059 json_object_object_add(poRet,
3060 bNotAnalyzed ? "term" : "match", poTerm);
3061 json_object_object_add(poTerm, osPath, poVal);
3062
3063 if( !bNotAnalyzed && m_poDS->m_nMajorVersion < 2 )
3064 {
3065 json_object* poNewRet = json_object_new_object();
3066 json_object_object_add(poNewRet, "query", poRet);
3067 poRet = poNewRet;
3068 }
3069 }
3070 return poRet;
3071 }
3072 else if( (poNode->nOperation == SWQ_LT ||
3073 poNode->nOperation == SWQ_LE ||
3074 poNode->nOperation == SWQ_GT ||
3075 poNode->nOperation == SWQ_GE) &&
3076 poNode->nSubExprCount == 2 &&
3077 poNode->papoSubExpr[1]->eNodeType == SNT_CONSTANT &&
3078 (nFieldIdx = OGRESGetFieldIndexFromSQL(poNode->papoSubExpr[0])) > 0 &&
3079 nFieldIdx < m_poFeatureDefn->GetFieldCount() )
3080 {
3081 json_object* poVal = GetValue(nFieldIdx,
3082 poNode->papoSubExpr[1]);
3083 if( poVal == nullptr )
3084 {
3085 return nullptr;
3086 }
3087 json_object* poRet = json_object_new_object();
3088 json_object* poRange = json_object_new_object();
3089 json_object_object_add(poRet, "range", poRange);
3090 json_object* poFieldConstraint = json_object_new_object();
3091 CPLString osFieldName(BuildPathFromArray(
3092 m_aaosFieldPaths[nFieldIdx]));
3093 json_object_object_add(poRange, osFieldName, poFieldConstraint);
3094 const char* pszOp = (poNode->nOperation == SWQ_LT) ? "lt" :
3095 (poNode->nOperation == SWQ_LE) ? "lte" :
3096 (poNode->nOperation == SWQ_GT) ? "gt" :
3097 /*(poNode->nOperation == SWQ_GE) ?*/ "gte";
3098 json_object_object_add(poFieldConstraint, pszOp, poVal);
3099 return poRet;
3100 }
3101 else if( poNode->nOperation == SWQ_BETWEEN &&
3102 poNode->nSubExprCount == 3 &&
3103 poNode->papoSubExpr[1]->eNodeType == SNT_CONSTANT &&
3104 poNode->papoSubExpr[2]->eNodeType == SNT_CONSTANT &&
3105 (nFieldIdx = OGRESGetFieldIndexFromSQL(poNode->papoSubExpr[0])) > 0 &&
3106 nFieldIdx < m_poFeatureDefn->GetFieldCount() )
3107 {
3108 json_object* poVal1 = GetValue(nFieldIdx,
3109 poNode->papoSubExpr[1]);
3110 if( poVal1 == nullptr )
3111 {
3112 return nullptr;
3113 }
3114 json_object* poVal2 = GetValue(nFieldIdx,
3115 poNode->papoSubExpr[2]);
3116 if( poVal2 == nullptr )
3117 {
3118 json_object_put(poVal1);
3119 return nullptr;
3120 }
3121
3122 json_object* poRet = json_object_new_object();
3123 json_object* poRange = json_object_new_object();
3124 json_object_object_add(poRet, "range", poRange);
3125 json_object* poFieldConstraint = json_object_new_object();
3126 CPLString osFieldName(BuildPathFromArray(
3127 m_aaosFieldPaths[nFieldIdx]));
3128 json_object_object_add(poRange, osFieldName, poFieldConstraint);
3129 json_object_object_add(poFieldConstraint, "gte", poVal1);
3130 json_object_object_add(poFieldConstraint, "lte", poVal2);
3131 return poRet;
3132 }
3133 else if( poNode->nOperation == SWQ_IN &&
3134 poNode->nSubExprCount > 1 &&
3135 (nFieldIdx = OGRESGetFieldIndexFromSQL(poNode->papoSubExpr[0])) >= 0 &&
3136 nFieldIdx < m_poFeatureDefn->GetFieldCount() )
3137 {
3138 bool bAllConstant = true;
3139 for( int i=1; i< poNode->nSubExprCount; i++ )
3140 {
3141 if( poNode->papoSubExpr[i]->eNodeType != SNT_CONSTANT )
3142 {
3143 bAllConstant = false;
3144 break;
3145 }
3146 }
3147 if( bAllConstant )
3148 {
3149 json_object* poRet = json_object_new_object();
3150 if( nFieldIdx == 0 )
3151 {
3152 json_object* poIds = json_object_new_object();
3153 json_object* poValues = json_object_new_array();
3154 json_object_object_add(poIds, "values", poValues);
3155 json_object_object_add(poRet, "ids", poIds);
3156 for( int i=1; i< poNode->nSubExprCount; i++ )
3157 {
3158 json_object* poVal = GetValue(
3159 nFieldIdx,
3160 poNode->papoSubExpr[i]);
3161 if( poVal == nullptr )
3162 {
3163 json_object_put(poRet);
3164 return nullptr;
3165 }
3166 json_object_array_add(poValues, poVal);
3167 }
3168 }
3169 else
3170 {
3171 bool bNotAnalyzed = true;
3172 CPLString osPath(BuildPathFromArray(
3173 m_aaosFieldPaths[nFieldIdx]));
3174 if( poNode->papoSubExpr[1]->field_type == SWQ_STRING )
3175 {
3176 const char* pszFieldName =
3177 m_poFeatureDefn->GetFieldDefn(nFieldIdx)->
3178 GetNameRef();
3179 bNotAnalyzed = CSLFindString(m_papszNotAnalyzedFields,
3180 pszFieldName) >= 0;
3181 if( !bNotAnalyzed &&
3182 CSLFindString(m_papszFieldsWithRawValue,
3183 pszFieldName) >= 0 )
3184 {
3185 osPath += ".raw";
3186 bNotAnalyzed = true;
3187 }
3188
3189 if( !bNotAnalyzed &&
3190 !m_bFilterMustBeClientSideEvaluated )
3191 {
3192 m_bFilterMustBeClientSideEvaluated = true;
3193 CPLDebug("ES",
3194 "Part or full filter will have to be "
3195 "evaluated on client side (IN test on a "
3196 "analyzed field).");
3197 }
3198 }
3199
3200 if( bNotAnalyzed )
3201 {
3202 json_object* poTerms = json_object_new_object();
3203 json_object_object_add(poRet, "terms", poTerms);
3204 json_object* poTermsValues = json_object_new_array();
3205 json_object_object_add(poTerms, osPath,
3206 poTermsValues);
3207 for( int i=1; i< poNode->nSubExprCount; i++ )
3208 {
3209 json_object* poVal = GetValue(
3210 nFieldIdx,
3211 poNode->papoSubExpr[i]);
3212 if( poVal == nullptr )
3213 {
3214 json_object_put(poRet);
3215 return nullptr;
3216 }
3217 json_object_array_add(poTermsValues, poVal);
3218 }
3219 }
3220 else
3221 {
3222 json_object* poBool = json_object_new_object();
3223 json_object_object_add(poRet, "bool", poBool);
3224 json_object* poShould = json_object_new_array();
3225 json_object_object_add(poBool, "should", poShould);
3226 for( int i=1; i< poNode->nSubExprCount; i++ )
3227 {
3228 json_object* poVal = GetValue(
3229 nFieldIdx,
3230 poNode->papoSubExpr[i]);
3231 if( poVal == nullptr )
3232 {
3233 json_object_put(poRet);
3234 return nullptr;
3235 }
3236 json_object* poShouldElt = json_object_new_object();
3237 json_object* poMatch = json_object_new_object();
3238 json_object_object_add(poShouldElt, "match",
3239 poMatch);
3240 json_object_object_add(poMatch, osPath, poVal);
3241
3242 if( m_poDS->m_nMajorVersion < 2 )
3243 {
3244 json_object* poNewShouldElt =
3245 json_object_new_object();
3246 json_object_object_add(poNewShouldElt,
3247 "query", poShouldElt);
3248 poShouldElt = poNewShouldElt;
3249 }
3250 json_object_array_add(poShould, poShouldElt);
3251 }
3252 }
3253 }
3254 return poRet;
3255 }
3256 }
3257 else if( (poNode->nOperation == SWQ_LIKE ||
3258 poNode->nOperation == SWQ_ILIKE ) && // ES actual semantics doesn't match exactly either...
3259 poNode->nSubExprCount >= 2 &&
3260 (nFieldIdx = OGRESGetFieldIndexFromSQL(poNode->papoSubExpr[0])) > 0 &&
3261 nFieldIdx < m_poFeatureDefn->GetFieldCount() )
3262 {
3263 char chEscape = '\0';
3264 if( poNode->nSubExprCount == 3 )
3265 chEscape = poNode->papoSubExpr[2]->string_value[0];
3266 const char* pszPattern = poNode->papoSubExpr[1]->string_value;
3267 const char* pszFieldName = m_poFeatureDefn->GetFieldDefn(
3268 nFieldIdx)->GetNameRef();
3269 bool bNotAnalyzed = CSLFindString(m_papszNotAnalyzedFields,
3270 pszFieldName) >= 0;
3271 CPLString osPath(BuildPathFromArray(
3272 m_aaosFieldPaths[nFieldIdx]));
3273 if( !bNotAnalyzed && CSLFindString(m_papszFieldsWithRawValue,
3274 pszFieldName) >= 0 )
3275 {
3276 osPath += ".raw";
3277 bNotAnalyzed = true;
3278 }
3279
3280 if( strchr(pszPattern, '*') || strchr(pszPattern, '?') )
3281 {
3282 CPLDebug("ES", "Cannot handle * or ? in LIKE pattern");
3283 }
3284 else if( !bNotAnalyzed )
3285 {
3286 if( !m_bFilterMustBeClientSideEvaluated )
3287 {
3288 m_bFilterMustBeClientSideEvaluated = true;
3289 CPLDebug("ES",
3290 "Part or full filter will have to be evaluated on "
3291 "client side (wildcard test on a analyzed field).");
3292 }
3293 }
3294 else
3295 {
3296 CPLString osUnescaped;
3297 for( int i=0; pszPattern[i] != '\0'; ++i )
3298 {
3299 if( chEscape == pszPattern[i] )
3300 {
3301 if( pszPattern[i+1] == '\0' )
3302 break;
3303 osUnescaped += pszPattern[i+1];
3304 i ++;
3305 }
3306 else if( pszPattern[i] == '%' )
3307 {
3308 osUnescaped += '*';
3309 }
3310 else if( pszPattern[i] == '_' )
3311 {
3312 osUnescaped += '?';
3313 }
3314 else
3315 {
3316 osUnescaped += pszPattern[i];
3317 }
3318 }
3319 json_object* poRet = json_object_new_object();
3320 json_object* poWildcard = json_object_new_object();
3321 json_object_object_add(poRet, "wildcard", poWildcard);
3322 json_object_object_add(poWildcard, osPath,
3323 json_object_new_string(osUnescaped));
3324 return poRet;
3325 }
3326 }
3327 }
3328
3329 if( !m_bFilterMustBeClientSideEvaluated )
3330 {
3331 m_bFilterMustBeClientSideEvaluated = true;
3332 CPLDebug("ES",
3333 "Part or full filter will have to be evaluated on "
3334 "client side.");
3335 }
3336 return nullptr;
3337 }
3338
3339 /************************************************************************/
3340 /* SetAttributeFilter() */
3341 /************************************************************************/
3342
3343 OGRErr OGRElasticLayer::SetAttributeFilter(const char* pszFilter)
3344 {
3345 m_bFilterMustBeClientSideEvaluated = false;
3346 if( pszFilter != nullptr && pszFilter[0] == '{' )
3347 {
3348 if( !m_osESSearch.empty() )
3349 {
3350 CPLError(CE_Failure, CPLE_AppDefined,
3351 "Setting an Elasticsearch filter on a resulting layer "
3352 "is not supported");
3353 return OGRERR_FAILURE;
3354 }
3355 OGRLayer::SetAttributeFilter(nullptr);
3356 m_osJSONFilter = pszFilter;
3357 return OGRERR_NONE;
3358 }
3359 else
3360 {
3361 m_osJSONFilter.clear();
3362 json_object_put(m_poJSONFilter);
3363 m_poJSONFilter = nullptr;
3364 OGRErr eErr = OGRLayer::SetAttributeFilter(pszFilter);
3365 if( eErr == OGRERR_NONE && m_poAttrQuery != nullptr )
3366 {
3367 swq_expr_node* poNode = reinterpret_cast<swq_expr_node*>(
3368 m_poAttrQuery->GetSWQExpr());
3369 m_poJSONFilter = TranslateSQLToFilter(poNode);
3370 }
3371 return eErr;
3372 }
3373 }
3374
3375 /************************************************************************/
3376 /* SetSpatialFilter() */
3377 /************************************************************************/
3378
3379 void OGRElasticLayer::SetSpatialFilter( int iGeomField, OGRGeometry * poGeomIn )
3380
3381 {
3382 FinalizeFeatureDefn();
3383
3384 if( iGeomField < 0 || iGeomField >= GetLayerDefn()->GetGeomFieldCount() ||
3385 GetLayerDefn()->GetGeomFieldDefn(iGeomField)->GetType() == wkbNone )
3386 {
3387 if( iGeomField != 0 )
3388 {
3389 CPLError(CE_Failure, CPLE_AppDefined,
3390 "Invalid geometry field index : %d", iGeomField);
3391 }
3392 return;
3393 }
3394 m_iGeomFieldFilter = iGeomField;
3395
3396 InstallFilter( poGeomIn );
3397
3398 json_object_put(m_poSpatialFilter);
3399 m_poSpatialFilter = nullptr;
3400
3401 if( poGeomIn == nullptr )
3402 return;
3403
3404 if( !m_osESSearch.empty() )
3405 {
3406 CPLError(CE_Failure, CPLE_AppDefined,
3407 "Setting a spatial filter on a resulting layer is not supported");
3408 return;
3409 }
3410
3411 OGREnvelope sEnvelope;
3412 poGeomIn->getEnvelope(&sEnvelope);
3413
3414 if( sEnvelope.MinX < -180 )
3415 sEnvelope.MinX = -180;
3416 if( sEnvelope.MinX > 180 )
3417 sEnvelope.MinX = 180;
3418
3419 if( sEnvelope.MinY < -90 )
3420 sEnvelope.MinY = -90;
3421 if( sEnvelope.MinY > 90 )
3422 sEnvelope.MinY = 90;
3423
3424 if( sEnvelope.MaxX > 180 )
3425 sEnvelope.MaxX = 180;
3426 if( sEnvelope.MaxX < -180 )
3427 sEnvelope.MaxX = -180;
3428
3429 if( sEnvelope.MaxY > 90 )
3430 sEnvelope.MaxY = 90;
3431 if( sEnvelope.MaxY < -90 )
3432 sEnvelope.MaxY = -90;
3433
3434 if( sEnvelope.MinX == -180 && sEnvelope.MinY == -90 &&
3435 sEnvelope.MaxX == 180 && sEnvelope.MaxY == 90 )
3436 {
3437 return;
3438 }
3439
3440 m_poSpatialFilter = json_object_new_object();
3441
3442 if( m_abIsGeoPoint[iGeomField] )
3443 {
3444 json_object* geo_bounding_box = json_object_new_object();
3445 json_object_object_add(m_poSpatialFilter, "geo_bounding_box", geo_bounding_box);
3446
3447 CPLString osPath = BuildPathFromArray(m_aaosGeomFieldPaths[iGeomField]);
3448
3449 json_object* field = json_object_new_object();
3450 json_object_object_add(geo_bounding_box, osPath.c_str(), field);
3451
3452 json_object* top_left = json_object_new_object();
3453 json_object_object_add(field, "top_left", top_left);
3454 json_object_object_add(top_left, "lat", json_object_new_double_with_precision(sEnvelope.MaxY, 6));
3455 json_object_object_add(top_left, "lon", json_object_new_double_with_precision(sEnvelope.MinX, 6));
3456
3457 json_object* bottom_right = json_object_new_object();
3458 json_object_object_add(field, "bottom_right", bottom_right);
3459 json_object_object_add(bottom_right, "lat", json_object_new_double_with_precision(sEnvelope.MinY, 6));
3460 json_object_object_add(bottom_right, "lon", json_object_new_double_with_precision(sEnvelope.MaxX, 6));
3461 }
3462 else
3463 {
3464 json_object* geo_shape = json_object_new_object();
3465 json_object_object_add(m_poSpatialFilter, "geo_shape", geo_shape);
3466
3467 CPLString osPath = BuildPathFromArray(m_aaosGeomFieldPaths[iGeomField]);
3468
3469 json_object* field = json_object_new_object();
3470 json_object_object_add(geo_shape, osPath.c_str(), field);
3471
3472 json_object* shape = json_object_new_object();
3473 json_object_object_add(field, "shape", shape);
3474
3475 json_object_object_add(shape, "type", json_object_new_string("envelope"));
3476
3477 json_object* coordinates = json_object_new_array();
3478 json_object_object_add(shape, "coordinates", coordinates);
3479
3480 json_object* top_left = json_object_new_array();
3481 json_object_array_add(top_left, json_object_new_double_with_precision(sEnvelope.MinX, 6));
3482 json_object_array_add(top_left, json_object_new_double_with_precision(sEnvelope.MaxY, 6));
3483 json_object_array_add(coordinates, top_left);
3484
3485 json_object* bottom_right = json_object_new_array();
3486 json_object_array_add(bottom_right, json_object_new_double_with_precision(sEnvelope.MaxX, 6));
3487 json_object_array_add(bottom_right, json_object_new_double_with_precision(sEnvelope.MinY, 6));
3488 json_object_array_add(coordinates, bottom_right);
3489 }
3490 }
3491
3492 /************************************************************************/
3493 /* GetExtent() */
3494 /************************************************************************/
3495
3496 OGRErr OGRElasticLayer::GetExtent(int iGeomField, OGREnvelope *psExtent, int bForce )
3497 {
3498 FinalizeFeatureDefn();
3499
3500 if( iGeomField < 0 || iGeomField >= GetLayerDefn()->GetGeomFieldCount() )
3501 {
3502 if( iGeomField != 0 )
3503 {
3504 CPLError(CE_Failure, CPLE_AppDefined,
3505 "Invalid geometry field index : %d", iGeomField);
3506 }
3507 return OGRERR_FAILURE;
3508 }
3509
3510 // geo_shape aggregation is only available since ES 7.8, but only with XPack
3511 // for now
3512 if( !m_abIsGeoPoint[iGeomField] &&
3513 !(m_poDS->m_nMajorVersion > 7 ||
3514 (m_poDS->m_nMajorVersion == 7 && m_poDS->m_nMinorVersion >= 8)) )
3515 {
3516 m_bUseSingleQueryParams = true;
3517 const auto eRet = OGRLayer::GetExtentInternal(iGeomField, psExtent, bForce);
3518 m_bUseSingleQueryParams = false;
3519 return eRet;
3520 }
3521
3522 CPLString osFilter = CPLSPrintf("{ \"size\": 0, \"aggs\" : { \"bbox\" : { \"geo_bounds\" : { \"field\" : \"%s\" } } } }",
3523 BuildPathFromArray(m_aaosGeomFieldPaths[iGeomField]).c_str() );
3524 CPLString osURL = CPLSPrintf("%s/%s", m_poDS->GetURL(), m_osIndexName.c_str());
3525 if (m_poDS->m_nMajorVersion < 7)
3526 osURL += CPLSPrintf("/%s", m_osMappingName.c_str());
3527 osURL += "/_search?pretty";
3528 AddTimeoutTerminateAfterToURL(osURL);
3529
3530 CPLPushErrorHandler(CPLQuietErrorHandler);
3531 json_object* poResponse = m_poDS->RunRequest(osURL.c_str(), osFilter.c_str());
3532 CPLPopErrorHandler();
3533 if( poResponse == nullptr )
3534 {
3535 const char* pszLastErrorMsg = CPLGetLastErrorMsg();
3536 if( !m_abIsGeoPoint[iGeomField] &&
3537 strstr(pszLastErrorMsg, "Fielddata is not supported on field") != nullptr )
3538 {
3539 CPLDebug("ES",
3540 "geo_bounds aggregation failed, likely because of lack "
3541 "of XPack. Using client-side method");
3542 CPLErrorReset();
3543 }
3544 else
3545 {
3546 CPLError(CE_Failure, CPLE_AppDefined, "%s", pszLastErrorMsg);
3547 }
3548 }
3549
3550 json_object* poBounds = json_ex_get_object_by_path(poResponse, "aggregations.bbox.bounds");
3551 json_object* poTopLeft = json_ex_get_object_by_path(poBounds, "top_left");
3552 json_object* poBottomRight = json_ex_get_object_by_path(poBounds, "bottom_right");
3553 json_object* poTopLeftLon = json_ex_get_object_by_path(poTopLeft, "lon");
3554 json_object* poTopLeftLat = json_ex_get_object_by_path(poTopLeft, "lat");
3555 json_object* poBottomRightLon = json_ex_get_object_by_path(poBottomRight, "lon");
3556 json_object* poBottomRightLat = json_ex_get_object_by_path(poBottomRight, "lat");
3557
3558 OGRErr eErr;
3559 if( poTopLeftLon == nullptr || poTopLeftLat == nullptr ||
3560 poBottomRightLon == nullptr || poBottomRightLat == nullptr )
3561 {
3562 m_bUseSingleQueryParams = true;
3563 const auto eRet = OGRLayer::GetExtentInternal(iGeomField, psExtent, bForce);
3564 m_bUseSingleQueryParams = false;
3565 return eRet;
3566 }
3567 else
3568 {
3569 double dfMinX = json_object_get_double( poTopLeftLon );
3570 double dfMaxY = json_object_get_double( poTopLeftLat );
3571 double dfMaxX = json_object_get_double( poBottomRightLon );
3572 double dfMinY = json_object_get_double( poBottomRightLat );
3573
3574 psExtent->MinX = dfMinX;
3575 psExtent->MaxY = dfMaxY;
3576 psExtent->MaxX = dfMaxX;
3577 psExtent->MinY = dfMinY;
3578
3579 eErr = OGRERR_NONE;
3580 }
3581 json_object_put(poResponse);
3582
3583 return eErr;
3584 }
3585