1"""! 2@brief Preparation of parameters for drivers, which download it, and managing downloaded data. 3 4List of classes: 5 - wms_base::WMSBase 6 - wms_base::GRASSImporter 7 - wms_base::WMSDriversInfo 8 9(C) 2012-2019 by the GRASS Development Team 10 11This program is free software under the GNU General Public License 12(>=v2). Read the file COPYING that comes with GRASS for details. 13 14@author Stepan Turek <stepan.turek seznam.cz> (Mentor: Martin Landa) 15""" 16 17import os 18from math import ceil 19 20import base64 21 22try: 23 from urllib2 import Request, urlopen, HTTPError 24 from httplib import HTTPException 25except ImportError: 26 from urllib.request import Request, urlopen 27 from urllib.error import HTTPError 28 from http.client import HTTPException 29 30 31import grass.script as grass 32from grass.exceptions import CalledModuleError 33 34 35class WMSBase(object): 36 37 def __init__(self): 38 # these variables are information for destructor 39 self.temp_files_to_cleanup = [] 40 41 self.params = {} 42 self.tile_size = {'bbox': None} 43 44 self.temp_map = None 45 self.temp_warpmap = None 46 47 def __del__(self): 48 49 # tries to remove temporary files, all files should be 50 # removed before, implemented just in case of unexpected 51 # stop of module 52 for temp_file in self.temp_files_to_cleanup: 53 grass.try_remove(temp_file) 54 55 def _debug(self, fn, msg): 56 grass.debug("%s.%s: %s" % 57 (self.__class__.__name__, fn, msg)) 58 59 def _initializeParameters(self, options, flags): 60 self._debug("_initialize_parameters", "started") 61 62 # initialization of module parameters (options, flags) 63 self.params['driver'] = options['driver'] 64 drv_info = WMSDriversInfo() 65 66 driver_props = drv_info.GetDrvProperties(options['driver']) 67 self._checkIgnoeredParams(options, flags, driver_props) 68 69 self.params['capfile'] = options['capfile'].strip() 70 71 for key in ['url', 'layers', 'styles', 'method']: 72 self.params[key] = options[key].strip() 73 74 self.flags = flags 75 76 if self.flags['o']: 77 self.params['transparent'] = 'FALSE' 78 else: 79 self.params['transparent'] = 'TRUE' 80 81 for key in ['password', 'username', 'urlparams']: 82 self.params[key] = options[key] 83 84 if (self.params ['password'] and self.params ['username'] == '') or \ 85 (self.params['password'] == '' and self.params['username']): 86 grass.fatal(_("Please insert both %s and %s parameters or none of them." % 87 ('password', 'username'))) 88 89 self.params['bgcolor'] = options['bgcolor'].strip() 90 91 if options['format'] == "jpeg" and \ 92 not 'format' in driver_props['ignored_params']: 93 if not flags['o'] and \ 94 'WMS' in self.params['driver']: 95 grass.warning(_("JPEG format does not support transparency")) 96 97 self.params['format'] = drv_info.GetFormat(options['format']) 98 if not self.params['format']: 99 self.params['format'] = self.params['format'] 100 101 # TODO: get srs from Tile Service file in OnEarth_GRASS driver 102 self.params['srs'] = int(options['srs']) 103 if self.params['srs'] <= 0 and not 'srs' in driver_props['ignored_params']: 104 grass.fatal(_("Invalid EPSG code %d") % self.params['srs']) 105 106 self.params['wms_version'] = options['wms_version'] 107 if "CRS" in GetSRSParamVal(self.params['srs']) and self.params['wms_version'] == "1.1.1": 108 self.params['wms_version'] = "1.3.0" 109 grass.warning( 110 _("WMS version <1.3.0> will be used, because version <1.1.1> does not support <%s>projection") % 111 GetSRSParamVal( 112 self.params['srs'])) 113 114 if self.params['wms_version'] == "1.3.0": 115 self.params['proj_name'] = "CRS" 116 else: 117 self.params['proj_name'] = "SRS" 118 119 # read projection info 120 self.proj_location = grass.read_command('g.proj', 121 flags='jf').rstrip('\n') 122 self.proj_location = self._modifyProj(self.proj_location) 123 124 self.source_epsg = str(GetEpsg(self.params['srs'])) 125 self.target_epsg = None 126 target_crs = grass.parse_command('g.proj', flags='g', delimiter = '=') 127 if 'epsg' in target_crs.keys(): 128 self.target_epsg = target_crs['epsg'] 129 if self.source_epsg != self.target_epsg: 130 grass.warning(_("SRS differences: WMS source EPSG %s != location EPSG %s (use srs=%s to adjust)") % 131 (self.source_epsg, self.target_epsg, self.target_epsg)) 132 133 self.proj_srs = grass.read_command('g.proj', 134 flags='jf', 135 epsg=str(GetEpsg(self.params['srs']))) 136 self.proj_srs = self.proj_srs.rstrip('\n') 137 138 self.proj_srs = self._modifyProj(self.proj_srs) 139 140 if not self.proj_srs or not self.proj_location: 141 grass.fatal(_("Unable to get projection info")) 142 143 self.region = options['region'] 144 145 min_tile_size = 100 146 maxcols = int(options['maxcols']) 147 if maxcols <= min_tile_size: 148 grass.fatal(_("Maxcols must be greater than 100")) 149 150 maxrows = int(options['maxrows']) 151 if maxrows <= min_tile_size: 152 grass.fatal(_("Maxrows must be greater than 100")) 153 154 # setting optimal tile size according to maxcols and maxrows constraint 155 # and region cols and rows 156 self.tile_size['cols'] = int( 157 self.region['cols'] / 158 ceil( 159 self.region['cols'] / 160 float(maxcols))) 161 self.tile_size['rows'] = int( 162 self.region['rows'] / 163 ceil( 164 self.region['rows'] / 165 float(maxrows))) 166 167 # default format for GDAL library 168 self.gdal_drv_format = "GTiff" 169 170 self._debug("_initialize_parameters", "finished") 171 172 def _modifyProj(self, proj): 173 """!Modify proj.4 string for usage in this module""" 174 175 # add +wktext parameter to avoid droping of +nadgrids parameter (if presented) in gdalwarp 176 if '+nadgrids=' in proj and ' +wktext' not in proj: 177 proj += ' +wktext' 178 179 return proj 180 181 def _checkIgnoeredParams(self, options, flags, driver_props): 182 """!Write warnings for set parameters and flags, which chosen driver does not use.""" 183 184 not_relevant_params = [] 185 for i_param in driver_props['ignored_params']: 186 187 if i_param in options and \ 188 options[i_param] and \ 189 i_param not in ['srs', 'wms_version', 'format']: # params with default value 190 not_relevant_params.append('<' + i_param + '>') 191 192 if len(not_relevant_params) > 0: 193 grass.warning(_("These parameter are ignored: %s\n\ 194 %s driver does not support the parameters." % 195 (','.join(not_relevant_params), options['driver']))) 196 197 not_relevant_flags = [] 198 for i_flag in driver_props['ignored_flags']: 199 200 if flags[i_flag]: 201 not_relevant_flags.append('<' + i_flag + '>') 202 203 if len(not_relevant_flags) > 0: 204 grass.warning(_("These flags are ignored: %s\n\ 205 %s driver does not support the flags." % 206 (','.join(not_relevant_flags), options['driver']))) 207 208 def GetMap(self, options, flags): 209 """!Download data from WMS server.""" 210 211 self._initializeParameters(options, flags) 212 self.bbox = self._computeBbox() 213 214 self.temp_map = self._download() 215 216 if not self.temp_map: 217 return 218 219 self._reprojectMap() 220 221 return self.temp_warpmap 222 223 def _fetchCapabilities(self, options): 224 """!Download capabilities from WMS server 225 """ 226 cap_url = options['url'].strip() 227 228 if "?" in cap_url: 229 cap_url += "&" 230 else: 231 cap_url += "?" 232 233 if 'WMTS' in options['driver']: 234 cap_url += "SERVICE=WMTS&REQUEST=GetCapabilities&VERSION=1.0.0" 235 elif 'OnEarth' in options['driver']: 236 cap_url += "REQUEST=GetTileService" 237 else: 238 cap_url += "SERVICE=WMS&REQUEST=GetCapabilities&VERSION=" + options['wms_version'] 239 240 if options['urlparams']: 241 cap_url += "&" + options['urlparams'] 242 243 grass.debug('Fetching capabilities file.\n%s' % cap_url) 244 245 try: 246 cap = self._fetchDataFromServer(cap_url, options['username'], options['password']) 247 except (IOError, HTTPException) as e: 248 if isinstance(e, HTTPError) and e.code == 401: 249 grass.fatal( 250 _("Authorization failed to <%s> when fetching capabilities") % 251 options['url']) 252 else: 253 msg = _("Unable to fetch capabilities from <{0}>. Reason: ").format( 254 options['url']) 255 256 if hasattr(e, 'reason'): 257 msg += '{0}'.format(e.reason) 258 else: 259 msg += '{0}'.format(e) 260 261 grass.fatal(msg) 262 263 grass.debug('Fetching capabilities OK') 264 return grass.decode(cap.read()) 265 266 def _fetchDataFromServer(self, url, username=None, password=None): 267 """!Fetch data from server 268 """ 269 request = Request(url) 270 if username and password: 271 base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '') 272 request.add_header("Authorization", "Basic %s" % base64string) 273 274 try: 275 return urlopen(request) 276 except ValueError as error: 277 grass.fatal("%s" % error) 278 279 def GetCapabilities(self, options): 280 """!Get capabilities from WMS server 281 """ 282 cap = self._fetchCapabilities(options) 283 capfile_output = options['capfile_output'].strip() 284 285 # save to file 286 if capfile_output: 287 try: 288 with open(capfile_output, "w") as temp: 289 temp.write(cap) 290 return 291 except IOError as error: 292 grass.fatal(_("Unable to open file '%s'.\n%s\n" % (capfile_output, error))) 293 294 # print to output 295 print(cap) 296 297 def _computeBbox(self): 298 """!Get region extent for WMS query (bbox) 299 """ 300 self._debug("_computeBbox", "started") 301 302 bbox_region_items = {'maxy': 'n', 'miny': 's', 'maxx': 'e', 'minx': 'w'} 303 bbox = {} 304 305 if self.proj_srs == self.proj_location: # TODO: do it better 306 for bbox_item, region_item in bbox_region_items.items(): 307 bbox[bbox_item] = self.region[region_item] 308 309 # if location projection and wms query projection are 310 # different, corner points of region are transformed into wms 311 # projection and then bbox is created from extreme coordinates 312 # of the transformed points 313 else: 314 for bbox_item, region_item in bbox_region_items.items(): 315 bbox[bbox_item] = None 316 317 temp_region = self._tempfile() 318 319 try: 320 temp_region_opened = open(temp_region, 'w') 321 temp_region_opened.write("%f %f\n%f %f\n%f %f\n%f %f\n" % 322 (self.region['e'], self.region['n'], 323 self.region['w'], self.region['n'], 324 self.region['w'], self.region['s'], 325 self.region['e'], self.region['s'])) 326 except IOError: 327 grass.fatal(_("Unable to write data into tempfile")) 328 finally: 329 temp_region_opened.close() 330 331 points = grass.read_command('m.proj', flags='d', 332 proj_out=self.proj_srs, 333 proj_in=self.proj_location, 334 input=temp_region, 335 quiet=True) # TODO: stdin 336 grass.try_remove(temp_region) 337 if not points: 338 grass.fatal(_("Unable to determine region, %s failed") % 'm.proj') 339 340 points = points.splitlines() 341 if len(points) != 4: 342 grass.fatal(_("Region definition: 4 points required")) 343 344 for point in points: 345 try: 346 point = list(map(float, point.split("|"))) 347 except ValueError: 348 grass.fatal(_('Reprojection of region using m.proj failed.')) 349 if not bbox['maxy']: 350 bbox['maxy'] = point[1] 351 bbox['miny'] = point[1] 352 bbox['maxx'] = point[0] 353 bbox['minx'] = point[0] 354 continue 355 356 if bbox['maxy'] < point[1]: 357 bbox['maxy'] = point[1] 358 elif bbox['miny'] > point[1]: 359 bbox['miny'] = point[1] 360 361 if bbox['maxx'] < point[0]: 362 bbox['maxx'] = point[0] 363 elif bbox['minx'] > point[0]: 364 bbox['minx'] = point[0] 365 366 self._debug("_computeBbox", "finished -> %s" % bbox) 367 368 # Ordering of coordinates axis of geographic coordinate 369 # systems in WMS 1.3.0 is flipped. If self.tile_size['flip_coords'] is 370 # True, coords in bbox need to be flipped in WMS query. 371 372 return bbox 373 374 def _reprojectMap(self): 375 """!Reproject data using gdalwarp if needed 376 """ 377 # reprojection of raster 378 do_reproject = True 379 if self.source_epsg is not None and self.target_epsg is not None \ 380 and self.source_epsg == self.target_epsg: 381 do_reproject = False 382 # TODO: correctly compare source and target crs 383 if do_reproject == True and self.proj_srs == self.proj_location: 384 do_reproject = False 385 if do_reproject: 386 grass.message(_("Reprojecting raster...")) 387 self.temp_warpmap = grass.tempfile() + '.tif' 388 389 if int(os.getenv('GRASS_VERBOSE', '2')) <= 2: 390 nuldev = open(os.devnull, 'w+') 391 else: 392 nuldev = None 393 394 if self.params['method'] == "nearest": 395 gdal_method = "near" 396 elif self.params['method'] == "linear": 397 gdal_method = "bilinear" 398 else: 399 gdal_method = self.params['method'] 400 401 # RGB rasters - alpha layer is added for cropping edges of projected raster 402 try: 403 if self.temp_map_bands_num == 3: 404 ps = grass.Popen(['gdalwarp', 405 '-s_srs', '%s' % self.proj_srs, 406 '-t_srs', '%s' % self.proj_location, 407 '-r', gdal_method, '-dstalpha', 408 self.temp_map, self.temp_warpmap], stdout=nuldev) 409 # RGBA rasters 410 else: 411 ps = grass.Popen(['gdalwarp', 412 '-s_srs', '%s' % self.proj_srs, 413 '-t_srs', '%s' % self.proj_location, 414 '-r', gdal_method, 415 self.temp_map, self.temp_warpmap], stdout=nuldev) 416 ps.wait() 417 except OSError as e: 418 grass.fatal('%s \nThis can be caused by missing %s utility. ' % (e, 'gdalwarp')) 419 420 if nuldev: 421 nuldev.close() 422 423 if ps.returncode != 0: 424 grass.fatal(_('%s failed') % 'gdalwarp') 425 grass.try_remove(self.temp_map) 426 # raster projection is same as projection of location 427 else: 428 self.temp_warpmap = self.temp_map 429 self.temp_files_to_cleanup.remove(self.temp_map) 430 431 return self.temp_warpmap 432 433 def _tempfile(self): 434 """!Create temp_file and append list self.temp_files_to_cleanup 435 with path of file 436 437 @return string path to temp_file 438 """ 439 temp_file = grass.tempfile() 440 if temp_file is None: 441 grass.fatal(_("Unable to create temporary files")) 442 443 # list of created tempfiles for destructor 444 self.temp_files_to_cleanup.append(temp_file) 445 446 return temp_file 447 448 449class GRASSImporter: 450 451 def __init__(self, opt_output, cleanup_bands): 452 453 self.cleanup_mask = False 454 self.cleanup_bands = cleanup_bands 455 456 # output map name 457 self.opt_output = opt_output 458 459 # suffix for existing mask (during overriding will be saved 460 # into raster named:self.opt_output + this suffix) 461 self.original_mask_suffix = "_temp_MASK" 462 463 # check names of temporary rasters, which module may create 464 maps = [] 465 for suffix in ('.red', '.green', '.blue', '.alpha', self.original_mask_suffix): 466 rast = self.opt_output + suffix 467 if grass.find_file(rast, element='cell', mapset='.')['file']: 468 maps.append(rast) 469 470 if len(maps) != 0: 471 grass.fatal(_("Please change output name, or change names of these rasters: %s, " 472 "module needs to create this temporary maps during execution.") % ",".join(maps)) 473 474 def __del__(self): 475 # removes temporary mask, used for import transparent or warped temp_map 476 if self.cleanup_mask: 477 # clear temporary mask, which was set by module 478 try: 479 grass.run_command('r.mask', quiet=True, flags='r') 480 except CalledModuleError: 481 grass.fatal(_('%s failed') % 'r.mask') 482 483 # restore original mask, if exists 484 if grass.find_file(self.opt_output + self.original_mask_suffix, 485 element='cell', mapset='.')['name']: 486 try: 487 mask_copy = self.opt_output + self.original_mask_suffix 488 grass.run_command('g.copy', quiet=True, 489 raster=mask_copy + ',MASK') 490 except CalledModuleError: 491 grass.fatal(_('%s failed') % 'g.copy') 492 493 # remove temporary created rasters 494 maps = [] 495 rast = self.opt_output + '.alpha' 496 if grass.find_file(rast, element='cell', mapset='.')['file']: 497 maps.append(rast) 498 499 if self.cleanup_bands: 500 for suffix in ('.red', '.green', '.blue', self.original_mask_suffix): 501 rast = self.opt_output + suffix 502 if grass.find_file(rast, element='cell', mapset='.')['file']: 503 maps.append(rast) 504 505 if maps: 506 grass.run_command('g.remove', 507 quiet=True, 508 flags='fb', 509 type='raster', 510 name=','.join(maps)) 511 512 # delete environmental variable which overrides region 513 if 'GRASS_REGION' in os.environ.keys(): 514 os.environ.pop('GRASS_REGION') 515 516 maplist = grass.read_command('g.list', type = 'raster', 517 pattern = '%s*' % (self.opt_output), 518 mapset = '.', separator = ',').rstrip('\n') 519 520 if len(maplist) == 0: 521 grass.fatal(_('WMS import failed, nothing imported')) 522 else: 523 for raster in maplist.split(','): 524 grass.raster_history(raster, overwrite = True) 525 grass.run_command('r.support', map=raster, description='generated by r.in.wms') 526 grass.message(_('<%s> created.') % raster) 527 528 529 def ImportMapIntoGRASS(self, raster): 530 """!Import raster into GRASS. 531 """ 532 # importing temp_map into GRASS 533 try: 534 # do not use -o flag ! 535 grass.run_command('r.in.gdal', flags='o', 536 quiet=True, overwrite=True, 537 input=raster, output=self.opt_output) 538 except CalledModuleError: 539 grass.fatal(_('%s failed') % 'r.in.gdal') 540 541 # information for destructor to cleanup temp_layers, created 542 # with r.in.gdal 543 544 # setting region for full extend of imported raster 545 if grass.find_file(self.opt_output + '.red', element='cell', mapset='.')['file']: 546 region_map = self.opt_output + '.red' 547 else: 548 region_map = self.opt_output 549 os.environ['GRASS_REGION'] = grass.region_env(rast=region_map) 550 551 # mask created from alpha layer, which describes real extend 552 # of warped layer (may not be a rectangle), also mask contains 553 # transparent parts of raster 554 if grass.find_file(self.opt_output + '.alpha', element='cell', mapset='.')['name']: 555 # saving current mask (if exists) into temp raster 556 if grass.find_file('MASK', element='cell', mapset='.')['name']: 557 try: 558 mask_copy = self.opt_output + self.original_mask_suffix 559 grass.run_command('g.copy', quiet=True, 560 raster='MASK,' + mask_copy) 561 except CalledModuleError: 562 grass.fatal(_('%s failed') % 'g.copy') 563 564 # info for destructor 565 self.cleanup_mask = True 566 try: 567 grass.run_command('r.mask', 568 quiet=True, overwrite=True, 569 maskcats="0", 570 flags='i', 571 raster=self.opt_output + '.alpha') 572 except CalledModuleError: 573 grass.fatal(_('%s failed') % 'r.mask') 574 575 if not self.cleanup_bands: 576 # use the MASK to set NULL vlues 577 for suffix in ('.red', '.green', '.blue'): 578 rast = self.opt_output + suffix 579 if grass.find_file(rast, element='cell', mapset='.')['file']: 580 grass.run_command('g.rename', rast='%s,%s' % (rast, rast + '_null'), quiet = True) 581 grass.run_command('r.mapcalc', expression = '%s = %s' % (rast, rast + '_null'), quiet = True) 582 grass.run_command('g.remove', type='raster', name='%s' % (rast + '_null'), flags = 'f', quiet = True) 583 584 # TODO one band + alpha band? 585 if grass.find_file(self.opt_output + '.red', element='cell', mapset='.')['file'] and self.cleanup_bands: 586 try: 587 grass.run_command('r.composite', 588 quiet=True, overwrite=True, 589 red=self.opt_output + '.red', 590 green=self.opt_output + '.green', 591 blue=self.opt_output + '.blue', 592 output=self.opt_output) 593 except CalledModuleError: 594 grass.fatal(_('%s failed') % 'r.composite') 595 596 597class WMSDriversInfo: 598 599 def __init__(self): 600 """!Provides information about driver parameters. 601 """ 602 603 # format labels 604 self.f_labels = ["geotiff", "tiff", "png", "jpeg", "gif", "png8"] 605 606 # form for request 607 self.formats = [ 608 "image/geotiff", 609 "image/tiff", 610 "image/png", 611 "image/jpeg", 612 "image/gif", 613 "image/png8"] 614 615 self.srs = ("epsg", "ogc") 616 617 def GetDrvProperties(self, driver): 618 """!Get information about driver parameters. 619 """ 620 if driver == 'WMS_GDAL': 621 return self._GDALDrvProperties() 622 if 'WMS' in driver: 623 return self._WMSProperties() 624 if 'WMTS' in driver: 625 return self._WMTSProperties() 626 if 'OnEarth' in driver: 627 return self._OnEarthProperties() 628 629 def _OnEarthProperties(self): 630 631 props = {} 632 props['ignored_flags'] = ['o'] 633 props['ignored_params'] = ['bgcolor', 'styles', 'capfile_output', 634 'format', 'srs', 'wms_version'] 635 props['req_multiple_layers'] = False 636 637 return props 638 639 def _WMSProperties(self): 640 641 props = {} 642 props['ignored_params'] = ['capfile'] 643 props['ignored_flags'] = [] 644 props['req_multiple_layers'] = True 645 646 return props 647 648 def _WMTSProperties(self): 649 650 props = {} 651 props['ignored_flags'] = ['o'] 652 props['ignored_params'] = ['urlparams', 'bgcolor', 'wms_version'] 653 props['req_multiple_layers'] = False 654 655 return props 656 657 def _GDALDrvProperties(self): 658 659 props = {} 660 props['ignored_flags'] = [] 661 props['ignored_params'] = ['urlparams', 'bgcolor', 'capfile', 'capfile_output', 662 'username', 'password'] 663 props['req_multiple_layers'] = True 664 665 return props 666 667 def GetFormatLabel(self, format): 668 """!Convert format request form to value in parameter 'format'. 669 """ 670 if format in self.formats: 671 return self.f_labels[self.formats.index(format)] 672 return None 673 674 def GetFormat(self, label): 675 """!Convert value in parameter 'format' to format request form. 676 """ 677 if label in self.f_labels: 678 return self.formats[self.f_labels.index(label)] 679 return None 680 681 def GetSrs(self): 682 """!Get supported srs prefixes (e.g. epsg/crs) 683 684 @todo filter according to version and driver params 685 """ 686 return self.srs 687 688 689# TODO move to utils? 690def GetSRSParamVal(srs): 691 """!Decides whether to use CRS or EPSG prefix according to srs number. 692 """ 693 694 if srs in [84, 83, 27]: 695 return "OGC:CRS{}".format(srs) 696 else: 697 return "EPSG:{}".format(srs) 698 699 700def GetEpsg(srs): 701 """ 702 @return EPSG number 703 If srs is CRS number, return EPSG number which corresponds to CRS number. 704 """ 705 if srs == 84: 706 return 4326 707 if srs == 83: 708 return 4269 709 if srs == 27: 710 return 4267 711 712 return srs 713