1""" 2Functions to register map layer in space time datasets and the temporal database 3 4Usage: 5 6.. code-block:: python 7 8 import grass.temporal as tgis 9 10 tgis.register_maps_in_space_time_dataset(type, name, maps) 11 12(C) 2012-2013 by the GRASS Development Team 13This program is free software under the GNU General Public 14License (>=v2). Read the file COPYING that comes with GRASS 15for details. 16 17:authors: Soeren Gebbert 18""" 19from datetime import datetime 20import grass.script as gscript 21from .core import get_tgis_message_interface, init_dbif, get_current_mapset 22from .open_stds import open_old_stds 23from .abstract_map_dataset import AbstractMapDataset 24from .factory import dataset_factory 25from .datetime_math import check_datetime_string, increment_datetime_by_string, string_to_datetime 26 27############################################################################### 28 29 30def register_maps_in_space_time_dataset( 31 type, name, maps=None, file=None, start=None, 32 end=None, unit=None, increment=None, dbif=None, 33 interval=False, fs="|", update_cmd_list=True): 34 """Use this method to register maps in space time datasets. 35 36 Additionally a start time string and an increment string can be 37 specified to assign a time interval automatically to the maps. 38 39 It takes care of the correct update of the space time datasets from all 40 registered maps. 41 42 :param type: The type of the maps raster, raster_3d or vector 43 :param name: The name of the space time dataset. Maps will be 44 registered in the temporal database if the name was set 45 to None 46 :param maps: A comma separated list of map names 47 :param file: Input file, one map per line map with start and optional 48 end time 49 :param start: The start date and time of the first map 50 (format absolute: "yyyy-mm-dd HH:MM:SS" or "yyyy-mm-dd", 51 format relative is integer 5) 52 :param end: The end date and time of the first map 53 (format absolute: "yyyy-mm-dd HH:MM:SS" or "yyyy-mm-dd", 54 format relative is integer 5) 55 :param unit: The unit of the relative time: years, months, days, 56 hours, minutes, seconds 57 :param increment: Time increment between maps for time stamp creation 58 (format absolute: NNN seconds, minutes, hours, days, 59 weeks, months, years; format relative: 1.0) 60 :param dbif: The database interface to be used 61 :param interval: If True, time intervals are created in case the start 62 time and an increment is provided 63 :param fs: Field separator used in input file 64 :param update_cmd_list: If is True, the command that was invoking this 65 process will be written to the process history 66 """ 67 start_time_in_file = False 68 end_time_in_file = False 69 msgr = get_tgis_message_interface() 70 71 # Make sure the arguments are of type string 72 if start != "" and start is not None: 73 start = str(start) 74 if end != "" and end is not None: 75 end = str(end) 76 if increment != "" and increment is not None: 77 increment = str(increment) 78 79 if maps and file: 80 msgr.fatal(_("%s= and %s= are mutually exclusive") % ("maps", "file")) 81 82 if end and increment: 83 msgr.fatal(_("%s= and %s= are mutually exclusive") % ("end", 84 "increment")) 85 86 if end and interval: 87 msgr.fatal(_("%s= and the %s flag are mutually exclusive") % ("end", 88 "interval")) 89 90 if increment and not start: 91 msgr.fatal(_("The increment option requires the start option")) 92 93 if interval and not start: 94 msgr.fatal(_("The interval flag requires the start option")) 95 96 if end and not start: 97 msgr.fatal(_("Please specify %s= and %s=") % ("start_time", 98 "end_time")) 99 100 if not maps and not file: 101 msgr.fatal(_("Please specify %s= or %s=") % ("maps", "file")) 102 # We may need the mapset 103 mapset = get_current_mapset() 104 dbif, connected = init_dbif(None) 105 106 # The name of the space time dataset is optional 107 if name: 108 sp = open_old_stds(name, type, dbif) 109 110 if sp.is_time_relative() and (start or end) and not unit: 111 dbif.close() 112 msgr.fatal(_("Space time %(sp)s dataset <%(name)s> with relative" 113 " time found, but no relative unit set for %(sp)s " 114 "maps") % {'name': name, 115 'sp': sp.get_new_map_instance(None).get_type()}) 116 117 maplist = [] 118 119 # Map names as comma separated string 120 if maps: 121 if maps.find(",") < 0: 122 maplist = [maps, ] 123 else: 124 maplist = maps.split(",") 125 126 # Build the map list again with the ids 127 for count in range(len(maplist)): 128 row = {} 129 mapid = AbstractMapDataset.build_id(maplist[count], mapset, None) 130 131 row["id"] = mapid 132 maplist[count] = row 133 134 # Read the map list from file 135 if file: 136 fd = open(file, "r") 137 138 line = True 139 while True: 140 line = fd.readline().strip() 141 if not line: 142 break 143 144 line_list = line.split(fs) 145 146 # Detect start and end time 147 if len(line_list) == 2: 148 start_time_in_file = True 149 end_time_in_file = False 150 elif len(line_list) == 3: 151 start_time_in_file = True 152 end_time_in_file = True 153 else: 154 start_time_in_file = False 155 end_time_in_file = False 156 157 mapname = line_list[0].strip() 158 row = {} 159 160 if start_time_in_file and end_time_in_file: 161 row["start"] = line_list[1].strip() 162 row["end"] = line_list[2].strip() 163 164 if start_time_in_file and not end_time_in_file: 165 row["start"] = line_list[1].strip() 166 167 row["id"] = AbstractMapDataset.build_id(mapname, mapset) 168 169 maplist.append(row) 170 171 if start_time_in_file is True and increment: 172 increment = None 173 msgr.warning(_("The increment option will be ignored because of time stamps in input file")) 174 175 if start_time_in_file is True and interval: 176 increment = None 177 msgr.warning(_("The interval flag will be ignored because of time stamps in input file")) 178 179 num_maps = len(maplist) 180 map_object_list = [] 181 statement = "" 182 # Store the ids of datasets that must be updated 183 datatsets_to_modify = {} 184 185 msgr.message(_("Gathering map information...")) 186 187 for count in range(len(maplist)): 188 if count % 50 == 0: 189 msgr.percent(count, num_maps, 1) 190 191 # Get a new instance of the map type 192 map = dataset_factory(type, maplist[count]["id"]) 193 194 if map.map_exists() is not True: 195 msgr.fatal(_("Unable to update %(t)s map <%(id)s>. " 196 "The map does not exist.") % {'t': map.get_type(), 197 'id': map.get_map_id()}) 198 199 # Use the time data from file 200 if "start" in maplist[count]: 201 start = maplist[count]["start"] 202 if "end" in maplist[count]: 203 end = maplist[count]["end"] 204 205 is_in_db = False 206 207 # Put the map into the database 208 if not map.is_in_db(dbif): 209 # Break in case no valid time is provided 210 if (start == "" or start is None) and not map.has_grass_timestamp(): 211 dbif.close() 212 if map.get_layer(): 213 msgr.fatal(_("Unable to register %(t)s map <%(id)s> with " 214 "layer %(l)s. The map has timestamp and " 215 "the start time is not set.") % { 216 't': map.get_type(), 'id': map.get_map_id(), 217 'l': map.get_layer()}) 218 else: 219 msgr.fatal(_("Unable to register %(t)s map <%(id)s>. The" 220 " map has no timestamp and the start time " 221 "is not set.") % {'t': map.get_type(), 222 'id': map.get_map_id()}) 223 if start != "" and start is not None: 224 # We need to check if the time is absolute and the unit was specified 225 time_object = check_datetime_string(start) 226 if isinstance(time_object, datetime) and unit: 227 msgr.fatal(_("%(u)s= can only be set for relative time") % 228 {'u': "unit"}) 229 if not isinstance(time_object, datetime) and not unit: 230 msgr.fatal(_("%(u)s= must be set in case of relative time" 231 " stamps") % {'u': "unit"}) 232 233 if unit: 234 map.set_time_to_relative() 235 else: 236 map.set_time_to_absolute() 237 238 else: 239 is_in_db = True 240 # Check the overwrite flag 241 if not gscript.overwrite(): 242 if map.get_layer(): 243 msgr.warning(_("Map is already registered in temporal " 244 "database. Unable to update %(t)s map " 245 "<%(id)s> with layer %(l)s. Overwrite flag" 246 " is not set.") % {'t': map.get_type(), 247 'id': map.get_map_id(), 248 'l': str(map.get_layer())}) 249 else: 250 msgr.warning(_("Map is already registered in temporal " 251 "database. Unable to update %(t)s map " 252 "<%(id)s>. Overwrite flag is not set.") % 253 {'t': map.get_type(), 'id': map.get_map_id()}) 254 255 # Simple registration is allowed 256 if name: 257 map_object_list.append(map) 258 # Jump to next map 259 continue 260 261 # Select information from temporal database 262 map.select(dbif) 263 264 # Save the datasets that must be updated 265 datasets = map.get_registered_stds(dbif) 266 if datasets is not None: 267 for dataset in datasets: 268 if dataset != "": 269 datatsets_to_modify[dataset] = dataset 270 271 if name and map.get_temporal_type() != sp.get_temporal_type(): 272 dbif.close() 273 if map.get_layer(): 274 msgr.fatal(_("Unable to update %(t)s map <%(id)s> " 275 "with layer %(l)s. The temporal types " 276 "are different.") % {'t': map.get_type(), 277 'id': map.get_map_id(), 278 'l': map.get_layer()}) 279 else: 280 msgr.fatal(_("Unable to update %(t)s map <%(id)s>. " 281 "The temporal types are different.") % 282 {'t': map.get_type(), 283 'id': map.get_map_id()}) 284 285 # Load the data from the grass file database 286 map.load() 287 288 # Try to read an existing time stamp from the grass spatial database 289 # in case this map wasn't already registered in the temporal database 290 # Read the spatial database time stamp only, if no time stamp was provided for this map 291 # as method argument or in the input file 292 if not is_in_db and not start: 293 map.read_timestamp_from_grass() 294 295 # Set the valid time 296 if start: 297 # In case the time is in the input file we ignore the increment 298 # counter 299 if start_time_in_file: 300 count = 1 301 assign_valid_time_to_map(ttype=map.get_temporal_type(), 302 map=map, start=start, end=end, unit=unit, 303 increment=increment, mult=count, 304 interval=interval) 305 306 if is_in_db: 307 # Gather the SQL update statement 308 statement += map.update_all(dbif=dbif, execute=False) 309 else: 310 # Gather the SQL insert statement 311 statement += map.insert(dbif=dbif, execute=False) 312 313 # Sqlite3 performance is better for huge datasets when committing in 314 # small chunks 315 if dbif.get_dbmi().__name__ == "sqlite3": 316 if count % 100 == 0: 317 if statement is not None and statement != "": 318 dbif.execute_transaction(statement) 319 statement = "" 320 321 # Store the maps in a list to register in a space time dataset 322 if name: 323 map_object_list.append(map) 324 325 msgr.percent(num_maps, num_maps, 1) 326 327 if statement is not None and statement != "": 328 msgr.message(_("Registering maps in the temporal database...")) 329 dbif.execute_transaction(statement) 330 331 # Finally Register the maps in the space time dataset 332 if name and map_object_list: 333 count = 0 334 num_maps = len(map_object_list) 335 msgr.message(_("Registering maps in the space time dataset...")) 336 for map in map_object_list: 337 if count % 50 == 0: 338 msgr.percent(count, num_maps, 1) 339 sp.register_map(map=map, dbif=dbif) 340 count += 1 341 342 # Update the space time tables 343 if name and map_object_list: 344 msgr.message(_("Updating space time dataset...")) 345 sp.update_from_registered_maps(dbif) 346 if update_cmd_list is True: 347 sp.update_command_string(dbif=dbif) 348 349 # Update affected datasets 350 if datatsets_to_modify: 351 for dataset in datatsets_to_modify: 352 if type == "rast" or type == "raster": 353 ds = dataset_factory("strds", dataset) 354 elif type == "raster_3d" or type == "rast3d" or type == "raster3d": 355 ds = dataset_factory("str3ds", dataset) 356 elif type == "vect" or type == "vector": 357 ds = dataset_factory("stvds", dataset) 358 ds.select(dbif) 359 ds.update_from_registered_maps(dbif) 360 361 if connected is True: 362 dbif.close() 363 364 msgr.percent(num_maps, num_maps, 1) 365 366 367############################################################################### 368 369def assign_valid_time_to_map(ttype, map, start, end, unit, increment=None, 370 mult=1, interval=False): 371 """Assign the valid time to a map dataset 372 373 :param ttype: The temporal type which should be assigned 374 and which the time format is of 375 :param map: A map dataset object derived from abstract_map_dataset 376 :param start: The start date and time of the first map 377 (format absolute: "yyyy-mm-dd HH:MM:SS" or "yyyy-mm-dd", 378 format relative is integer 5) 379 :param end: The end date and time of the first map 380 (format absolute: "yyyy-mm-dd HH:MM:SS" or "yyyy-mm-dd", 381 format relative is integer 5) 382 :param unit: The unit of the relative time: years, months, 383 days, hours, minutes, seconds 384 :param increment: Time increment between maps for time stamp creation 385 (format absolute: NNN seconds, minutes, hours, days, 386 weeks, months, years; format relative is integer 1) 387 :param mult: A multiplier for the increment 388 :param interval: If True, time intervals are created in case the start 389 time and an increment is provided 390 """ 391 392 msgr = get_tgis_message_interface() 393 394 if ttype == "absolute": 395 start_time = string_to_datetime(start) 396 if start_time is None: 397 msgr.fatal(_("Unable to convert string \"%s\"into a " 398 "datetime object") % (start)) 399 end_time = None 400 401 if end: 402 end_time = string_to_datetime(end) 403 if end_time is None: 404 msgr.fatal(_("Unable to convert string \"%s\"into a " 405 "datetime object") % (end)) 406 407 # Add the increment 408 if increment: 409 start_time = increment_datetime_by_string( 410 start_time, increment, mult) 411 if start_time is None: 412 msgr.fatal(_("Error occurred in increment computation")) 413 if interval: 414 end_time = increment_datetime_by_string( 415 start_time, increment, 1) 416 if end_time is None: 417 msgr.fatal(_("Error occurred in increment computation")) 418 419 if map.get_layer(): 420 msgr.debug(1, _("Set absolute valid time for map <%(id)s> with " 421 "layer %(layer)s to %(start)s - %(end)s") % 422 {'id': map.get_map_id(), 'layer': map.get_layer(), 423 'start': str(start_time), 'end': str(end_time)}) 424 else: 425 msgr.debug(1, _("Set absolute valid time for map <%s> to %s - %s") 426 % (map.get_map_id(), str(start_time), str(end_time))) 427 428 map.set_absolute_time(start_time, end_time) 429 else: 430 start_time = int(start) 431 end_time = None 432 433 if end: 434 end_time = int(end) 435 436 if increment: 437 start_time = start_time + mult * int(increment) 438 if interval: 439 end_time = start_time + int(increment) 440 441 if map.get_layer(): 442 msgr.debug(1, _("Set relative valid time for map <%s> with layer" 443 " %s to %i - %s with unit %s") % 444 (map.get_map_id(), map.get_layer(), start_time, 445 str(end_time), unit)) 446 else: 447 msgr.debug(1, _("Set relative valid time for map <%s> to %i - %s " 448 "with unit %s") % (map.get_map_id(), start_time, 449 str(end_time), unit)) 450 451 map.set_relative_time(start_time, end_time, unit) 452 453 454############################################################################## 455 456def register_map_object_list(type, map_list, output_stds, 457 delete_empty=False, unit=None, dbif=None): 458 """Register a list of AbstractMapDataset objects in the temporal database 459 and optional in a space time dataset. 460 461 :param type: The type of the map layer (raster, raster_3d, vector) 462 :param map_list: List of AbstractMapDataset objects 463 :param output_stds: The output stds 464 :param delete_empty: Set True to delete empty map layer found in the map_list 465 :param unit: The temporal unit of the space time dataset 466 :param dbif: The database interface to be used 467 468 """ 469 import grass.pygrass.modules as pymod 470 import copy 471 472 dbif, connected = init_dbif(dbif) 473 474 filename = gscript.tempfile(True) 475 file = open(filename, 'w') 476 477 empty_maps = [] 478 for map_layer in map_list: 479 # Read the map data 480 map_layer.load() 481 # In case of a empty map continue, do not register empty maps 482 483 if delete_empty: 484 if type in ["raster", "raster_3d", "rast", "rast3d"]: 485 if map_layer.metadata.get_min() is None and \ 486 map_layer.metadata.get_max() is None: 487 empty_maps.append(map_layer) 488 continue 489 if type == "vector": 490 if map_layer.metadata.get_number_of_primitives() == 0: 491 empty_maps.append(map_layer) 492 continue 493 494 start, end = map_layer.get_temporal_extent_as_tuple() 495 id = map_layer.get_id() 496 if not end: 497 end = start 498 string = "%s|%s|%s\n" % (id, str(start), str(end)) 499 file.write(string) 500 file.close() 501 502 if output_stds: 503 output_stds_id = output_stds.get_id() 504 else: 505 output_stds_id = None 506 507 register_maps_in_space_time_dataset(type, output_stds_id, unit=unit, 508 file=filename, dbif=dbif) 509 510 g_remove = pymod.Module("g.remove", flags='f', quiet=True, 511 run_=False, finish_=True) 512 513 # Remove empty maps and unregister them from the temporal database 514 if len(empty_maps) > 0: 515 for map in empty_maps: 516 mod = copy.deepcopy(g_remove) 517 if map.get_name(): 518 if map.get_type() == "raster": 519 mod(type='raster', name=map.get_name()) 520 if map.get_type() == "raster3d": 521 mod(type='raster_3d', name=map.get_name()) 522 if map.get_type() == "vector": 523 mod(type='vector', name=map.get_name()) 524 mod.run() 525 if map.is_in_db(dbif): 526 map.delete(dbif) 527 528 if connected: 529 dbif.close() 530 531if __name__ == "__main__": 532 import doctest 533 doctest.testmod() 534