1# -*- coding: utf-8 -*- 2"""GRASS Python testing framework test case 3 4Copyright (C) 2014 by the GRASS Development Team 5This program is free software under the GNU General Public 6License (>=v2). Read the file COPYING that comes with GRASS GIS 7for details. 8 9:authors: Vaclav Petras 10""" 11from __future__ import print_function 12 13import os 14import subprocess 15import sys 16import hashlib 17import uuid 18import unittest 19 20from grass.pygrass.modules import Module 21from grass.exceptions import CalledModuleError 22from grass.script import shutil_which, text_to_string, encode 23 24from .gmodules import call_module, SimpleModule 25from .checkers import (check_text_ellipsis, 26 text_to_keyvalue, keyvalue_equals, diff_keyvalue, 27 file_md5, text_file_md5, files_equal_md5) 28from .utils import safe_repr 29from .gutils import is_map_in_mapset 30 31pyversion = sys.version_info[0] 32if pyversion == 2: 33 from StringIO import StringIO 34else: 35 from io import StringIO 36 unicode = str 37 38 39class TestCase(unittest.TestCase): 40 # we dissable R0904 for all TestCase classes because their purpose is to 41 # provide a lot of assert methods 42 # pylint: disable=R0904 43 """ 44 45 Always use keyword arguments for all parameters other than first two. For 46 the first two, it is recommended to use keyword arguments but not required. 47 Be especially careful and always use keyword argument syntax for *msg* 48 parameter. 49 """ 50 longMessage = True # to get both standard and custom message 51 maxDiff = None # we can afford long diffs 52 _temp_region = None # to control the temporary region 53 html_reports = False # output additional HTML files with failure details 54 readable_names = False # prefer shorter but unreadable map and file names 55 56 def __init__(self, methodName): 57 super(TestCase, self).__init__(methodName) 58 self.grass_modules = [] 59 self.supplementary_files = [] 60 # Python unittest doc is saying that strings use assertMultiLineEqual 61 # but only unicode type is registered 62 # TODO: report this as a bug? is this in Python 3.x? 63 self.addTypeEqualityFunc(str, 'assertMultiLineEqual') 64 65 def _formatMessage(self, msg, standardMsg): 66 """Honor the longMessage attribute when generating failure messages. 67 68 If longMessage is False this means: 69 70 * Use only an explicit message if it is provided 71 * Otherwise use the standard message for the assert 72 73 If longMessage is True: 74 75 * Use the standard message 76 * If an explicit message is provided, return string with both messages 77 78 Based on Python unittest _formatMessage, formatting changed. 79 """ 80 if not self.longMessage: 81 return msg or standardMsg 82 if msg is None: 83 return standardMsg 84 try: 85 # don't switch to '{}' formatting in Python 2.X 86 # it changes the way unicode input is handled 87 return '%s \n%s' % (msg, standardMsg) 88 except UnicodeDecodeError: 89 return '%s \n%s' % (safe_repr(msg), safe_repr(standardMsg)) 90 91 @classmethod 92 def use_temp_region(cls): 93 """Use temporary region instead of the standard one for this process. 94 95 If you use this method, you have to call it in `setUpClass()` 96 and call `del_temp_region()` in `tearDownClass()`. By this you 97 ensure that each test method will have its own region and will 98 not influence other classes. 99 100 :: 101 102 @classmethod 103 def setUpClass(self): 104 self.use_temp_region() 105 106 @classmethod 107 def tearDownClass(self): 108 self.del_temp_region() 109 110 You can also call the methods in `setUp()` and `tearDown()` if 111 you are using them. 112 113 Copies the current region to a temporary region with 114 ``g.region save=``, then sets ``WIND_OVERRIDE`` to refer 115 to that region. 116 """ 117 # we use just the class name since we rely on the invocation system 118 # where each test file is separate process and nothing runs 119 # in parallel inside 120 name = "tmp.%s" % (cls.__name__) 121 call_module("g.region", save=name, overwrite=True) 122 os.environ['WIND_OVERRIDE'] = name 123 cls._temp_region = name 124 125 @classmethod 126 def del_temp_region(cls): 127 """Remove the temporary region. 128 129 Unsets ``WIND_OVERRIDE`` and removes any region named by it. 130 """ 131 assert cls._temp_region 132 name = os.environ.pop('WIND_OVERRIDE') 133 if name != cls._temp_region: 134 # be strict about usage of region 135 raise RuntimeError("Inconsistent use of" 136 " TestCase.use_temp_region, WIND_OVERRIDE" 137 " or temporary region in general\n" 138 "Region to which should be now deleted ({n})" 139 " by TestCase class" 140 "does not correspond to currently set" 141 " WIND_OVERRIDE ({c})", 142 n=cls._temp_region, c=name) 143 call_module("g.remove", quiet=True, flags='f', type='region', name=name) 144 # TODO: we don't know if user calls this 145 # so perhaps some decorator which would use with statemet 146 # but we have zero chance of infuencing another test class 147 # since we use class-specific name for temporary region 148 149 def assertMultiLineEqual(self, first, second, msg=None): 150 r"""Test that the multiline string first is equal to the string second. 151 152 When not equal a diff of the two strings highlighting the differences 153 will be included in the error message. This method is used by default 154 when comparing strings with assertEqual(). 155 156 This method replaces platform dependent newline characters 157 by ``\n`` (LF) in both parameters. This is 158 different from the same method implemented in Python ``unittest`` 159 package which preserves the original newline characters. 160 161 This function removes the burden of getting the newline characters 162 right on each platform. You can just use ``\n`` everywhere and this 163 function will ensure that it does not matter if for example, 164 a module generates (as expected) ``\r\n`` (CRLF) newline characters 165 on MS Windows. 166 167 .. warning:: 168 If you need to test the actual newline characters, use the standard 169 string comparison and functions such as ``find()``. 170 """ 171 if os.linesep != '\n': 172 if os.linesep in first: 173 first = first.replace(os.linesep, '\n') 174 if os.linesep in second: 175 second = second.replace(os.linesep, '\n') 176 return super(TestCase, self).assertMultiLineEqual( 177 first=first, second=second, msg=msg) 178 179 def assertLooksLike(self, actual, reference, msg=None): 180 r"""Test that ``actual`` text is the same as ``reference`` with ellipses. 181 182 If ``actual`` contains platform dependent newline characters, 183 these will replaced by ``\n`` which is expected to be in the test data. 184 185 See :func:`check_text_ellipsis` for details of behavior. 186 """ 187 self.assertTrue(isinstance(actual, (str, unicode)), ( 188 'actual argument is not a string')) 189 self.assertTrue(isinstance(reference, (str, unicode)), ( 190 'reference argument is not a string')) 191 if os.linesep != '\n' and os.linesep in actual: 192 actual = actual.replace(os.linesep, '\n') 193 if not check_text_ellipsis(actual=actual, reference=reference): 194 # TODO: add support for multiline (first line general, others with details) 195 standardMsg = '"%s" does not correspond with "%s"' % (actual, 196 reference) 197 self.fail(self._formatMessage(msg, standardMsg)) 198 199 # TODO: decide if precision is mandatory 200 # (note that we don't need precision for strings and usually for integers) 201 # TODO: auto-determine precision based on the map type 202 # TODO: we can have also more general function without the subset reference 203 # TODO: change name to Module 204 def assertModuleKeyValue(self, module, reference, sep, 205 precision, msg=None, **parameters): 206 """Test that output of a module is the same as provided subset. 207 208 :: 209 210 self.assertModuleKeyValue('r.info', map='elevation', flags='gr', 211 reference=dict(min=55.58, max=156.33), 212 precision=0.01, sep='=') 213 214 :: 215 216 module = SimpleModule('r.info', map='elevation', flags='gr') 217 self.assertModuleKeyValue(module, 218 reference=dict(min=55.58, max=156.33), 219 precision=0.01, sep='=') 220 221 The output of the module should be key-value pairs (shell script style) 222 which is typically obtained using ``-g`` flag. 223 """ 224 if isinstance(reference, str): 225 reference = text_to_keyvalue(reference, sep=sep, skip_empty=True) 226 module = _module_from_parameters(module, **parameters) 227 self.runModule(module, expecting_stdout=True) 228 raster_univar = text_to_keyvalue(module.outputs.stdout, 229 sep=sep, skip_empty=True) 230 if not keyvalue_equals(dict_a=reference, dict_b=raster_univar, 231 a_is_subset=True, precision=precision): 232 unused, missing, mismatch = diff_keyvalue(dict_a=reference, 233 dict_b=raster_univar, 234 a_is_subset=True, 235 precision=precision) 236 # TODO: add region vs map extent and res check in case of error 237 if missing: 238 raise ValueError("%s output does not contain" 239 " the following keys" 240 " provided in reference" 241 ": %s\n" % (module, ", ".join(missing))) 242 if mismatch: 243 stdMsg = "%s difference:\n" % module 244 stdMsg += "mismatch values" 245 stdMsg += " (key, reference, actual): %s\n" % mismatch 246 stdMsg += 'command: %s %s' % (module, parameters) 247 else: 248 # we can probably remove this once we have more tests 249 # of keyvalue_equals and diff_keyvalue against each other 250 raise RuntimeError("keyvalue_equals() showed difference but" 251 " diff_keyvalue() did not. This can be" 252 " a bug in one of them or in the caller" 253 " (assertModuleKeyValue())") 254 self.fail(self._formatMessage(msg, stdMsg)) 255 256 def assertRasterFitsUnivar(self, raster, reference, 257 precision=None, msg=None): 258 r"""Test that raster map has the values obtained by r.univar module. 259 260 The function does not require all values from r.univar. 261 Only the provided values are tested. 262 Typical example is checking minimum, maximum and number of NULL cells 263 in the map:: 264 265 values = 'null_cells=0\nmin=55.5787925720215\nmax=156.329864501953' 266 self.assertRasterFitsUnivar(raster='elevation', reference=values) 267 268 Use keyword arguments syntax for all function parameters. 269 270 Does not -e (extended statistics) flag, use `assertModuleKeyValue()` 271 for the full interface of arbitrary module. 272 """ 273 self.assertModuleKeyValue(module='r.univar', 274 map=raster, 275 separator='=', 276 flags='g', 277 reference=reference, msg=msg, sep='=', 278 precision=precision) 279 280 def assertRasterFitsInfo(self, raster, reference, 281 precision=None, msg=None): 282 r"""Test that raster map has the values obtained by r.univar module. 283 284 The function does not require all values from r.univar. 285 Only the provided values are tested. 286 Typical example is checking minimum, maximum and type of the map:: 287 288 minmax = 'min=0\nmax=1451\ndatatype=FCELL' 289 self.assertRasterFitsInfo(raster='elevation', reference=minmax) 290 291 Use keyword arguments syntax for all function parameters. 292 293 This function supports values obtained -r (range) and 294 -e (extended metadata) flags. 295 """ 296 self.assertModuleKeyValue(module='r.info', 297 map=raster, flags='gre', 298 reference=reference, msg=msg, sep='=', 299 precision=precision) 300 301 def assertRaster3dFitsUnivar(self, raster, reference, 302 precision=None, msg=None): 303 r"""Test that 3D raster map has the values obtained by r3.univar module. 304 305 The function does not require all values from r3.univar. 306 Only the provided values are tested. 307 308 Use keyword arguments syntax for all function parameters. 309 310 Function does not use -e (extended statistics) flag, 311 use `assertModuleKeyValue()` for the full interface of arbitrary 312 module. 313 """ 314 self.assertModuleKeyValue(module='r3.univar', 315 map=raster, 316 separator='=', 317 flags='g', 318 reference=reference, msg=msg, sep='=', 319 precision=precision) 320 321 def assertRaster3dFitsInfo(self, raster, reference, 322 precision=None, msg=None): 323 r"""Test that raster map has the values obtained by r3.info module. 324 325 The function does not require all values from r3.info. 326 Only the provided values are tested. 327 328 Use keyword arguments syntax for all function parameters. 329 330 This function supports values obtained by -g (info) and -r (range). 331 """ 332 self.assertModuleKeyValue(module='r3.info', 333 map=raster, flags='gr', 334 reference=reference, msg=msg, sep='=', 335 precision=precision) 336 337 def assertVectorFitsTopoInfo(self, vector, reference, msg=None): 338 r"""Test that raster map has the values obtained by ``v.info`` module. 339 340 This function uses ``-t`` flag of ``v.info`` module to get topology 341 info, so the reference dictionary should contain appropriate set or 342 subset of values (only the provided values are tested). 343 344 A example of checking number of points:: 345 346 topology = dict(points=10938, primitives=10938) 347 self.assertVectorFitsTopoInfo(vector='bridges', reference=topology) 348 349 Note that here we are checking also the number of primitives to prove 350 that there are no other features besides points. 351 352 No precision is applied (no difference is required). So, this function 353 is not suitable for testing items which are floating point number 354 (no such items are currently in topological information). 355 356 Use keyword arguments syntax for all function parameters. 357 """ 358 self.assertModuleKeyValue(module='v.info', 359 map=vector, flags='t', 360 reference=reference, msg=msg, sep='=', 361 precision=0) 362 363 def assertVectorFitsRegionInfo(self, vector, reference, 364 precision, msg=None): 365 r"""Test that raster map has the values obtained by ``v.info`` module. 366 367 This function uses ``-g`` flag of ``v.info`` module to get topology 368 info, so the reference dictionary should contain appropriate set or 369 subset of values (only the provided values are tested). 370 371 Use keyword arguments syntax for all function parameters. 372 """ 373 self.assertModuleKeyValue(module='v.info', 374 map=vector, flags='g', 375 reference=reference, msg=msg, sep='=', 376 precision=precision) 377 378 def assertVectorFitsExtendedInfo(self, vector, reference, msg=None): 379 r"""Test that raster map has the values obtained by ``v.info`` module. 380 381 This function uses ``-e`` flag of ``v.info`` module to get topology 382 info, so the reference dictionary should contain appropriate set or 383 subset of values (only the provided values are tested). 384 385 The most useful items for testing (considering circumstances of test 386 invocation) are name, title, level and num_dblinks. (When testing 387 storing of ``v.info -e`` metadata, the selection might be different.) 388 389 No precision is applied (no difference is required). So, this function 390 is not suitable for testing items which are floating point number. 391 392 Use keyword arguments syntax for all function parameters. 393 """ 394 self.assertModuleKeyValue(module='v.info', 395 map=vector, flags='e', 396 reference=reference, msg=msg, sep='=', 397 precision=0) 398 399 def assertVectorInfoEqualsVectorInfo(self, actual, reference, precision, 400 msg=None): 401 """Test that two vectors are equal according to ``v.info -tg``. 402 403 This function does not test geometry itself just the region of the 404 vector map and number of features. 405 """ 406 module = SimpleModule('v.info', flags='t', map=reference) 407 self.runModule(module) 408 ref_topo = text_to_keyvalue(module.outputs.stdout, sep='=') 409 module = SimpleModule('v.info', flags='g', map=reference) 410 self.runModule(module) 411 ref_info = text_to_keyvalue(module.outputs.stdout, sep='=') 412 self.assertVectorFitsTopoInfo(vector=actual, reference=ref_topo, 413 msg=msg) 414 self.assertVectorFitsRegionInfo(vector=actual, reference=ref_info, 415 precision=precision, msg=msg) 416 417 def assertVectorFitsUnivar(self, map, column, reference, msg=None, 418 layer=None, type=None, where=None, 419 precision=None): 420 r"""Test that vector map has the values obtained by v.univar module. 421 422 The function does not require all values from v.univar. 423 Only the provided values are tested. 424 Typical example is checking minimum and maximum of a column:: 425 426 minmax = 'min=0\nmax=1451' 427 self.assertVectorFitsUnivar(map='bridges', column='WIDTH', 428 reference=minmax) 429 430 Use keyword arguments syntax for all function parameters. 431 432 Does not support -d (geometry distances) flag, -e (extended statistics) 433 flag and few other, use `assertModuleKeyValue` for the full interface 434 of arbitrary module. 435 """ 436 parameters = dict(map=map, column=column, flags='g') 437 if layer: 438 parameters.update(layer=layer) 439 if type: 440 parameters.update(type=type) 441 if where: 442 parameters.update(where=where) 443 self.assertModuleKeyValue(module='v.univar', 444 reference=reference, msg=msg, sep='=', 445 precision=precision, 446 **parameters) 447 448 # TODO: use precision? 449 # TODO: write a test for this method with r.in.ascii 450 def assertRasterMinMax(self, map, refmin, refmax, msg=None): 451 """Test that raster map minimum and maximum are within limits. 452 453 Map minimum and maximum is tested against expression:: 454 455 refmin <= actualmin and refmax >= actualmax 456 457 Use keyword arguments syntax for all function parameters. 458 459 To check that more statistics have certain values use 460 `assertRasterFitsUnivar()` or `assertRasterFitsInfo()` 461 """ 462 stdout = call_module('r.info', map=map, flags='r') 463 actual = text_to_keyvalue(stdout, sep='=') 464 if refmin > actual['min']: 465 stdmsg = ('The actual minimum ({a}) is smaller than the reference' 466 ' one ({r}) for raster map {m}' 467 ' (with maximum {o})'.format( 468 a=actual['min'], r=refmin, m=map, o=actual['max'])) 469 self.fail(self._formatMessage(msg, stdmsg)) 470 if refmax < actual['max']: 471 stdmsg = ('The actual maximum ({a}) is greater than the reference' 472 ' one ({r}) for raster map {m}' 473 ' (with minimum {o})'.format( 474 a=actual['max'], r=refmax, m=map, o=actual['min'])) 475 self.fail(self._formatMessage(msg, stdmsg)) 476 477 # TODO: use precision? 478 # TODO: write a test for this method with r.in.ascii 479 # TODO: almost the same as 2D version 480 def assertRaster3dMinMax(self, map, refmin, refmax, msg=None): 481 """Test that 3D raster map minimum and maximum are within limits. 482 483 Map minimum and maximum is tested against expression:: 484 485 refmin <= actualmin and refmax >= actualmax 486 487 Use keyword arguments syntax for all function parameters. 488 489 To check that more statistics have certain values use 490 `assertRaster3DFitsUnivar()` or `assertRaster3DFitsInfo()` 491 """ 492 stdout = call_module('r3.info', map=map, flags='r') 493 actual = text_to_keyvalue(stdout, sep='=') 494 if refmin > actual['min']: 495 stdmsg = ('The actual minimum ({a}) is smaller than the reference' 496 ' one ({r}) for 3D raster map {m}' 497 ' (with maximum {o})'.format( 498 a=actual['min'], r=refmin, m=map, o=actual['max'])) 499 self.fail(self._formatMessage(msg, stdmsg)) 500 if refmax < actual['max']: 501 stdmsg = ('The actual maximum ({a}) is greater than the reference' 502 ' one ({r}) for 3D raster map {m}' 503 ' (with minimum {o})'.format( 504 a=actual['max'], r=refmax, m=map, o=actual['min'])) 505 self.fail(self._formatMessage(msg, stdmsg)) 506 507 def _get_detailed_message_about_no_map(self, name, type): 508 msg = ("There is no map <{n}> of type <{t}>" 509 " in the current mapset".format(n=name, t=type)) 510 related = call_module('g.list', type='raster,raster3d,vector', 511 flags='imt', pattern='*' + name + '*') 512 if related: 513 msg += "\nSee available maps:\n" 514 msg += related 515 else: 516 msg += "\nAnd there are no maps containing the name anywhere\n" 517 return msg 518 519 def assertRasterExists(self, name, msg=None): 520 """Checks if the raster map exists in current mapset""" 521 if not is_map_in_mapset(name, type='raster'): 522 stdmsg = self._get_detailed_message_about_no_map(name, 'raster') 523 self.fail(self._formatMessage(msg, stdmsg)) 524 525 def assertRasterDoesNotExist(self, name, msg=None): 526 """Checks if the raster map does not exist in current mapset""" 527 if is_map_in_mapset(name, type='raster'): 528 stdmsg = self._get_detailed_message_about_no_map(name, 'raster') 529 self.fail(self._formatMessage(msg, stdmsg)) 530 531 def assertRaster3dExists(self, name, msg=None): 532 """Checks if the 3D raster map exists in current mapset""" 533 if not is_map_in_mapset(name, type='raster3d'): 534 stdmsg = self._get_detailed_message_about_no_map(name, 'raster3d') 535 self.fail(self._formatMessage(msg, stdmsg)) 536 537 def assertRaster3dDoesNotExist(self, name, msg=None): 538 """Checks if the 3D raster map does not exist in current mapset""" 539 if is_map_in_mapset(name, type='raster3d'): 540 stdmsg = self._get_detailed_message_about_no_map(name, 'raster3d') 541 self.fail(self._formatMessage(msg, stdmsg)) 542 543 def assertVectorExists(self, name, msg=None): 544 """Checks if the vector map exists in current mapset""" 545 if not is_map_in_mapset(name, type='vector'): 546 stdmsg = self._get_detailed_message_about_no_map(name, 'vector') 547 self.fail(self._formatMessage(msg, stdmsg)) 548 549 def assertVectorDoesNotExist(self, name, msg=None): 550 """Checks if the vector map does not exist in current mapset""" 551 if is_map_in_mapset(name, type='vector'): 552 stdmsg = self._get_detailed_message_about_no_map(name, 'vector') 553 self.fail(self._formatMessage(msg, stdmsg)) 554 555 def assertFileExists(self, filename, msg=None, 556 skip_size_check=False, skip_access_check=False): 557 """Test the existence of a file. 558 559 .. note: 560 By default this also checks if the file size is greater than 0 561 since we rarely want a file to be empty. It also checks 562 if the file is accessible for reading since we expect that user 563 wants to look at created files. 564 """ 565 if not os.path.isfile(filename): 566 stdmsg = 'File %s does not exist' % filename 567 self.fail(self._formatMessage(msg, stdmsg)) 568 if not skip_size_check and not os.path.getsize(filename): 569 stdmsg = 'File %s is empty' % filename 570 self.fail(self._formatMessage(msg, stdmsg)) 571 if not skip_access_check and not os.access(filename, os.R_OK): 572 stdmsg = 'File %s is not accessible for reading' % filename 573 self.fail(self._formatMessage(msg, stdmsg)) 574 575 def assertFileMd5(self, filename, md5, text=False, msg=None): 576 r"""Test that file MD5 sum is equal to the provided sum. 577 578 Usually, this function is used to test binary files or large text files 579 which cannot be tested in some other way. Text files can be usually 580 tested by some finer method. 581 582 To test text files with this function, you should always use parameter 583 *text* set to ``True``. Note that function ``checkers.text_file_md5()`` 584 offers additional parameters which might be advantageous when testing 585 text files. 586 587 The typical workflow is that you create a file in a way you 588 trust (that you obtain the right file). Then you compute MD5 589 sum of the file. And provide the sum in a test as a string:: 590 591 self.assertFileMd5('result.png', md5='807bba4ffa...') 592 593 Use `file_md5()` function from this package:: 594 595 file_md5('original_result.png') 596 597 Or in command line, use ``md5sum`` command if available: 598 599 .. code-block:: sh 600 601 md5sum some_file.png 602 603 Finally, you can use Python ``hashlib`` to obtain MD5:: 604 605 import hashlib 606 hasher = hashlib.md5() 607 # expecting the file to fit into memory 608 hasher.update(open('original_result.png', 'rb').read()) 609 hasher.hexdigest() 610 611 .. note: 612 For text files, always create MD5 sum using ``\n`` (LF) 613 as newline characters for consistency. Also use newline 614 at the end of file (as for example, Git or PEP8 requires). 615 """ 616 self.assertFileExists(filename, msg=msg) 617 if text: 618 actual = text_file_md5(filename) 619 else: 620 actual = file_md5(filename) 621 if not actual == md5: 622 standardMsg = ('File <{name}> does not have the right MD5 sum.\n' 623 'Expected is <{expected}>,' 624 ' actual is <{actual}>'.format( 625 name=filename, expected=md5, actual=actual)) 626 self.fail(self._formatMessage(msg, standardMsg)) 627 628 def assertFilesEqualMd5(self, filename, reference, msg=None): 629 """Test that files are the same using MD5 sum. 630 631 This functions requires you to provide a file to test and 632 a reference file. For both, MD5 sum will be computed and compared with 633 each other. 634 """ 635 self.assertFileExists(filename, msg=msg) 636 # nothing for ref, missing ref_filename is an error not a test failure 637 if not files_equal_md5(filename, reference): 638 stdmsg = 'Files %s and %s don\'t have the same MD5 sums' % (filename, 639 reference) 640 self.fail(self._formatMessage(msg, stdmsg)) 641 642 def _get_unique_name(self, name): 643 """Create standardized map or file name which is unique 644 645 If ``readable_names`` attribute is `True`, it uses the *name* string 646 to create the unique name. Otherwise, it creates a unique name. 647 Even if you expect ``readable_names`` to be `True`, provide *name* 648 which is unique 649 650 The *name* parameter should be valid raster name, vector name and file 651 name and should be always provided. 652 """ 653 # TODO: possible improvement is to require some descriptive name 654 # and ensure uniqueness by add UUID 655 if self.readable_names: 656 return 'tmp_' + self.id().replace('.', '_') + '_' + name 657 else: 658 # UUID might be overkill (and expensive) but it's safe and simple 659 # alternative is to create hash from the readable name 660 return 'tmp_' + str(uuid.uuid4()).replace('-', '') 661 662 def _compute_difference_raster(self, first, second, name_part): 663 """Compute difference of two rasters (first - second) 664 665 The name of the new raster is a long name designed to be as unique as 666 possible and contains names of two input rasters. 667 668 :param first: raster to subtract from 669 :param second: raster used as decrement 670 :param name_part: a unique string to be used in the difference name 671 672 :returns: name of a new raster 673 """ 674 diff = self._get_unique_name('compute_difference_raster_' + name_part 675 + '_' + first + '_minus_' + second) 676 expression = '"{diff}" = "{first}" - "{second}"'.format( 677 diff=diff, 678 first=first, 679 second=second 680 ) 681 call_module('r.mapcalc', stdin=expression.encode("utf-8")) 682 return diff 683 684 # TODO: name of map generation is repeated three times 685 # TODO: this method is almost the same as the one for 2D 686 def _compute_difference_raster3d(self, first, second, name_part): 687 """Compute difference of two rasters (first - second) 688 689 The name of the new raster is a long name designed to be as unique as 690 possible and contains names of two input rasters. 691 692 :param first: raster to subtract from 693 :param second: raster used as decrement 694 :param name_part: a unique string to be used in the difference name 695 696 :returns: name of a new raster 697 """ 698 diff = self._get_unique_name('compute_difference_raster_' + name_part 699 + '_' + first + '_minus_' + second) 700 701 call_module('r3.mapcalc', 702 stdin='"{d}" = "{f}" - "{s}"'.format(d=diff, 703 f=first, 704 s=second)) 705 return diff 706 707 def _compute_vector_xor(self, ainput, alayer, binput, blayer, name_part): 708 """Compute symmetric difference (xor) of two vectors 709 710 :returns: name of a new vector 711 """ 712 diff = self._get_unique_name('compute_difference_vector_' + name_part 713 + '_' + ainput + '_' + alayer + '_minus_' 714 + binput + '_' + blayer) 715 call_module('v.overlay', operator='xor', ainput=ainput, binput=binput, 716 alayer=alayer, blayer=blayer, 717 output=diff, atype='area', btype='area', olayer='') 718 # trying to avoid long reports full of categories by olayer='' 719 # olayer Output layer for new category, ainput and binput 720 # If 0 or not given, the category is not written 721 return diff 722 723 # TODO: -z and 3D support 724 def _import_ascii_vector(self, filename, name_part): 725 """Import a vector stored in GRASS vector ASCII format. 726 727 :returns: name of a new vector 728 """ 729 # hash is the easiest way how to get a valid vector name 730 # TODO: introduce some function which will make file valid 731 hasher = hashlib.md5() 732 hasher.update(encode(filename)) 733 namehash = hasher.hexdigest() 734 vector = self._get_unique_name('import_ascii_vector_' + name_part 735 + '_' + namehash) 736 call_module('v.in.ascii', input=filename, 737 output=vector, format='standard') 738 return vector 739 740 # TODO: -z and 3D support 741 def _export_ascii_vector(self, vector, name_part, digits): 742 """Import a vector stored in GRASS vector ASCII format. 743 744 :returns: name of a new vector 745 """ 746 # TODO: perhaps we can afford just simple file name 747 filename = self._get_unique_name('export_ascii_vector_' 748 + name_part + '_' + vector) 749 call_module('v.out.ascii', input=vector, 750 output=filename, format='standard', layer='-1', 751 precision=digits) 752 return filename 753 754 def assertRastersNoDifference(self, actual, reference, 755 precision, statistics=None, msg=None): 756 """Test that `actual` raster is not different from `reference` raster 757 758 Method behaves in the same way as `assertRasterFitsUnivar()` 759 but works on difference ``reference - actual``. 760 If statistics is not given ``dict(min=-precision, max=precision)`` 761 is used. 762 """ 763 if statistics is None or sorted(statistics.keys()) == ['max', 'min']: 764 if statistics is None: 765 statistics = dict(min=-precision, max=precision) 766 diff = self._compute_difference_raster(reference, actual, 767 'assertRastersNoDifference') 768 try: 769 self.assertModuleKeyValue('r.info', map=diff, flags='r', 770 sep='=', precision=precision, 771 reference=statistics, msg=msg) 772 finally: 773 call_module('g.remove', flags='f', type='raster', name=diff) 774 else: 775 # general case 776 # TODO: we are using r.info min max and r.univar min max interchangeably 777 # but they might be different if region is different from map 778 # not considered as an huge issue since we expect the tested maps 779 # to match with region, however a documentation should containe a notice 780 self.assertRastersDifference(actual=actual, reference=reference, 781 statistics=statistics, 782 precision=precision, msg=msg) 783 784 def assertRastersDifference(self, actual, reference, 785 statistics, precision, msg=None): 786 """Test statistical values of difference of reference and actual rasters 787 788 For cases when you are interested in no or minimal difference, 789 use `assertRastersNoDifference()` instead. 790 791 This method should not be used to test r.mapcalc or r.univar. 792 """ 793 diff = self._compute_difference_raster(reference, actual, 794 'assertRastersDifference') 795 try: 796 self.assertRasterFitsUnivar(raster=diff, reference=statistics, 797 precision=precision, msg=msg) 798 finally: 799 call_module('g.remove', flags='f', type='raster', name=diff) 800 801 def assertRasters3dNoDifference(self, actual, reference, 802 precision, statistics=None, msg=None): 803 """Test that `actual` raster is not different from `reference` raster 804 805 Method behaves in the same way as `assertRasterFitsUnivar()` 806 but works on difference ``reference - actual``. 807 If statistics is not given ``dict(min=-precision, max=precision)`` 808 is used. 809 """ 810 if statistics is None or sorted(statistics.keys()) == ['max', 'min']: 811 if statistics is None: 812 statistics = dict(min=-precision, max=precision) 813 diff = self._compute_difference_raster3d(reference, actual, 814 'assertRasters3dNoDifference') 815 try: 816 self.assertModuleKeyValue('r3.info', map=diff, flags='r', 817 sep='=', precision=precision, 818 reference=statistics, msg=msg) 819 finally: 820 call_module('g.remove', flags='f', type='raster_3d', name=diff) 821 else: 822 # general case 823 # TODO: we are using r.info min max and r.univar min max interchangeably 824 # but they might be different if region is different from map 825 # not considered as an huge issue since we expect the tested maps 826 # to match with region, however a documentation should contain a notice 827 self.assertRasters3dDifference(actual=actual, reference=reference, 828 statistics=statistics, 829 precision=precision, msg=msg) 830 831 def assertRasters3dDifference(self, actual, reference, 832 statistics, precision, msg=None): 833 """Test statistical values of difference of reference and actual rasters 834 835 For cases when you are interested in no or minimal difference, 836 use `assertRastersNoDifference()` instead. 837 838 This method should not be used to test r3.mapcalc or r3.univar. 839 """ 840 diff = self._compute_difference_raster3d(reference, actual, 841 'assertRasters3dDifference') 842 try: 843 self.assertRaster3dFitsUnivar(raster=diff, reference=statistics, 844 precision=precision, msg=msg) 845 finally: 846 call_module('g.remove', flags='f', type='raster_3d', name=diff) 847 848 # TODO: this works only in 2D 849 # TODO: write tests 850 def assertVectorIsVectorBuffered(self, actual, reference, precision, msg=None): 851 """ 852 853 This method should not be used to test v.buffer, v.overlay or v.select. 854 """ 855 # TODO: if msg is None: add info specific to this function 856 layer = '-1' 857 self.assertVectorInfoEqualsVectorInfo(actual=actual, 858 reference=reference, 859 precision=precision, msg=msg) 860 remove = [] 861 buffered = reference + '_buffered' # TODO: more unique name 862 intersection = reference + '_intersection' # TODO: more unique name 863 self.runModule('v.buffer', input=reference, layer=layer, 864 output=buffered, distance=precision) 865 remove.append(buffered) 866 try: 867 self.runModule('v.overlay', operator='and', ainput=actual, 868 binput=reference, 869 alayer=layer, blayer=layer, 870 output=intersection, atype='area', btype='area', 871 olayer='') 872 remove.append(intersection) 873 # TODO: this would use some refactoring 874 # perhaps different functions or more low level functions would 875 # be more appropriate 876 module = SimpleModule('v.info', flags='t', map=reference) 877 self.runModule(module) 878 ref_topo = text_to_keyvalue(module.outputs.stdout, sep='=') 879 self.assertVectorFitsTopoInfo(vector=intersection, 880 reference=ref_topo, 881 msg=msg) 882 module = SimpleModule('v.info', flags='g', map=reference) 883 self.runModule(module) 884 ref_info = text_to_keyvalue(module.outputs.stdout, sep='=') 885 self.assertVectorFitsRegionInfo(vector=intersection, 886 reference=ref_info, 887 msg=msg, precision=precision) 888 finally: 889 call_module('g.remove', flags='f', type='vector', name=remove) 890 891 # TODO: write tests 892 def assertVectorsNoAreaDifference(self, actual, reference, precision, 893 layer=1, msg=None): 894 """Test statistical values of difference of reference and actual rasters 895 896 Works only for areas. 897 898 Use keyword arguments syntax for all function parameters. 899 900 This method should not be used to test v.overlay or v.select. 901 """ 902 diff = self._compute_xor_vectors(ainput=reference, binput=actual, 903 alayer=layer, blayer=layer, 904 name_part='assertVectorsNoDifference') 905 try: 906 module = SimpleModule('v.to.db', map=diff, 907 flags='pc', separator='=') 908 self.runModule(module) 909 # the output of v.to.db -pc sep== should look like: 910 # ... 911 # 43=98606087.5818323 912 # 44=727592.902311112 913 # total area=2219442027.22035 914 total_area = module.outputs.stdout.splitlines()[-1].split('=')[-1] 915 if total_area > precision: 916 stdmsg = ("Area of difference of vectors <{va}> and <{vr}>" 917 " should be 0" 918 " in the given precision ({p}) not {a}").format( 919 va=actual, vr=reference, p=precision, a=total_area) 920 self.fail(self._formatMessage(msg, stdmsg)) 921 finally: 922 call_module('g.remove', flags='f', type='vector', name=diff) 923 924 # TODO: here we have to have significant digits which is not consistent 925 # TODO: documentation for all new asserts 926 # TODO: same can be created for raster and 3D raster 927 def assertVectorEqualsVector(self, actual, reference, digits, precision, msg=None): 928 """Test that two vectors are equal. 929 930 .. note: 931 This test should not be used to test ``v.in.ascii`` and 932 ``v.out.ascii`` modules. 933 934 .. warning: 935 ASCII files for vectors are loaded into memory, so this 936 function works well only for "not too big" vector maps. 937 """ 938 # both vectors to ascii 939 # text diff of two ascii files 940 # may also do other comparisons on vectors themselves (asserts) 941 self.assertVectorInfoEqualsVectorInfo(actual=actual, reference=reference, precision=precision, msg=msg) 942 factual = self._export_ascii_vector(vector=actual, 943 name_part='assertVectorEqualsVector_actual', 944 digits=digits) 945 freference = self._export_ascii_vector(vector=reference, 946 name_part='assertVectorEqualsVector_reference', 947 digits=digits) 948 self.assertVectorAsciiEqualsVectorAscii(actual=factual, 949 reference=freference, 950 remove_files=True, 951 msg=msg) 952 953 def assertVectorEqualsAscii(self, actual, reference, digits, precision, msg=None): 954 """Test that vector is equal to the vector stored in GRASS ASCII file. 955 956 .. note: 957 This test should not be used to test ``v.in.ascii`` and 958 ``v.out.ascii`` modules. 959 960 .. warning: 961 ASCII files for vectors are loaded into memory, so this 962 function works well only for "not too big" vector maps. 963 """ 964 # vector to ascii 965 # text diff of two ascii files 966 # it may actually import the file and do other asserts 967 factual = self._export_ascii_vector(vector=actual, 968 name_part='assertVectorEqualsAscii_actual', 969 digits=digits) 970 vreference = None 971 try: 972 vreference = self._import_ascii_vector(filename=reference, 973 name_part='assertVectorEqualsAscii_reference') 974 self.assertVectorInfoEqualsVectorInfo(actual=actual, 975 reference=vreference, 976 precision=precision, msg=msg) 977 self.assertVectorAsciiEqualsVectorAscii(actual=factual, 978 reference=reference, 979 remove_files=False, 980 msg=msg) 981 finally: 982 # TODO: manage using cleanup settings 983 # we rely on fail method to either raise or return (soon) 984 os.remove(factual) 985 if vreference: 986 self.runModule('g.remove', flags='f', type='vector', name=vreference) 987 988 # TODO: we expect v.out.ascii to give the same order all the time, is that OK? 989 def assertVectorAsciiEqualsVectorAscii(self, actual, reference, 990 remove_files=False, msg=None): 991 """Test that two GRASS ASCII vector files are equal. 992 993 .. note: 994 This test should not be used to test ``v.in.ascii`` and 995 ``v.out.ascii`` modules. 996 997 .. warning: 998 ASCII files for vectors are loaded into memory, so this 999 function works well only for "not too big" vector maps. 1000 """ 1001 import difflib 1002 # 'U' taken from difflib documentation 1003 fromlines = open(actual, 'U').readlines() 1004 tolines = open(reference, 'U').readlines() 1005 context_lines = 3 # number of context lines 1006 # TODO: filenames are set to "actual" and "reference", isn't it too general? 1007 # it is even more useful if map names or file names are some generated 1008 # with hash or some other unreadable things 1009 # other styles of diffs are available too 1010 # but unified is a good choice if you are used to svn or git 1011 # workaround for missing -h (do not print header) flag in v.out.ascii 1012 num_lines_of_header = 10 1013 diff = difflib.unified_diff(fromlines[num_lines_of_header:], 1014 tolines[num_lines_of_header:], 1015 'reference', 'actual', n=context_lines) 1016 # TODO: this should be solved according to cleanup policy 1017 # but the parameter should be kept if it is an existing file 1018 # or using this method by itself 1019 if remove_files: 1020 os.remove(actual) 1021 os.remove(reference) 1022 stdmsg = ("There is a difference between vectors when compared as" 1023 " ASCII files.\n") 1024 1025 output = StringIO() 1026 # TODO: there is a diff size constant which we can use 1027 # we are setting it unlimited but we can just set it large 1028 maxlines = 100 1029 i = 0 1030 for line in diff: 1031 if i >= maxlines: 1032 break 1033 output.write(line) 1034 i += 1 1035 stdmsg += output.getvalue() 1036 output.close() 1037 # it seems that there is not better way of asking whether there was 1038 # a difference (always a iterator object is returned) 1039 if i > 0: 1040 # do HTML diff only if there is not too many lines 1041 # TODO: this might be tough to do with some more sophisticated way of reports 1042 if self.html_reports and i < maxlines: 1043 # TODO: this might be here and somehow stored as file or done in reporter again if right information is stored 1044 # i.e., files not deleted or the whole strings passed 1045 # alternative is make_table() which is the same but creates just a table not a whole document 1046 # TODO: all HTML files might be collected by the main reporter 1047 # TODO: standardize the format of name of HTML file 1048 # for one test id there is only one possible file of this name 1049 htmldiff_file_name = self.id() + '_ascii_diff' + '.html' 1050 self.supplementary_files.append(htmldiff_file_name) 1051 htmldiff = difflib.HtmlDiff().make_file(fromlines, tolines, 1052 'reference', 'actual', 1053 context=True, 1054 numlines=context_lines) 1055 htmldiff_file = open(htmldiff_file_name, 'w') 1056 for line in htmldiff: 1057 htmldiff_file.write(line) 1058 htmldiff_file.close() 1059 1060 self.fail(self._formatMessage(msg, stdmsg)) 1061 1062 @classmethod 1063 def runModule(cls, module, expecting_stdout=False, **kwargs): 1064 """Run PyGRASS module. 1065 1066 Runs the module and raises an exception if the module ends with 1067 non-zero return code. Usually, this is the same as testing the 1068 return code and raising exception but by using this method, 1069 you give testing framework more control over the execution, 1070 error handling and storing of output. 1071 1072 In terms of testing framework, this function causes a common error, 1073 not a test failure. 1074 1075 :raises CalledModuleError: if the module failed 1076 """ 1077 module = _module_from_parameters(module, **kwargs) 1078 _check_module_run_parameters(module) 1079 try: 1080 module.run() 1081 except CalledModuleError: 1082 # here exception raised by run() with finish_=True would be 1083 # almost enough but we want some additional info to be included 1084 # in the test report 1085 errors = module.outputs.stderr 1086 # provide diagnostic at least in English locale 1087 # TODO: standardized error code would be handy here 1088 import re 1089 if re.search('Raster map.*not found', errors, flags=re.DOTALL): 1090 errors += "\nSee available raster maps:\n" 1091 errors += call_module('g.list', type='raster') 1092 if re.search('Vector map.*not found', errors, flags=re.DOTALL): 1093 errors += "\nSee available vector maps:\n" 1094 errors += call_module('g.list', type='vector') 1095 # TODO: message format, parameters 1096 raise CalledModuleError( 1097 module.returncode, module.name, module.get_python(), errors=errors 1098 ) 1099 # TODO: use this also in assert and apply when appropriate 1100 if expecting_stdout and not module.outputs.stdout.strip(): 1101 1102 if module.outputs.stderr: 1103 errors = " The errors are:\n" + module.outputs.stderr 1104 else: 1105 errors = " There were no error messages." 1106 if module.outputs.stdout: 1107 # this is not appropriate for translation but we don't want 1108 # and don't need testing to be translated 1109 got = "only whitespace." 1110 else: 1111 got = "nothing." 1112 raise RuntimeError("Module call " + module.get_python() + 1113 " ended successfully but we were expecting" 1114 " output and got " + got + errors) 1115 # TODO: we can also comapre time to some expected but that's tricky 1116 # maybe we should measure time but the real benchmarks with stdin/stdout 1117 # should be done by some other function 1118 # TODO: this should be the function used for valgrind or profiling or debug 1119 # TODO: it asserts the rc but it does much more, so testModule? 1120 # TODO: do we need special function for testing module failures or just add parameter returncode=0? 1121 # TODO: consider not allowing to call this method more than once 1122 # the original idea was to run this method just once for test method 1123 # but for "integration" tests (script-like tests with more than one module) 1124 # it would be better to be able to use this multiple times 1125 # TODO: enable merging streams? 1126 def assertModule(self, module, msg=None, **kwargs): 1127 """Run PyGRASS module in controlled way and assert non-zero return code. 1128 1129 You should use this method to invoke module you are testing. 1130 By using this method, you give testing framework more control over 1131 the execution, error handling and storing of output. 1132 1133 It will not print module stdout and stderr, instead it will always 1134 store them for further examination. Streams are stored separately. 1135 1136 This method is not suitable for testing error states of the module. 1137 If you want to test behavior which involves non-zero return codes 1138 and examine stderr in test, use `assertModuleFail()` method. 1139 1140 Runs the module and causes test failure if module ends with 1141 non-zero return code. 1142 """ 1143 module = _module_from_parameters(module, **kwargs) 1144 _check_module_run_parameters(module) 1145 if not shutil_which(module.name): 1146 stdmsg = "Cannot find the module '{0}'".format(module.name) 1147 self.fail(self._formatMessage(msg, stdmsg)) 1148 try: 1149 module.run() 1150 self.grass_modules.append(module.name) 1151 except CalledModuleError: 1152 print(text_to_string(module.outputs.stdout)) 1153 print(text_to_string(module.outputs.stderr)) 1154 # TODO: message format 1155 # TODO: stderr? 1156 stdmsg = ( 1157 "Running <{m.name}> module ended" 1158 " with non-zero return code ({m.returncode})\n" 1159 "Called: {code}\n" 1160 "See the following errors:\n" 1161 "{errors}".format( 1162 m=module, code=module.get_python(), errors=module.outputs.stderr 1163 ) 1164 ) 1165 self.fail(self._formatMessage(msg, stdmsg)) 1166 print(text_to_string(module.outputs.stdout)) 1167 print(text_to_string(module.outputs.stderr)) 1168 # log these to final report 1169 # TODO: always or only if the calling test method failed? 1170 # in any case, this must be done before self.fail() 1171 # module.outputs['stdout'].value 1172 # module.outputs['stderr'].value 1173 1174 # TODO: should we merge stderr to stdout in this case? 1175 def assertModuleFail(self, module, msg=None, **kwargs): 1176 """Test that module fails with a non-zero return code. 1177 1178 Works like `assertModule()` but expects module to fail. 1179 """ 1180 module = _module_from_parameters(module, **kwargs) 1181 _check_module_run_parameters(module) 1182 # note that we cannot use finally because we do not leave except 1183 try: 1184 module.run() 1185 self.grass_modules.append(module.name) 1186 except CalledModuleError: 1187 print(text_to_string(module.outputs.stdout)) 1188 print(text_to_string(module.outputs.stderr)) 1189 else: 1190 print(text_to_string(module.outputs.stdout)) 1191 print(text_to_string(module.outputs.stderr)) 1192 stdmsg = ('Running <%s> ended with zero (successful) return code' 1193 ' when expecting module to fail' % module.get_python()) 1194 self.fail(self._formatMessage(msg, stdmsg)) 1195 1196 1197# TODO: add tests and documentation to methods which are using this function 1198# some test and documentation add to assertModuleKeyValue 1199def _module_from_parameters(module, **kwargs): 1200 if kwargs: 1201 if not isinstance(module, str): 1202 raise ValueError('module can be only string or PyGRASS Module') 1203 if isinstance(module, Module): 1204 raise ValueError('module can be only string if other' 1205 ' parameters are given') 1206 # allow passing all parameters in one dictionary called parameters 1207 if list(kwargs.keys()) == ['parameters']: 1208 kwargs = kwargs['parameters'] 1209 module = SimpleModule(module, **kwargs) 1210 return module 1211 1212 1213def _check_module_run_parameters(module): 1214 # in this case module already run and we would start it again 1215 if module.run_: 1216 raise ValueError('Do not run the module manually, set run_=False') 1217 if not module.finish_: 1218 raise ValueError('This function will always finish module run,' 1219 ' set finish_=None or finish_=True.') 1220 # we expect most of the usages with stdout=PIPE 1221 # TODO: in any case capture PIPE always? 1222 if module.stdout_ is None: 1223 module.stdout_ = subprocess.PIPE 1224 elif module.stdout_ != subprocess.PIPE: 1225 raise ValueError('stdout_ can be only PIPE or None') 1226 if module.stderr_ is None: 1227 module.stderr_ = subprocess.PIPE 1228 elif module.stderr_ != subprocess.PIPE: 1229 raise ValueError('stderr_ can be only PIPE or None') 1230 # because we want to capture it 1231