1""" 2 Base class for testing syncing algorithm 3 4 (c) 2013-2014 by Mega Limited, Wellsford, New Zealand 5 6 This file is part of the MEGA SDK - Client Access Engine. 7 8 Applications using the MEGA API must present a valid application key 9 and comply with the the rules set forth in the Terms of Service. 10 11 The MEGA SDK is distributed in the hope that it will be useful, 12 but WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 14 15 @copyright Simplified (2-clause) BSD License. 16 17 You should have received a copy of the license along with this 18 program. 19""" 20 21import os 22import random 23import string 24import shutil 25import hashlib 26import unittest 27import logging 28import platform 29import unicodedata 30import time 31 32def get_unicode_str(size=10, max_char=0xFFFF, onlyNormalized=False, includeUnexisting=False): 33 ''' 34 generates valid (for current OS) Unicode file name 35 Notice: if includeUnexisting==True, it is possible that files don't get synchronized 36 ''' 37 if platform.system() == "Windows": 38 # Unicode characters 1 through 31, as well as quote ("), less than (<), greater than (>), pipe (|), backspace (\b), null (\0) and tab (\t). 39 exclude = string.punctuation + u"\t" + u''.join([unichr(x) for x in range(0, 32)]) 40 else: 41 # I guess it mainly depends on fs type 42 #exclude = u"/" + u"." + u''.join([unichr(x) for x in range(0, 1)]) 43 exclude = u"/" + u"." + u''.join([unichr(x) for x in range(0, 32)]) 44 45 46 name = u"" 47 while len(name) < size: 48 c = unichr(random.randint(0, max_char)) 49 if c not in exclude: 50 try: 51 if not includeUnexisting: 52 unicodedata.name(c) #this will cause invalid unicode character to throw exception 53 if onlyNormalized: 54 name = name + unicodedata.normalize('NFC',c) #only normalized chars 55 else: 56 name = name + c 57 # except UnicodeDecodeError: 58 # print "UnicodeDecodeError con",c,repr(c),c.encode('utf-8') 59 # c.decode('utf-8') 60 # try: 61 # unicodedata.name(c) 62 # except: 63 # pass 64 # pass 65 except ValueError: 66 # try: 67 # unicodedata.name(c) 68# print "that one was valid!",c,repr(c) 69 # pass 70 # except: 71 # pass 72 pass 73 return name 74 75def get_exotic_str(size=10): 76 """ 77 generate string containing random combinations of % and ASCII symbols 78 """ 79 name = u"" 80 while len(name) < size: 81 num = random.randint(1, 3) 82 name = name + u"%" * num + get_random_str(2) 83 return name 84 85def get_random_str(size=10, chars=string.ascii_lowercase + string.ascii_uppercase + string.digits): 86 """ 87 return a random string 88 size: size of an output string 89 chars: characters to use 90 """ 91 return ''.join(random.choice(chars) for x in range(size)) 92 93def generate_ascii_name(first_symbol, i): 94 """ 95 generate random ASCII string 96 """ 97 strlen = random.randint(0, 20) 98 return first_symbol + get_random_str(size=strlen) + str(i) 99 100def generate_unicode_name_old(first_symbol, i): 101 """ 102 generate random UTF string 103 """ 104 strlen = random.randint(10, 30) 105 c = random.choice(['short-utf', 'utf', 'exotic']) 106 if c == 'short-utf': 107 s = get_unicode_str(strlen, 0xFF) 108 elif c == 'utf': 109 s = get_unicode_str(strlen) 110 else: 111 s = get_exotic_str(strlen) 112 #logging.debug("Creating Unicode file: %s" % (s.encode("unicode-escape"))) 113 return s 114 115cogen=0 116def generate_unicode_name(first_symbol, i): 117 """ 118 generate random UTF string 119 """ 120 strlen = random.randint(10, 30) 121 c = random.choice(['short-utf', 'utf', 'exotic']) 122 if c == 'short-utf': 123 s = get_unicode_str(strlen, 0xFF) 124 elif c == 'utf': 125 s = get_unicode_str(strlen) 126 else: 127 s = get_exotic_str(strlen) 128 #logging.debug("Creating Unicode file: %s" % (s.encode("unicode-escape"))) 129 global cogen 130 cogen=cogen+1 131 return str(cogen)+"_"+s 132 133def normalizeandescape(name): 134 name=escapefsincompatible(name) 135 name=unicodedata.normalize('NFC',unicode(name)) 136 #name=unicodedata.normalize('NFC',name) 137 #name=escapefsincompatible(name) 138 return name 139 140def escapefsincompatible(name): 141 """ 142 Escape file system incompatible characters 143 """ 144 import urllib 145 for i in "\\/:?\"<>|*": 146 name=name.replace(i,urllib.quote(i).lower()) 147 return name 148 149class SyncTestBase(unittest.TestCase): 150 """ 151 Base class with MEGA SDK test helper methods 152 """ 153 def __init__(self, methodName, app): 154 """ 155 local_mount_in: local upsync folder 156 local_mount_out: local downsync folder 157 work_folder: a temporary out of sync folder 158 """ 159 super(SyncTestBase, self).__init__(methodName) 160 161 self.app = app 162 163 self.nr_retries = 200 164 self.nr_files = 10 165 self.nr_dirs = 10 166 self.nr_time_changes = 10 167 self.nr_changes = 10 168 self.local_obj_nr = 5 169 self.force_syncing = False 170 171 def check_empty(self, folder_name): 172 """ 173 return True if folder is empty 174 """ 175 logging.debug("Checking if folder %s is empty" % folder_name) 176 177 # leave old files 178 if not self.app.delete_tmp_files: 179 return True 180 181 for r in range(0, self.nr_retries): 182 self.app.attempt=r 183 try: 184 res = not os.listdir(folder_name) 185 except OSError, e: 186 logging.error("Failed to list dir: %s (%s)" % (folder_name, e)) 187 return False 188 189 if res: 190 return True 191 192 logging.debug("Directory %s is not empty! Retrying [%d/%d] .." % (folder_name, r + 1, self.nr_retries)) 193 self.app.sync() 194 #~ try: 195 #~ shutil.rmtree(folder_name) 196 #~ except OSError, e: 197 #~ logging.error("Failed to delete folder: %s (%s)" % (folder_name, e)) 198 #~ return False 199 200 @staticmethod 201 def md5_for_file(fname, block_size=2**20): 202 """ 203 calculates md5 of a file 204 """ 205 fout = open(fname, 'r') 206 md5 = hashlib.md5() 207 while True: 208 data = fout.read(block_size) 209 if not data: 210 break 211 md5.update(data) 212 fout.close() 213 return md5.hexdigest() 214 215 @staticmethod 216 def touch(path): 217 """ 218 create an empty file 219 update utime 220 """ 221 with open(path, 'a'): 222 os.utime(path, None) 223 224 @staticmethod 225 def file_create(fname, fsize): 226 """ 227 create a file of a size fsize and fill with a random data 228 """ 229 fout = open(fname, 'wb') 230 fout.write(get_random_str(fsize)) 231 fout.close() 232 233 def files_create_size(self, first_symbol, maxsize, nr_files, dname, file_generate_name_func, l_files): 234 """ 235 create a list of files of a specific size 236 """ 237 for i in range(nr_files): 238 fname = file_generate_name_func(first_symbol, i) 239 ffname = os.path.join(dname, fname) 240 if maxsize == 0: 241 fsize = 0 242 else: 243 fsize = random.randint(1, maxsize) 244 245 try: 246 self.file_create(ffname, fsize) 247 except IOError, e: 248 logging.error("Failed to create file: %s (%s)" % (ffname, e)) 249 return False 250 except UnicodeEncodeError, e: 251 logging.debug("Discarded filename due to UnicodeEncodeError: %s" % (ffname)) 252 i=i-1 253 continue 254 255 md5_str = self.md5_for_file(ffname) 256 l_files.append({"name":fname, "size":fsize, "md5":md5_str, "name_orig":fname}) 257 logging.debug("File created: %s [%s, %db]" % (ffname, md5_str, fsize)) 258 return True 259 260 def files_create(self, file_generate_name_func=generate_ascii_name): 261 """ 262 create files in "in" instance and check files presence in "out" instance 263 Return list of files 264 """ 265 logging.debug("Creating files.. (nrfiles="+str(self.nr_files)+")") 266 267 l_files = [] 268 269 # empty files 270 res = self.files_create_size("e", 0, self.nr_files, self.app.local_folder_in, file_generate_name_func, l_files) 271 if not res: 272 return None 273 274 # small files < 1k 275 if not hasattr(self.app, 'only_empty_files') or not self.app.only_empty_files: 276 res = self.files_create_size("s", 1024, self.nr_files, self.app.local_folder_in, file_generate_name_func, l_files) 277 if not res: 278 return None 279 280 if self.app.use_large_files: 281 # medium files < 1mb 282 res = self.files_create_size("m", 1024*1024, self.nr_files, self.app.local_folder_in, file_generate_name_func, l_files) 283 if not res: 284 return None 285 286 # large files < 10mb 287 res = self.files_create_size("l", 10*1024*1024, self.nr_files, self.app.local_folder_in, file_generate_name_func, l_files) 288 if not res: 289 return None 290 291 # randomize list 292 random.shuffle(l_files) 293 294 return l_files 295 296 def files_check(self, l_files, dir_name=""): 297 """ 298 check files on both folders 299 compare names, size, md5 sums 300 """ 301 logging.debug("Checking files..") 302 303 # check files 304 for f in l_files: 305 dd_out = os.path.join(self.app.local_folder_out, dir_name) 306 ffname = os.path.join(dd_out, f["name"]) 307 #when saving mega alters some characters (we will look for 308 #destiny file having that in mind) 309 ffname=normalizeandescape(ffname) 310 311 312 dd_in = os.path.join(self.app.local_folder_in, dir_name) 313 ffname_in = os.path.join(dd_in, f["name"]) 314 315 success = False 316 317 logging.debug("Comparing %s and %s" % (ffname_in, ffname)) 318 # try to access the file 319 for r in range(0, self.nr_retries): 320 self.app.attempt=r 321 try: 322 with open(ffname): 323 pass 324 success = True 325 break 326 except IOError as ex: 327 # wait for a file 328 logging.debug("File %s not found! Retrying [%d/%d] .." % (ffname, r + 1, self.nr_retries)) 329 logging.debug("%s" % (ffname.encode("unicode-escape"))) 330 self.app.sync() 331 if success is False: 332 logging.error("Failed to compare files: %s and %s" % (ffname_in, ffname)) 333 return False 334 # get md5 of synced file 335 md5_str = self.md5_for_file(ffname) 336 if md5_str != f["md5"]: 337 logging.error("MD5 sums don't match for file: %s" % ffname) 338 return False 339 return True 340 341 def dir_create(self, dname, files_num, files_maxsize, file_generate_name_func=generate_ascii_name): 342 """ 343 create and fill directory with files 344 return files list 345 """ 346 try: 347 os.makedirs(dname) 348 except OSError, e: 349 logging.error("Failed to create directory: %s (%s)" % (dname, e)) 350 return None 351 352 l_files = [] 353 res = self.files_create_size("s", files_maxsize, files_num, dname, file_generate_name_func, l_files) 354 if not res: 355 return None 356 357 return l_files 358 359 def dir_create_size(self, symbol, dirs_num, files_num, files_maxsize, parent_dir, dir_generate_name_func, l_dirs): 360 """ 361 create dirs_num directories with directories 362 """ 363 for i in range(dirs_num): 364 dname = dir_generate_name_func(symbol, i) 365 ddname = os.path.join(parent_dir, dname) 366 l_files = self.dir_create(ddname, files_num, files_maxsize, dir_generate_name_func) 367 if l_files is None: 368 logging.error("Failed to create directory: %s" % ddname) 369 return False 370 l_dirs.append({"name":dname, "files_nr":files_num, "name_orig":dname, "l_files":l_files}) 371 logging.debug("Directory created: %s [%d files]" % (ddname, files_num)) 372 return True 373 374 def dirs_create(self, dir_generate_name_func=generate_ascii_name): 375 """ 376 create dirs 377 """ 378 logging.debug("Creating "+str(self.nr_dirs)+" directories..") 379 380 l_dirs = [] 381 382 # create empty dirs 383 res = self.dir_create_size("z", self.nr_dirs, 0, 0, self.app.local_folder_in, dir_generate_name_func, l_dirs) 384 if not res: 385 return None 386 387 # create dirs with #nr_files files 388 if not hasattr(self.app, 'only_empty_folders') or not self.app.only_empty_folders: 389 res = self.dir_create_size("d", self.nr_dirs, self.nr_files, 1024, self.app.local_folder_in, dir_generate_name_func, l_dirs) 390 if not res: 391 return None 392 393 # randomize list 394 random.shuffle(l_dirs) 395 396 return l_dirs 397 398 def dirs_check(self, l_dirs): 399 """ 400 check directories for both folders 401 """ 402 logging.debug("Checking directories..") 403 404 for d in l_dirs: 405 dname = os.path.join(self.app.local_folder_out, d["name"]) 406 ##when saving mega alters some characters (we will look for 407 #destiny file having that in mind) 408 dname=normalizeandescape(dname) 409 410 dname_in = os.path.join(self.app.local_folder_in, d["name"]) 411 success = False 412 413 logging.debug("Comparing dirs: %s and %s" % (dname_in, dname)) 414 415 # try to access the dir 416 for r in range(0, self.nr_retries): 417 self.app.attempt=r 418 try: 419 if os.path.isdir(dname): 420 success = True 421 break 422 else: 423 # wait for a dir 424 logging.debug("Directory %s not found! Retrying [%d/%d].." % (dname, r + 1, self.nr_retries)) 425 self.app.sync() 426 except OSError: 427 # wait for a dir 428 logging.debug("Directory %s not found! Retrying [%d/%d].." % (dname, r + 1, self.nr_retries)) 429 self.app.sync() 430 if success is False: 431 logging.error("Failed to access directories: %s and " % dname) 432 return False 433 434 # check files 435 res = self.files_check(d["l_files"], d["name"]) 436 if not res: 437 logging.error("Directories do not match !") 438 return False 439 440 return True 441 442 def file_rename(self, ffname_src, ffname_dst): 443 """ 444 renaming file 445 return True if renamed 446 """ 447 for r in range(0, self.nr_retries): 448 self.app.attempt=r 449 if os.path.exists(ffname_src): 450 try: 451 shutil.move(ffname_src, ffname_dst) 452 except OSError, e: 453 logging.error("Failed to rename file: %s (%s)" % (ffname_src, e)) 454 return False 455 456 if self.force_syncing: 457 self.app.sync() 458 459 # try to access both files (old and new) 460 if not os.path.exists(ffname_dst): 461 logging.debug("Failed to access a newly renamed file: %s, retrying [%d/%d].." % (ffname_dst, r + 1, self.nr_retries)) 462 continue 463 if os.path.exists(ffname_src): 464 logging.debug("Still can access an old renamed file: %s, retrying [%d/%d]" % (ffname_src, r + 1, self.nr_retries)) 465 continue 466 break 467 468 # try to access both files (old and new) 469 if not os.path.exists(ffname_dst): 470 logging.error("Failed to access a newly renamed file: %s. Aborting.." % ffname_dst) 471 return False 472 if os.path.exists(ffname_src): 473 logging.error("Still can access an old renamed file: %s. Aborting.." % ffname_src) 474 return False 475 return True 476 477 def files_rename(self, l_files, file_generate_name_func=generate_ascii_name): 478 """ 479 rename objects in "in" instance and check new files in "out" instance 480 """ 481 logging.debug("Renaming files..") 482 483 i = 0 484 for f in l_files: 485 ffname_src = os.path.join(self.app.local_folder_in, f["name"]) 486 f["name"] = file_generate_name_func("renamed_", i) 487 i = i + 1 488 ffname_dst = os.path.join(self.app.local_folder_in, f["name"]) 489 490 logging.debug("Renaming file: %s => %s" % (ffname_src, ffname_dst)) 491 492 if not self.file_rename(ffname_src, ffname_dst): 493 return False 494 495 return True 496 497 def files_moveanddelete(self, l_files, where=".", timeout=0, file_generate_name_func=generate_ascii_name): 498 """ 499 moves and deletes objects in "in" instance and check new files in "out" instance 500 """ 501 logging.debug("Move&Rename files..") 502 try: 503 os.makedirs(os.path.join(self.app.local_folder_in,where)) 504 except Exception, e: 505 logging.debug("Unable to create subfolder: %s (%s)" % (where, e)) 506 507 i = 0 508 for f in l_files: 509 ffname_src = os.path.join(self.app.local_folder_in, f["name"]) 510 f["name"] = file_generate_name_func("renamed_", i) 511 i = i + 1 512 ffname_dst = os.path.join(self.app.local_folder_in, where, f["name"]) 513 514 logging.debug("move&delete file: %s => %s" % (ffname_src, ffname_dst)) 515 516 if os.path.exists(ffname_src): 517 try: 518 shutil.move(ffname_src, ffname_dst) 519 except OSError, e: 520 logging.error("Failed to rename file: %s (%s)" % (ffname_src, e)) 521 return False 522 try: 523 time.sleep(timeout) 524 os.remove(ffname_dst) 525 except OSError, e: 526 logging.error("Failed to delete file: %s (%s)" % (ffname_dst, e)) 527 return False 528 529 if (where != "."): 530 shutil.rmtree(os.path.join(self.app.local_folder_in,where)) 531 return True 532 533 def files_mimic_update_with_backup(self, l_files, timeout=0, file_generate_name_func=generate_ascii_name): 534 """ 535 moves and deletes objects in "in" instance and check new files in "out" instance 536 """ 537 logging.debug("Mimic update with backup files..") 538 539 i = 0 540 for f in l_files: 541 ffname_src = os.path.join(self.app.local_folder_in, f["name"]) 542 #f["name"] = file_generate_name_func("renamed_", i) 543 i = i + 1 544 ffname_dst = os.path.join(self.app.local_folder_in, "renamed_"+f["name"]) 545 ffname_dst_out = os.path.join(self.app.local_folder_out, "renamed_"+f["name"]) 546 547 logging.debug("Mimic update with backup file: %s => %s" % (ffname_src, ffname_dst)) 548 549 if os.path.exists(ffname_src): 550 try: 551 shutil.move(ffname_src, ffname_dst) 552 except OSError, e: 553 logging.error("Failed to rename file: %s (%s)" % (ffname_src, e)) 554 return False 555 try: 556 time.sleep(timeout) 557 with open(ffname_dst, 'r') as f: 558 with open(ffname_src, 'w') as f2: 559 for r in range(100): 560 f2.write("whatever") 561 time.sleep(0.03) 562 if os.path.exists(ffname_dst_out): #existing temporary file 563 logging.error("ERROR in sync: Temporary file being created in syncout: : %s!" % (ffname_dst)) 564 os.remove(ffname_dst) 565 return False; 566 os.remove(ffname_dst) 567 except OSError, e: 568 logging.error("Failed to delete file: %s (%s)" % (ffname_dst, e)) 569 return False 570 571 return True 572 573 def files_remove(self, l_files): 574 """ 575 remove files in "in" instance and check files absence in "out" instance 576 """ 577 logging.debug("Removing files..") 578 579 for f in l_files: 580 ffname = os.path.join(self.app.local_folder_in, f["name"]) 581 582 logging.debug("Deleting: %s" % ffname) 583 584 for r in range(0, self.nr_retries): 585 self.app.attempt=r 586 try: 587 os.remove(ffname) 588 except OSError, e: 589 logging.error("Failed to delete file: %s (%s)" % (ffname, e)) 590 return False 591 592 if self.force_syncing: 593 self.app.sync() 594 595 # check if local file does not exist 596 if not os.path.exists(ffname): 597 break 598 logging.debug("Deleted file %s still exists, retrying [%d/%d].." % (ffname, r + 1, self.nr_retries)) 599 600 if os.path.exists(ffname): 601 logging.debug("Deleted file %s still exists, aborting.." % ffname) 602 603 success = False 604 for f in l_files: 605 ffname = os.path.join(self.app.local_folder_out, f["name"]) 606 607 for r in range(0, self.nr_retries): 608 self.app.attempt=r 609 try: 610 # file must be deleted 611 with open(ffname): 612 pass 613 logging.debug("File %s is not deleted. Retrying [%d/%d] .." % (ffname, r + 1, self.nr_retries)) 614 self.app.sync() 615 except IOError: 616 success = True 617 break 618 if success is False: 619 logging.error("Failed to delete file: %s" % ffname) 620 return False 621 return True 622 623 def dirs_rename(self, l_dirs, dir_generate_name_func=generate_ascii_name): 624 """ 625 rename directories in "in" instance and check directories new names in "out" instance 626 """ 627 logging.debug("Renaming directories..") 628 629 i = 0 630 for d in l_dirs: 631 dname_src = os.path.join(self.app.local_folder_in, d["name"]) 632 d["name"] = dir_generate_name_func("renamed_", i) 633 i = i + 1 634 dname_dst = os.path.join(self.app.local_folder_in, d["name"]) 635 try: 636 shutil.move(dname_src, dname_dst) 637 except OSError, e: 638 logging.error("Failed to rename directory: %s (%s)" % (dname_src, e)) 639 return False 640 641 if self.force_syncing: 642 self.app.sync() 643 644 # try to both dirs 645 if not os.path.exists(dname_dst): 646 logging.error("Failed to access a newly renamed directory: %s" % dname_dst) 647 return False 648 if os.path.exists(dname_src): 649 logging.error("Still can access an old directory: %s" % dname_src) 650 return False 651 652 logging.debug("Directory renamed: %s => %s" % (dname_src, dname_dst)) 653 654 return True 655 656 def dirs_remove(self, l_dirs): 657 """ 658 remove directories in "in" instance and check directories absence in "out" instance 659 """ 660 logging.debug("Removing directories..") 661 662 for d in l_dirs: 663 dname = os.path.join(self.app.local_folder_in, d["name"]) 664 try: 665 shutil.rmtree(dname) 666 except OSError, e: 667 logging.error("Failed to delete dir: %s (%s)" % (dname, e)) 668 return False 669 logging.debug("Directory removed: %s" % dname) 670 671 if self.force_syncing: 672 self.app.sync() 673 674 if os.path.exists(dname): 675 logging.error("Still can access a renamed directory: %s" % dname) 676 return False 677 678 success = False 679 for d in l_dirs: 680 dname = os.path.join(self.app.local_folder_out, d["name"]) 681 for r in range(0, self.nr_retries): 682 self.app.attempt=r 683 try: 684 # dir must be deleted 685 if not os.path.isdir(dname): 686 success = True 687 break 688 logging.debug("Directory %s is not deleted! Retrying [%d/%d] .." % (dname, r + 1, self.nr_retries)) 689 self.app.sync() 690 except OSError: 691 success = True 692 break 693 if success is False: 694 logging.error("Failed to delete dir: %s" % dname) 695 return False 696 return True 697 698 def dirs_check_empty(self): 699 """ 700 return True if both folders are empty 701 """ 702 return self.check_empty(self.app.local_folder_in) and self.check_empty(self.app.local_folder_out) 703 704 def local_tree_create_dir(self, parent_dir): 705 """ 706 generate directory 707 """ 708 strlen = random.randint(10, 20) 709 dname = get_random_str(size=strlen) 710 ddname = os.path.join(parent_dir, dname) 711 real_dname = os.path.join(self.app.local_folder_in, ddname) 712 713 try: 714 os.makedirs(real_dname) 715 except OSError, e: 716 logging.error("Failed to create directory: %s (%s)" % (real_dname, e)) 717 return None, None, None, None 718 719 # populate with random amount of files 720 obj_nr = random.randint(1, self.local_obj_nr) 721 l_files = [] 722 for _ in range(0, obj_nr): 723 strlen = random.randint(10, 20) 724 fname = get_random_str(size=strlen) + ".txt" 725 ffname = os.path.join(ddname, fname) 726 fname_real = os.path.join(self.app.local_folder_in, ffname) 727 try: 728 self.file_create(fname_real, random.randint(10, 100)) 729 except IOError, e: 730 logging.error("Failed to create file: %s (%s)" % (fname_real, e)) 731 return None, None, None, None 732 l_files.append({"name":fname, "fname":ffname}) 733 734 # populate with random amount of dirs 735 obj_nr = random.randint(1, self.local_obj_nr) 736 l_dirs = [] 737 for _ in range(0, obj_nr): 738 strlen = random.randint(10, 20) 739 cname = get_random_str(size=strlen) 740 ccname = os.path.join(ddname, cname) 741 cname_real = os.path.join(self.app.local_folder_in, ccname) 742 try: 743 os.makedirs(cname_real) 744 except OSError, e: 745 logging.error("Failed to create directory: %s (%s)" % (cname_real, e)) 746 return None, None, None, None 747 l_dirs.append({"name":cname, "fname":ccname}) 748 749 # recursively create subtree 750 return dname, ddname, l_files, l_dirs 751 752 def local_tree_create(self, parent_dir, dirs_nr): 753 """ 754 generate local directory tree, recursively populate with random number of directories / files 755 return list of dictionaries 756 """ 757 if dirs_nr == 0: 758 return None 759 760 l_tree = [] 761 762 for _ in range(0, dirs_nr): 763 dname, ddname, l_files, l_dirs = self.local_tree_create_dir(parent_dir) 764 if dname is None: 765 return None 766 l_child = self.local_tree_create(ddname, dirs_nr - 1) 767 l_tree.append({"name":dname, "fname":ddname, "files":l_files, "dirs":l_dirs, "child":l_child}) 768 return l_tree 769 770 def local_tree_get_dir(self, l_tree): 771 """ 772 directory walk generator 773 Returns relative directory path 774 """ 775 for i in l_tree: 776 yield i["fname"] 777 for dd in i["dirs"]: 778 yield dd["fname"] 779 if i["child"] is not None: 780 for x in self.local_tree_get_dir(i["child"]): 781 yield x 782 783 def local_tree_get_dirs(self, l_tree): 784 """ 785 directory walk generator 786 Returns dir dict 787 """ 788 for i in l_tree: 789 yield i 790 if i["child"] is not None: 791 for x in self.local_tree_get_dirs(i["child"]): 792 yield x 793 794 def local_tree_get_file(self, l_tree): 795 """ 796 directory walk generator 797 Returns relative file path 798 """ 799 for i in l_tree: 800 for ff in i["files"]: 801 yield ff["fname"] 802 if i["child"] is not None: 803 for x in self.local_tree_get_file(i["child"]): 804 yield x 805 806 def local_tree_compare(self, l_tree): 807 """ 808 compare two local trees 809 return True if they are the same 810 """ 811 total_dirs = total_files = 0 812 813 # try to access directories in "out" folder 814 for d in self.local_tree_get_dir(l_tree): 815 dname = os.path.join(self.app.local_folder_out, d) 816 success = False 817 total_dirs = total_dirs + 1 818 819 # logging.debug("Trying to access dir: %s" % dname) 820 for r in range(0, self.nr_retries): 821 self.app.attempt=r 822 try: 823 if os.path.isdir(dname): 824 success = True 825 break 826 else: 827 # wait for a dir 828 logging.debug("Directory %s not found! Retrying [%d/%d].." % (dname, r + 1, self.nr_retries)) 829 self.app.sync() 830 except OSError: 831 # wait for a dir 832 logging.debug("Directory %s not found! Retrying [%d/%d].." % (dname, r + 1, self.nr_retries)) 833 self.app.sync() 834 if success is False: 835 logging.error("Failed to access directories: %s and " % dname) 836 return False 837 838 # try to access files in "out" folder 839 for f in self.local_tree_get_file(l_tree): 840 fname = os.path.join(self.app.local_folder_out, f) 841 success = False 842 total_files = total_files + 1 843 844 # logging.debug("Trying to access file: %s" % fname) 845 for r in range(0, self.nr_retries): 846 self.app.attempt=r 847 try: 848 with open(fname): 849 pass 850 success = True 851 break 852 except IOError: 853 # wait for a file 854 logging.debug("File %s not found! Retrying [%d/%d] .." % (fname, r + 1, self.nr_retries)) 855 self.app.sync() 856 if success is False: 857 logging.error("Failed to access file: %s" % fname) 858 return False 859 860 logging.debug("Total dirs: %d, files: %d" % (total_dirs, total_files)) 861 return True 862 863 def local_tree_create_and_move(self, l_tree): 864 """ 865 create a folder and fill with content 866 randomly select an existing folder and create a subfolder of it 867 move the first folder to the newly created one 868 compare results, repeat 10 times 869 return True if success 870 """ 871 872 for _ in range(0, self.nr_dirs): 873 # Create a dir 874 l_dir = [] 875 dname, ddname, l_files, l_dirs = self.local_tree_create_dir("") 876 if dname is None: 877 return False 878 logging.debug("Directory created: %s" % ddname) 879 l_dir.append({"name":dname, "fname":ddname, "files":l_files, "dirs":l_dirs, "child":None}) 880 881 # wait for a sync and compare 882 if not self.local_tree_compare(l_dir): 883 return False 884 885 # select random existing folder 886 dir_dicts_l = [d for d in self.local_tree_get_dirs(l_tree)] 887 dd = random.choice(dir_dicts_l) 888 889 if dd["dirs"] is None: 890 dd["dirs"] = [] 891 892 # create a new subfolder 893 strlen = random.randint(10, 20) 894 dname = get_random_str(size=strlen) 895 ddname = os.path.join(dd["fname"], dname) 896 dname_real = os.path.join(self.app.local_folder_in, ddname) 897 logging.debug("Creating new dir: %s, parent: %s" % (ddname, dd["fname"])) 898 try: 899 os.makedirs(dname_real) 900 except OSError, e: 901 logging.error("Failed to create directory: %s (%s)" % (dname_real, e)) 902 return False 903 904 dd["dirs"].append({"name":dname, "fname":ddname, "files":None, "dirs":None, "child":None}) 905 906 # move existing folder into newly created folder 907 old_name = os.path.join(self.app.local_folder_in, l_dir[0]["fname"]) 908 new_name = os.path.join(dname_real, l_dir[0]["name"]) 909 910 logging.debug("Moving %s to %s" % (old_name, new_name)) 911 912 try: 913 shutil.move(old_name, new_name) 914 except OSError, e: 915 logging.error("Failed to move dir: %s to new: %s (%s)" % (old_name, new_name, e)) 916 return False 917 918 # fix existing dir dict 919 new_ffname = os.path.join(ddname, l_dir[0]["name"]) 920 l_dir[0]["fname"] = new_ffname 921 for f in l_dir[0]["files"]: 922 f["ffname"] = os.path.join(new_ffname, f["name"]) 923 for d in l_dir[0]["dirs"]: 924 d["ffname"] = os.path.join(new_ffname, d["name"]) 925 926 # wait for a sync and compare 927 if not self.local_tree_compare(l_tree): 928 return False 929 930 # all good ! 931 return True 932 933 def local_tree_multiple_renames(self, l_tree): 934 """ 935 perform several object renames 936 then rename back to the original name 937 """ 938 939 # rename dirs 940 for _ in range(0, self.nr_changes): 941 # select random existing folder 942 dir_dicts_l = [d for d in self.local_tree_get_dirs(l_tree)] 943 dd = random.choice(dir_dicts_l) 944 if dd["dirs"] is None: 945 continue 946 947 oname = dd["dirs"][0]["name"] 948 odname = os.path.join(dd["fname"], oname) 949 orig_name = os.path.join(self.app.local_folder_in, odname) 950 prev_name = orig_name 951 952 # rename 10 times 953 for _ in range(0, self.nr_changes): 954 strlen = random.randint(10, 20) 955 dname = get_random_str(size=strlen) 956 ddname = os.path.join(dd["fname"], dname) 957 dname_real = os.path.join(self.app.local_folder_in, ddname) 958 959 logging.debug("Renaming %s to %s" % (prev_name, dname_real)) 960 961 if not self.file_rename(prev_name, dname_real): 962 return False 963 964 prev_name = dname_real 965 966 # rename back to origin 967 logging.debug("Moving %s to %s" % (prev_name, orig_name)) 968 969 if not self.file_rename(prev_name, orig_name): 970 return False 971 972 self.app.sync() 973 974 # wait for a sync and compare 975 if not self.local_tree_compare(l_tree): 976 return False 977 978 # rename files 979 for _ in range(0, self.nr_changes): 980 # select random existing folder 981 dir_dicts_l = [d for d in self.local_tree_get_dirs(l_tree)] 982 dd = random.choice(dir_dicts_l) 983 if dd["files"] is None: 984 continue 985 986 oname = dd["files"][0]["name"] 987 odname = os.path.join(dd["fname"], oname) 988 orig_name = os.path.join(self.app.local_folder_in, odname) 989 prev_name = orig_name 990 991 # rename 10 times 992 for _ in range(0, self.nr_changes): 993 strlen = random.randint(10, 20) 994 dname = get_random_str(size=strlen) 995 ddname = os.path.join(dd["fname"], dname) 996 dname_real = os.path.join(self.app.local_folder_in, ddname) 997 998 logging.debug("Renaming %s to %s" % (prev_name, dname_real)) 999 1000 if not self.file_rename(prev_name, dname_real): 1001 return False 1002 1003 prev_name = dname_real 1004 1005 # rename back to origin 1006 logging.debug("Moving %s to %s" % (prev_name, orig_name)) 1007 1008 if not self.file_rename(prev_name, orig_name): 1009 return False 1010 1011 self.app.sync() 1012 1013 # wait for a sync and compare 1014 if not self.local_tree_compare(l_tree): 1015 return False 1016 1017 return True 1018