1""" 2 :codeauthor: Pedro Algarvio (pedro@algarvio.me) 3 4 5 salt.utils.filebuffer 6 ~~~~~~~~~~~~~~~~~~~~~ 7 8 This utility allows parsing a file in chunks. 9""" 10 11import salt.utils.files 12import salt.utils.stringutils 13from salt.exceptions import SaltException 14 15 16class InvalidFileMode(SaltException): 17 """ 18 An invalid file mode was used to open the file passed to the buffer 19 """ 20 21 22class BufferedReader: 23 """ 24 This object allows iterating through the contents of a file keeping 25 X configurable bytes in memory which can be used to, for example, 26 do regex search/matching on more than a single line. 27 28 So, **an imaginary, non accurate**, example could be: 29 30 1 - Initiate the BufferedReader filling it to max_in_men: 31 br = [1, 2, 3] 32 33 2 - next chunk(pop chunk_size from the left, append chunk_size to the 34 right): 35 br = [2, 3, 4] 36 37 38 :type path: str 39 :param path: The file path to be read 40 41 :type max_in_mem: int 42 :param max_in_mem: The maximum bytes kept in memory while iterating through 43 the file. Default 256KB. 44 45 :type chunk_size: int 46 :param chunk_size: The size of each consequent read chunk. Default 32KB. 47 48 :type mode: str 49 :param mode: The mode the file should be opened. **Only read modes**. 50 51 """ 52 53 def __init__(self, path, max_in_mem=256 * 1024, chunk_size=32 * 1024, mode="r"): 54 if "a" in mode or "w" in mode: 55 raise InvalidFileMode("Cannot open file in write or append mode") 56 self.__path = path 57 # pylint: disable=resource-leakage 58 self.__file = salt.utils.files.fopen(self.__path, mode) 59 # pylint: enable=resource-leakage 60 self.__max_in_mem = max_in_mem 61 self.__chunk_size = chunk_size 62 self.__buffered = None 63 64 # Public attributes 65 @property 66 def buffered(self): 67 return self.__buffered 68 69 # Support iteration 70 def __iter__(self): 71 return self 72 73 def next(self): 74 """ 75 Return the next iteration by popping `chunk_size` from the left and 76 appending `chunk_size` to the right if there's info on the file left 77 to be read. 78 """ 79 if self.__buffered is None: 80 # Use floor division to force multiplier to an integer 81 multiplier = self.__max_in_mem // self.__chunk_size 82 self.__buffered = "" 83 else: 84 multiplier = 1 85 self.__buffered = self.__buffered[self.__chunk_size :] 86 87 data = self.__file.read(self.__chunk_size * multiplier) 88 # Data is a byte object in Python 3 89 # Decode it in order to append to self.__buffered str later 90 # Use the salt util in case it's already a string (Windows) 91 data = salt.utils.stringutils.to_str(data) 92 93 if not data: 94 self.__file.close() 95 raise StopIteration 96 97 self.__buffered += data 98 return self.__buffered 99 100 # Alias next to __next__ for Py3 compatibility 101 __next__ = next 102 103 # Support with statements 104 def __enter__(self): 105 return self 106 107 def __exit__(self, exc_type, exc_value, traceback): 108 if self.__file.closed is False: 109 self.__file.close() 110