1# -*- coding: utf-8 -*- 2r""" 3 werkzeug.contrib.iterio 4 ~~~~~~~~~~~~~~~~~~~~~~~ 5 6 This module implements a :class:`IterIO` that converts an iterator into 7 a stream object and the other way round. Converting streams into 8 iterators requires the `greenlet`_ module. 9 10 To convert an iterator into a stream all you have to do is to pass it 11 directly to the :class:`IterIO` constructor. In this example we pass it 12 a newly created generator:: 13 14 def foo(): 15 yield "something\n" 16 yield "otherthings" 17 stream = IterIO(foo()) 18 print stream.read() # read the whole iterator 19 20 The other way round works a bit different because we have to ensure that 21 the code execution doesn't take place yet. An :class:`IterIO` call with a 22 callable as first argument does two things. The function itself is passed 23 an :class:`IterIO` stream it can feed. The object returned by the 24 :class:`IterIO` constructor on the other hand is not an stream object but 25 an iterator:: 26 27 def foo(stream): 28 stream.write("some") 29 stream.write("thing") 30 stream.flush() 31 stream.write("otherthing") 32 iterator = IterIO(foo) 33 print iterator.next() # prints something 34 print iterator.next() # prints otherthing 35 iterator.next() # raises StopIteration 36 37 .. _greenlet: https://github.com/python-greenlet/greenlet 38 39 :copyright: 2007 Pallets 40 :license: BSD-3-Clause 41""" 42import warnings 43 44from .._compat import implements_iterator 45 46try: 47 import greenlet 48except ImportError: 49 greenlet = None 50 51warnings.warn( 52 "'werkzeug.contrib.iterio' is deprecated as of version 0.15 and" 53 " will be removed in version 1.0.", 54 DeprecationWarning, 55 stacklevel=2, 56) 57 58 59def _mixed_join(iterable, sentinel): 60 """concatenate any string type in an intelligent way.""" 61 iterator = iter(iterable) 62 first_item = next(iterator, sentinel) 63 if isinstance(first_item, bytes): 64 return first_item + b"".join(iterator) 65 return first_item + u"".join(iterator) 66 67 68def _newline(reference_string): 69 if isinstance(reference_string, bytes): 70 return b"\n" 71 return u"\n" 72 73 74@implements_iterator 75class IterIO(object): 76 """Instances of this object implement an interface compatible with the 77 standard Python :class:`file` object. Streams are either read-only or 78 write-only depending on how the object is created. 79 80 If the first argument is an iterable a file like object is returned that 81 returns the contents of the iterable. In case the iterable is empty 82 read operations will return the sentinel value. 83 84 If the first argument is a callable then the stream object will be 85 created and passed to that function. The caller itself however will 86 not receive a stream but an iterable. The function will be executed 87 step by step as something iterates over the returned iterable. Each 88 call to :meth:`flush` will create an item for the iterable. If 89 :meth:`flush` is called without any writes in-between the sentinel 90 value will be yielded. 91 92 Note for Python 3: due to the incompatible interface of bytes and 93 streams you should set the sentinel value explicitly to an empty 94 bytestring (``b''``) if you are expecting to deal with bytes as 95 otherwise the end of the stream is marked with the wrong sentinel 96 value. 97 98 .. versionadded:: 0.9 99 `sentinel` parameter was added. 100 """ 101 102 def __new__(cls, obj, sentinel=""): 103 try: 104 iterator = iter(obj) 105 except TypeError: 106 return IterI(obj, sentinel) 107 return IterO(iterator, sentinel) 108 109 def __iter__(self): 110 return self 111 112 def tell(self): 113 if self.closed: 114 raise ValueError("I/O operation on closed file") 115 return self.pos 116 117 def isatty(self): 118 if self.closed: 119 raise ValueError("I/O operation on closed file") 120 return False 121 122 def seek(self, pos, mode=0): 123 if self.closed: 124 raise ValueError("I/O operation on closed file") 125 raise IOError(9, "Bad file descriptor") 126 127 def truncate(self, size=None): 128 if self.closed: 129 raise ValueError("I/O operation on closed file") 130 raise IOError(9, "Bad file descriptor") 131 132 def write(self, s): 133 if self.closed: 134 raise ValueError("I/O operation on closed file") 135 raise IOError(9, "Bad file descriptor") 136 137 def writelines(self, list): 138 if self.closed: 139 raise ValueError("I/O operation on closed file") 140 raise IOError(9, "Bad file descriptor") 141 142 def read(self, n=-1): 143 if self.closed: 144 raise ValueError("I/O operation on closed file") 145 raise IOError(9, "Bad file descriptor") 146 147 def readlines(self, sizehint=0): 148 if self.closed: 149 raise ValueError("I/O operation on closed file") 150 raise IOError(9, "Bad file descriptor") 151 152 def readline(self, length=None): 153 if self.closed: 154 raise ValueError("I/O operation on closed file") 155 raise IOError(9, "Bad file descriptor") 156 157 def flush(self): 158 if self.closed: 159 raise ValueError("I/O operation on closed file") 160 raise IOError(9, "Bad file descriptor") 161 162 def __next__(self): 163 if self.closed: 164 raise StopIteration() 165 line = self.readline() 166 if not line: 167 raise StopIteration() 168 return line 169 170 171class IterI(IterIO): 172 """Convert an stream into an iterator.""" 173 174 def __new__(cls, func, sentinel=""): 175 if greenlet is None: 176 raise RuntimeError("IterI requires greenlet support") 177 stream = object.__new__(cls) 178 stream._parent = greenlet.getcurrent() 179 stream._buffer = [] 180 stream.closed = False 181 stream.sentinel = sentinel 182 stream.pos = 0 183 184 def run(): 185 func(stream) 186 stream.close() 187 188 g = greenlet.greenlet(run, stream._parent) 189 while 1: 190 rv = g.switch() 191 if not rv: 192 return 193 yield rv[0] 194 195 def close(self): 196 if not self.closed: 197 self.closed = True 198 self._flush_impl() 199 200 def write(self, s): 201 if self.closed: 202 raise ValueError("I/O operation on closed file") 203 if s: 204 self.pos += len(s) 205 self._buffer.append(s) 206 207 def writelines(self, list): 208 for item in list: 209 self.write(item) 210 211 def flush(self): 212 if self.closed: 213 raise ValueError("I/O operation on closed file") 214 self._flush_impl() 215 216 def _flush_impl(self): 217 data = _mixed_join(self._buffer, self.sentinel) 218 self._buffer = [] 219 if not data and self.closed: 220 self._parent.switch() 221 else: 222 self._parent.switch((data,)) 223 224 225class IterO(IterIO): 226 """Iter output. Wrap an iterator and give it a stream like interface.""" 227 228 def __new__(cls, gen, sentinel=""): 229 self = object.__new__(cls) 230 self._gen = gen 231 self._buf = None 232 self.sentinel = sentinel 233 self.closed = False 234 self.pos = 0 235 return self 236 237 def __iter__(self): 238 return self 239 240 def _buf_append(self, string): 241 """Replace string directly without appending to an empty string, 242 avoiding type issues.""" 243 if not self._buf: 244 self._buf = string 245 else: 246 self._buf += string 247 248 def close(self): 249 if not self.closed: 250 self.closed = True 251 if hasattr(self._gen, "close"): 252 self._gen.close() 253 254 def seek(self, pos, mode=0): 255 if self.closed: 256 raise ValueError("I/O operation on closed file") 257 if mode == 1: 258 pos += self.pos 259 elif mode == 2: 260 self.read() 261 self.pos = min(self.pos, self.pos + pos) 262 return 263 elif mode != 0: 264 raise IOError("Invalid argument") 265 buf = [] 266 try: 267 tmp_end_pos = len(self._buf or "") 268 while pos > tmp_end_pos: 269 item = next(self._gen) 270 tmp_end_pos += len(item) 271 buf.append(item) 272 except StopIteration: 273 pass 274 if buf: 275 self._buf_append(_mixed_join(buf, self.sentinel)) 276 self.pos = max(0, pos) 277 278 def read(self, n=-1): 279 if self.closed: 280 raise ValueError("I/O operation on closed file") 281 if n < 0: 282 self._buf_append(_mixed_join(self._gen, self.sentinel)) 283 result = self._buf[self.pos :] 284 self.pos += len(result) 285 return result 286 new_pos = self.pos + n 287 buf = [] 288 try: 289 tmp_end_pos = 0 if self._buf is None else len(self._buf) 290 while new_pos > tmp_end_pos or (self._buf is None and not buf): 291 item = next(self._gen) 292 tmp_end_pos += len(item) 293 buf.append(item) 294 except StopIteration: 295 pass 296 if buf: 297 self._buf_append(_mixed_join(buf, self.sentinel)) 298 299 if self._buf is None: 300 return self.sentinel 301 302 new_pos = max(0, new_pos) 303 try: 304 return self._buf[self.pos : new_pos] 305 finally: 306 self.pos = min(new_pos, len(self._buf)) 307 308 def readline(self, length=None): 309 if self.closed: 310 raise ValueError("I/O operation on closed file") 311 312 nl_pos = -1 313 if self._buf: 314 nl_pos = self._buf.find(_newline(self._buf), self.pos) 315 buf = [] 316 try: 317 if self._buf is None: 318 pos = self.pos 319 else: 320 pos = len(self._buf) 321 while nl_pos < 0: 322 item = next(self._gen) 323 local_pos = item.find(_newline(item)) 324 buf.append(item) 325 if local_pos >= 0: 326 nl_pos = pos + local_pos 327 break 328 pos += len(item) 329 except StopIteration: 330 pass 331 if buf: 332 self._buf_append(_mixed_join(buf, self.sentinel)) 333 334 if self._buf is None: 335 return self.sentinel 336 337 if nl_pos < 0: 338 new_pos = len(self._buf) 339 else: 340 new_pos = nl_pos + 1 341 if length is not None and self.pos + length < new_pos: 342 new_pos = self.pos + length 343 try: 344 return self._buf[self.pos : new_pos] 345 finally: 346 self.pos = min(new_pos, len(self._buf)) 347 348 def readlines(self, sizehint=0): 349 total = 0 350 lines = [] 351 line = self.readline() 352 while line: 353 lines.append(line) 354 total += len(line) 355 if 0 < sizehint <= total: 356 break 357 line = self.readline() 358 return lines 359