Mercurial > moin > 1.9
view MoinMoin/support/werkzeug/contrib/iterio.py @ 6111:1fdd537e9d83
SubProcess: reimplement exec_cmd
subclassing Popen and overriding some methods isn't pretty.
the code we have was written for py 2.4 or so and the py 2.7
Popen looked quite different.
this way with the timer should be less problematic.
author | Thomas Waldmann <tw AT waldmann-edv DOT de> |
---|---|
date | Tue, 06 Sep 2016 04:39:28 +0200 |
parents | 9f12f41504fc |
children | 7f12cf241d5e |
line wrap: on
line source
# -*- coding: utf-8 -*- r""" werkzeug.contrib.iterio ~~~~~~~~~~~~~~~~~~~~~~~ This module implements a :class:`IterIO` that converts an iterator into a stream object and the other way round. Converting streams into iterators requires the `greenlet`_ module. To convert an iterator into a stream all you have to do is to pass it directly to the :class:`IterIO` constructor. In this example we pass it a newly created generator:: def foo(): yield "something\n" yield "otherthings" stream = IterIO(foo()) print stream.read() # read the whole iterator The other way round works a bit different because we have to ensure that the code execution doesn't take place yet. An :class:`IterIO` call with a callable as first argument does two things. The function itself is passed an :class:`IterIO` stream it can feed. The object returned by the :class:`IterIO` constructor on the other hand is not an stream object but an iterator:: def foo(stream): stream.write("some") stream.write("thing") stream.flush() stream.write("otherthing") iterator = IterIO(foo) print iterator.next() # prints something print iterator.next() # prints otherthing iterator.next() # raises StopIteration .. _greenlet: http://codespeak.net/py/dist/greenlet.html :copyright: (c) 2014 by the Werkzeug Team, see AUTHORS for more details. :license: BSD, see LICENSE for more details. """ try: import greenlet except ImportError: greenlet = None from werkzeug._compat import implements_iterator def _mixed_join(iterable, sentinel): """concatenate any string type in an intelligent way.""" iterator = iter(iterable) first_item = next(iterator, sentinel) if isinstance(first_item, bytes): return first_item + b''.join(iterator) return first_item + u''.join(iterator) def _newline(reference_string): if isinstance(reference_string, bytes): return b'\n' return u'\n' @implements_iterator class IterIO(object): """Instances of this object implement an interface compatible with the standard Python :class:`file` object. Streams are either read-only or write-only depending on how the object is created. If the first argument is an iterable a file like object is returned that returns the contents of the iterable. In case the iterable is empty read operations will return the sentinel value. If the first argument is a callable then the stream object will be created and passed to that function. The caller itself however will not receive a stream but an iterable. The function will be be executed step by step as something iterates over the returned iterable. Each call to :meth:`flush` will create an item for the iterable. If :meth:`flush` is called without any writes in-between the sentinel value will be yielded. Note for Python 3: due to the incompatible interface of bytes and streams you should set the sentinel value explicitly to an empty bytestring (``b''``) if you are expecting to deal with bytes as otherwise the end of the stream is marked with the wrong sentinel value. .. versionadded:: 0.9 `sentinel` parameter was added. """ def __new__(cls, obj, sentinel=''): try: iterator = iter(obj) except TypeError: return IterI(obj, sentinel) return IterO(iterator, sentinel) def __iter__(self): return self def tell(self): if self.closed: raise ValueError('I/O operation on closed file') return self.pos def isatty(self): if self.closed: raise ValueError('I/O operation on closed file') return False def seek(self, pos, mode=0): if self.closed: raise ValueError('I/O operation on closed file') raise IOError(9, 'Bad file descriptor') def truncate(self, size=None): if self.closed: raise ValueError('I/O operation on closed file') raise IOError(9, 'Bad file descriptor') def write(self, s): if self.closed: raise ValueError('I/O operation on closed file') raise IOError(9, 'Bad file descriptor') def writelines(self, list): if self.closed: raise ValueError('I/O operation on closed file') raise IOError(9, 'Bad file descriptor') def read(self, n=-1): if self.closed: raise ValueError('I/O operation on closed file') raise IOError(9, 'Bad file descriptor') def readlines(self, sizehint=0): if self.closed: raise ValueError('I/O operation on closed file') raise IOError(9, 'Bad file descriptor') def readline(self, length=None): if self.closed: raise ValueError('I/O operation on closed file') raise IOError(9, 'Bad file descriptor') def flush(self): if self.closed: raise ValueError('I/O operation on closed file') raise IOError(9, 'Bad file descriptor') def __next__(self): if self.closed: raise StopIteration() line = self.readline() if not line: raise StopIteration() return line class IterI(IterIO): """Convert an stream into an iterator.""" def __new__(cls, func, sentinel=''): if greenlet is None: raise RuntimeError('IterI requires greenlet support') stream = object.__new__(cls) stream._parent = greenlet.getcurrent() stream._buffer = [] stream.closed = False stream.sentinel = sentinel stream.pos = 0 def run(): func(stream) stream.close() g = greenlet.greenlet(run, stream._parent) while 1: rv = g.switch() if not rv: return yield rv[0] def close(self): if not self.closed: self.closed = True self._flush_impl() def write(self, s): if self.closed: raise ValueError('I/O operation on closed file') if s: self.pos += len(s) self._buffer.append(s) def writelines(self, list): for item in list: self.write(item) def flush(self): if self.closed: raise ValueError('I/O operation on closed file') self._flush_impl() def _flush_impl(self): data = _mixed_join(self._buffer, self.sentinel) self._buffer = [] if not data and self.closed: self._parent.switch() else: self._parent.switch((data,)) class IterO(IterIO): """Iter output. Wrap an iterator and give it a stream like interface.""" def __new__(cls, gen, sentinel=''): self = object.__new__(cls) self._gen = gen self._buf = None self.sentinel = sentinel self.closed = False self.pos = 0 return self def __iter__(self): return self def _buf_append(self, string): '''Replace string directly without appending to an empty string, avoiding type issues.''' if not self._buf: self._buf = string else: self._buf += string def close(self): if not self.closed: self.closed = True if hasattr(self._gen, 'close'): self._gen.close() def seek(self, pos, mode=0): if self.closed: raise ValueError('I/O operation on closed file') if mode == 1: pos += self.pos elif mode == 2: self.read() self.pos = min(self.pos, self.pos + pos) return elif mode != 0: raise IOError('Invalid argument') buf = [] try: tmp_end_pos = len(self._buf) while pos > tmp_end_pos: item = self._gen.next() tmp_end_pos += len(item) buf.append(item) except StopIteration: pass if buf: self._buf_append(_mixed_join(buf, self.sentinel)) self.pos = max(0, pos) def read(self, n=-1): if self.closed: raise ValueError('I/O operation on closed file') if n < 0: self._buf_append(_mixed_join(self._gen, self.sentinel)) result = self._buf[self.pos:] self.pos += len(result) return result new_pos = self.pos + n buf = [] try: tmp_end_pos = 0 if self._buf is None else len(self._buf) while new_pos > tmp_end_pos or (self._buf is None and not buf): item = next(self._gen) tmp_end_pos += len(item) buf.append(item) except StopIteration: pass if buf: self._buf_append(_mixed_join(buf, self.sentinel)) if self._buf is None: return self.sentinel new_pos = max(0, new_pos) try: return self._buf[self.pos:new_pos] finally: self.pos = min(new_pos, len(self._buf)) def readline(self, length=None): if self.closed: raise ValueError('I/O operation on closed file') nl_pos = -1 if self._buf: nl_pos = self._buf.find(_newline(self._buf), self.pos) buf = [] try: if self._buf is None: pos = self.pos else: pos = len(self._buf) while nl_pos < 0: item = next(self._gen) local_pos = item.find(_newline(item)) buf.append(item) if local_pos >= 0: nl_pos = pos + local_pos break pos += len(item) except StopIteration: pass if buf: self._buf_append(_mixed_join(buf, self.sentinel)) if self._buf is None: return self.sentinel if nl_pos < 0: new_pos = len(self._buf) else: new_pos = nl_pos + 1 if length is not None and self.pos + length < new_pos: new_pos = self.pos + length try: return self._buf[self.pos:new_pos] finally: self.pos = min(new_pos, len(self._buf)) def readlines(self, sizehint=0): total = 0 lines = [] line = self.readline() while line: lines.append(line) total += len(line) if 0 < sizehint <= total: break line = self.readline() return lines