""" This file makes open() and friends RPython. Note that RFile should not be used directly and instead it's magically appearing each time you call python builtin open() """ import os, stat, errno, sys from rpython.rlib import rposix from rpython.rlib.objectmodel import enforceargs from rpython.rlib.rarithmetic import intmask from rpython.rlib.rstring import StringBuilder from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.tool import rffi_platform as platform from rpython.translator.tool.cbuild import ExternalCompilationInfo includes = ['stdio.h', 'sys/types.h'] if os.name == "posix": includes += ['unistd.h'] ftruncate = 'ftruncate' fileno = 'fileno' else: ftruncate = '_chsize' fileno = '_fileno' eci = ExternalCompilationInfo(includes=includes) class CConfig(object): _compilation_info_ = eci off_t = platform.SimpleType('off_t') _IONBF = platform.DefinedConstantInteger('_IONBF') _IOLBF = platform.DefinedConstantInteger('_IOLBF') _IOFBF = platform.DefinedConstantInteger('_IOFBF') BUFSIZ = platform.DefinedConstantInteger('BUFSIZ') EOF = platform.DefinedConstantInteger('EOF') config = platform.configure(CConfig) FILEP = rffi.COpaquePtr("FILE") OFF_T = config['off_t'] _IONBF = config['_IONBF'] _IOLBF = config['_IOLBF'] _IOFBF = config['_IOFBF'] BUFSIZ = config['BUFSIZ'] EOF = config['EOF'] BASE_BUF_SIZE = 4096 BASE_LINE_SIZE = 100 NEWLINE_UNKNOWN = 0 NEWLINE_CR = 1 NEWLINE_LF = 2 NEWLINE_CRLF = 4 def llexternal(*args, **kwargs): return rffi.llexternal(*args, compilation_info=eci, **kwargs) c_fopen = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], FILEP, save_err=rffi.RFFI_SAVE_ERRNO) c_popen = llexternal('popen', [rffi.CCHARP, rffi.CCHARP], FILEP, save_err=rffi.RFFI_SAVE_ERRNO) c_fdopen = llexternal(('_' if os.name == 'nt' else '') + 'fdopen', [rffi.INT, rffi.CCHARP], FILEP, save_err=rffi.RFFI_SAVE_ERRNO) c_tmpfile = llexternal('tmpfile', [], FILEP, save_err=rffi.RFFI_SAVE_ERRNO) c_setvbuf = llexternal('setvbuf', [FILEP, rffi.CCHARP, rffi.INT, rffi.SIZE_T], rffi.INT) c_fclose = llexternal('fclose', [FILEP], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) c_pclose = llexternal('pclose', [FILEP], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) # Note: the following two functions are called from __del__ methods, # so must be 'releasegil=False'. Otherwise, a program using both # threads and the RFile class cannot translate. See c684bf704d1f c_fclose_in_del = llexternal('fclose', [FILEP], rffi.INT, releasegil=False) c_pclose_in_del = llexternal('pclose', [FILEP], rffi.INT, releasegil=False) _fclose2 = (c_fclose, c_fclose_in_del) _pclose2 = (c_pclose, c_pclose_in_del) c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True) c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT) c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP) c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP], rffi.SIZE_T) c_fwrite = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP], rffi.SIZE_T, save_err=rffi.RFFI_SAVE_ERRNO) c_fflush = llexternal('fflush', [FILEP], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) c_ftruncate = llexternal(ftruncate, [rffi.INT, OFF_T], rffi.INT, macro=True, save_err=rffi.RFFI_SAVE_ERRNO) c_fseek = llexternal('fseek', [FILEP, rffi.LONG, rffi.INT], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) c_ftell = llexternal('ftell', [FILEP], rffi.LONG, save_err=rffi.RFFI_SAVE_ERRNO) c_fileno = llexternal(fileno, [FILEP], rffi.INT) c_feof = llexternal('feof', [FILEP], rffi.INT) c_ferror = llexternal('ferror', [FILEP], rffi.INT) c_clearerr = llexternal('clearerr', [FILEP], lltype.Void) c_stdin = rffi.CExternVariable(FILEP, 'stdin', eci, c_type='FILE*', getter_only=True) c_stdout = rffi.CExternVariable(FILEP, 'stdout', eci, c_type='FILE*', getter_only=True) c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*', getter_only=True) def _error(ll_file): err = c_ferror(ll_file) c_clearerr(ll_file) raise IOError(err, os.strerror(err)) def _dircheck(ll_file): try: st = os.fstat(c_fileno(ll_file)) except OSError: pass else: if stat.S_ISDIR(st[0]): err = errno.EISDIR raise IOError(err, os.strerror(err)) def _sanitize_mode(mode): if len(mode) == 0: raise ValueError("empty mode string") upos = mode.find('U') if upos >= 0: mode = mode[:upos] + mode[upos+1:] first = mode[0:1] if first == 'w' or first == 'a': raise ValueError("universal newline mode can only be used with " "modes starting with 'r'") if first != 'r': mode = 'r' + mode if 'b' not in mode: mode = mode[0] + 'b' + mode[1:] elif mode[0] != 'r' and mode[0] != 'w' and mode[0] != 'a': raise ValueError("mode string must begin with one of 'r', 'w', 'a' " "or 'U', not '%s'" % mode) return mode def create_file(filename, mode="r", buffering=-1): newmode = _sanitize_mode(mode) ll_name = rffi.str2charp(filename) try: ll_mode = rffi.str2charp(newmode) try: ll_file = c_fopen(ll_name, ll_mode) if not ll_file: errno = rposix.get_saved_errno() raise IOError(errno, os.strerror(errno)) finally: lltype.free(ll_mode, flavor='raw') finally: lltype.free(ll_name, flavor='raw') _dircheck(ll_file) f = RFile(ll_file, mode) f._setbufsize(buffering) return f def create_fdopen_rfile(fd, mode="r", buffering=-1): newmode = _sanitize_mode(mode) rposix.validate_fd(fd) ll_mode = rffi.str2charp(newmode) try: ll_file = c_fdopen(fd, ll_mode) if not ll_file: errno = rposix.get_saved_errno() raise OSError(errno, os.strerror(errno)) finally: lltype.free(ll_mode, flavor='raw') _dircheck(ll_file) f = RFile(ll_file, mode) f._setbufsize(buffering) return f def create_temp_rfile(): res = c_tmpfile() if not res: errno = rposix.get_saved_errno() raise OSError(errno, os.strerror(errno)) return RFile(res) def create_popen_file(command, type): ll_command = rffi.str2charp(command) try: ll_type = rffi.str2charp(type) try: ll_file = c_popen(ll_command, ll_type) if not ll_file: errno = rposix.get_saved_errno() raise OSError(errno, os.strerror(errno)) finally: lltype.free(ll_type, flavor='raw') finally: lltype.free(ll_command, flavor='raw') return RFile(ll_file, close2=_pclose2) def create_stdio(): close2 = (None, None) stdin = RFile(c_stdin(), close2=close2) stdout = RFile(c_stdout(), close2=close2) stderr = RFile(c_stderr(), close2=close2) return stdin, stdout, stderr def write_int(f, l): if sys.maxint == 2147483647: f.write(chr(l & 0xff) + chr((l >> 8) & 0xff) + chr((l >> 16) & 0xff) + chr((l >> 24) & 0xff)) else: f.write(chr(l & 0xff) + chr((l >> 8) & 0xff) + chr((l >> 16) & 0xff) + chr((l >> 24) & 0xff) + chr((l >> 32) & 0xff) + chr((l >> 40) & 0xff) + chr((l >> 48) & 0xff) + chr((l >> 56) & 0xff)) class RFile(object): _setbuf = lltype.nullptr(rffi.CCHARP.TO) _univ_newline = False _newlinetypes = NEWLINE_UNKNOWN _skipnextlf = False def __init__(self, ll_file, mode=None, close2=_fclose2): self._ll_file = ll_file if mode is not None: self._univ_newline = 'U' in mode self._close2 = close2 def _setbufsize(self, bufsize): if bufsize >= 0: if bufsize == 0: mode = _IONBF elif bufsize == 1: mode = _IOLBF bufsize = BUFSIZ else: mode = _IOFBF if self._setbuf: lltype.free(self._setbuf, flavor='raw') if mode == _IONBF: self._setbuf = lltype.nullptr(rffi.CCHARP.TO) else: self._setbuf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw') c_setvbuf(self._ll_file, self._setbuf, mode, bufsize) def __del__(self): """Closes the described file when the object's last reference goes away. Unlike an explicit call to close(), this is meant as a last-resort solution and cannot release the GIL or return an error code.""" ll_file = self._ll_file if ll_file: do_close = self._close2[1] if do_close: do_close(ll_file) # return value ignored if self._setbuf: lltype.free(self._setbuf, flavor='raw') def _cleanup_(self): self._ll_file = lltype.nullptr(FILEP.TO) def close(self): """Closes the described file. Attention! Unlike Python semantics, `close' does not return `None' upon success but `0', to be able to return an exit code for popen'ed files. The actual return value may be determined with os.WEXITSTATUS. """ res = 0 ll_file = self._ll_file if ll_file: # double close is allowed self._ll_file = lltype.nullptr(FILEP.TO) do_close = self._close2[0] try: if do_close: res = do_close(ll_file) if res == -1: errno = rposix.get_saved_errno() raise IOError(errno, os.strerror(errno)) finally: if self._setbuf: lltype.free(self._setbuf, flavor='raw') self._setbuf = lltype.nullptr(rffi.CCHARP.TO) return res def _check_closed(self): if not self._ll_file: raise ValueError("I/O operation on closed file") def _fread(self, buf, n, stream): if not self._univ_newline: return c_fread(buf, 1, n, stream) i = 0 dst = buf newlinetypes = self._newlinetypes skipnextlf = self._skipnextlf while n: nread = c_fread(dst, 1, n, stream) if nread == 0: break src = dst n -= nread shortread = n != 0 while nread: nread -= 1 c = src[0] src = rffi.ptradd(src, 1) if c == '\r': dst[0] = '\n' dst = rffi.ptradd(dst, 1) i += 1 skipnextlf = True elif skipnextlf and c == '\n': skipnextlf = False newlinetypes |= NEWLINE_CRLF n += 1 else: if c == '\n': newlinetypes |= NEWLINE_LF elif skipnextlf: newlinetypes |= NEWLINE_CR dst[0] = c dst = rffi.ptradd(dst, 1) i += 1 skipnextlf = False if shortread: if skipnextlf and c_feof(stream): newlinetypes |= NEWLINE_CR break self._newlinetypes = newlinetypes self._skipnextlf = skipnextlf return i def read(self, size=-1): # XXX CPython uses a more delicate logic here self._check_closed() ll_file = self._ll_file if size == 0: return "" elif size < 0: # read the entire contents buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw') try: s = StringBuilder() while True: returned_size = self._fread(buf, BASE_BUF_SIZE, ll_file) returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE if returned_size == 0: if c_feof(ll_file): # ok, finished return s.build() raise _error(ll_file) s.append_charpsize(buf, returned_size) finally: lltype.free(buf, flavor='raw') else: # size > 0 with rffi.scoped_alloc_buffer(size) as buf: returned_size = self._fread(buf.raw, size, ll_file) returned_size = intmask(returned_size) # is between 0 and size if returned_size == 0: if not c_feof(ll_file): raise _error(ll_file) s = buf.str(returned_size) assert s is not None return s def _readline1(self, raw_buf): ll_file = self._ll_file for i in range(BASE_LINE_SIZE): raw_buf[i] = '\n' result = c_fgets(raw_buf, BASE_LINE_SIZE, ll_file) if not result: if c_feof(ll_file): # ok return 0 raise _error(ll_file) # Assume that fgets() works as documented, and additionally # never writes beyond the final \0, which the CPython # fileobject.c says appears to be the case everywhere. # The only case where the buffer was not big enough is the # case where the buffer is full, ends with \0, and doesn't # end with \n\0. p = 0 while raw_buf[p] != '\n': p += 1 if p == BASE_LINE_SIZE: # fgets read whole buffer without finding newline return -1 # p points to first \n if p + 1 < BASE_LINE_SIZE and raw_buf[p + 1] == '\0': # \n followed by \0, fgets read and found newline return p + 1 else: # \n not followed by \0, fgets read but didnt find newline assert p > 0 and raw_buf[p - 1] == '\0' return p - 1 def readline(self, size=-1): self._check_closed() if size == 0: return "" elif size < 0 and not self._univ_newline: with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf: c = self._readline1(buf.raw) if c >= 0: return buf.str(c) # this is the rare case: the line is longer than BASE_LINE_SIZE s = StringBuilder() while True: s.append_charpsize(buf.raw, BASE_LINE_SIZE - 1) c = self._readline1(buf.raw) if c >= 0: break s.append_charpsize(buf.raw, c) return s.build() else: # size > 0 or self._univ_newline ll_file = self._ll_file c = 0 s = StringBuilder() if self._univ_newline: newlinetypes = self._newlinetypes skipnextlf = self._skipnextlf while size < 0 or s.getlength() < size: c = c_getc(ll_file) if c == EOF: break if skipnextlf: skipnextlf = False if c == ord('\n'): newlinetypes |= NEWLINE_CRLF c = c_getc(ll_file) if c == EOF: break else: newlinetypes |= NEWLINE_CR if c == ord('\r'): skipnextlf = True c = ord('\n') elif c == ord('\n'): newlinetypes |= NEWLINE_LF s.append(chr(c)) if c == ord('\n'): break if c == EOF: if skipnextlf: newlinetypes |= NEWLINE_CR self._newlinetypes = newlinetypes self._skipnextlf = skipnextlf else: while s.getlength() < size: c = c_getc(ll_file) if c == EOF: break s.append(chr(c)) if c == ord('\n'): break if c == EOF: if c_ferror(ll_file): raise _error(ll_file) return s.build() @enforceargs(None, str) def write(self, value): self._check_closed() with rffi.scoped_nonmovingbuffer(value) as ll_value: # note that since we got a nonmoving buffer, it is either raw # or already cannot move, so the arithmetics below are fine length = len(value) bytes = c_fwrite(ll_value, 1, length, self._ll_file) if bytes != length: errno = rposix.get_saved_errno() c_clearerr(self._ll_file) raise IOError(errno, os.strerror(errno)) def flush(self): self._check_closed() res = c_fflush(self._ll_file) if res != 0: errno = rposix.get_saved_errno() raise IOError(errno, os.strerror(errno)) def truncate(self, arg=-1): self._check_closed() if arg == -1: arg = self.tell() self.flush() res = c_ftruncate(self.fileno(), arg) if res == -1: errno = rposix.get_saved_errno() raise IOError(errno, os.strerror(errno)) def seek(self, pos, whence=0): self._check_closed() res = c_fseek(self._ll_file, pos, whence) if res == -1: errno = rposix.get_saved_errno() raise IOError(errno, os.strerror(errno)) self._skipnextlf = False def tell(self): self._check_closed() res = intmask(c_ftell(self._ll_file)) if res == -1: errno = rposix.get_saved_errno() raise IOError(errno, os.strerror(errno)) if self._skipnextlf: c = c_getc(self._ll_file) if c == ord('\n'): self._newlinetypes |= NEWLINE_CRLF res += 1 self._skipnextlf = False elif c != EOF: c_ungetc(c, self._ll_file) return res def fileno(self): self._check_closed() return intmask(c_fileno(self._ll_file)) def isatty(self): self._check_closed() return os.isatty(c_fileno(self._ll_file)) def __enter__(self): return self def __exit__(self, *args): self.close()