/* * Copyright 2000 Finn Bock * * This program contains material copyrighted by: * Copyright (c) Corporation for National Research Initiatives. * Originally written by Marc-Andre Lemburg (mal@lemburg.com). */ package org.python.core; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Iterator; import org.python.core.util.StringUtil; /** * Contains the implementation of the builtin codecs. * @since Jython 2.0 */ public class codecs { public static final String BACKSLASHREPLACE = "backslashreplace"; public static final String IGNORE = "ignore"; public static final String REPLACE = "replace"; public static final String XMLCHARREFREPLACE = "xmlcharrefreplace"; private static char Py_UNICODE_REPLACEMENT_CHARACTER = 0xFFFD; private static PyList searchPath; private static PyStringMap searchCache; private static PyStringMap errorHandlers; /** Used to synchronize registry_init. */ private static final Object INIT_LOCK = new Object(); private static String default_encoding = "ascii"; public static String getDefaultEncoding() { return default_encoding; } public static void setDefaultEncoding(String encoding) { lookup(encoding); default_encoding = encoding; } public static PyObject lookup_error(String handlerName) { registry_init(); if (handlerName == null) { handlerName = "strict"; } PyObject handler = errorHandlers.__finditem__(handlerName.intern()); if (handler == null) { throw new PyException(Py.LookupError, "unknown error handler name '" + handlerName + "'"); } return handler; } public static void register_error(String name, PyObject error) { registry_init(); if (!error.isCallable()) { throw Py.TypeError("argument must be callable"); } errorHandlers.__setitem__(name.intern(), error); } public static void register(PyObject search_function) { registry_init(); if (!search_function.isCallable()) { throw Py.TypeError("argument must be callable"); } searchPath.append(search_function); } public static PyTuple lookup(String encoding) { registry_init(); PyString v = new PyString(normalizestring(encoding)); PyObject cached = searchCache.__finditem__(v); if (cached != null) { return (PyTuple)cached; } if (searchPath.__len__() == 0) { throw new PyException(Py.LookupError, "no codec search functions registered: can't find encoding '" + encoding + "'"); } for (PyObject func : searchPath.asIterable()) { PyObject created = func.__call__(v); if (created == Py.None) { continue; } if (!(created instanceof PyTuple) || created.__len__() != 4) { throw Py.TypeError("codec search functions must return 4-tuples"); } searchCache.__setitem__(v, created); return (PyTuple)created; } throw new PyException(Py.LookupError, "unknown encoding '" + encoding + "'"); } private static String normalizestring(String string) { return string.toLowerCase().replace(' ', '-'); } private static boolean import_encodings_called; private static void import_encodings() { if (!import_encodings_called) { import_encodings_called = true; try { imp.load("encodings"); } catch (PyException exc) { if (exc.type != Py.ImportError) { throw exc; } } } } public static PyObject decode(PyString v, String encoding, String errors) { if (encoding == null) { encoding = getDefaultEncoding(); } else { encoding = normalizestring(encoding); } if (errors != null) { errors = errors.intern(); } /* Shortcut for ascii encoding */ if (encoding.equals("ascii")) { return wrapDecodeResult(PyUnicode_DecodeASCII(v.toString(), v.__len__(), errors)); } /* Decode via the codec registry */ PyObject decoder; try { decoder = lookup(encoding).__getitem__(1); } catch (PyException ex) { if (ex.match(Py.LookupError)) { // If we couldn't find an encoding, see if we have a builtin if (encoding.equals("utf-8")) { return wrapDecodeResult(PyUnicode_DecodeUTF8(v.toString(), errors)); } else if(encoding.equals("utf-7")) { return wrapDecodeResult(PyUnicode_DecodeUTF7(v.toString(), errors)); } else if(encoding.equals("latin-1")) { return wrapDecodeResult(PyUnicode_DecodeLatin1(v.toString(), v.__len__(), errors)); } } throw ex; } PyObject result; if (errors != null) { result = decoder.__call__(v, new PyString(errors)); } else { result = decoder.__call__(v); } if (!(result instanceof PyTuple) || result.__len__() != 2) { throw Py.TypeError("decoder must return a tuple (object,integer)"); } return result.__getitem__(0); } private static PyUnicode wrapDecodeResult(String result) { return new PyUnicode(result, true); } public static String encode(PyString v, String encoding, String errors) { if (encoding == null) { encoding = getDefaultEncoding(); } else { encoding = normalizestring(encoding); } if (errors != null) { errors = errors.intern(); } /* Shortcuts for common default encodings. latin-1 must not use the * lookup registry for the encodings module to work correctly */ if (encoding.equals("latin-1")) { return PyUnicode_EncodeLatin1(v.toString(), v.__len__(), errors); } else if (encoding.equals("ascii")) { return PyUnicode_EncodeASCII(v.toString(), v.__len__(), errors); } /* Encode via the codec registry */ PyObject encoder; try { encoder = lookup(encoding).__getitem__(0); } catch (PyException ex) { if (ex.match(Py.LookupError)) { // If we couldn't find an encoding, see if we have a builtin if (encoding.equals("utf-8")) { return PyUnicode_EncodeUTF8(v.toString(), errors); } else if(encoding.equals("utf-7")) { return codecs.PyUnicode_EncodeUTF7(v.toString(), false, false, errors); } } throw ex; } PyObject result; if (errors != null) { result = encoder.__call__(v, new PyString(errors)); } else { result = encoder.__call__(v); } if (!(result instanceof PyTuple) || result.__len__() != 2) { throw Py.TypeError("encoder must return a tuple (object,integer)"); } PyObject encoded = result.__getitem__(0); if (encoded instanceof PyString) { return encoded.toString(); } else { throw Py.TypeError("encoder did not return a string/unicode object (type=" + encoded.getType().fastGetName() + ")"); } } public static PyObject strict_errors(PyObject[] args, String[] kws) { ArgParser ap = new ArgParser("strict_errors", args, kws, "exc"); PyObject exc = ap.getPyObject(0); if (Py.isInstance(exc, Py.UnicodeDecodeError)) { throw new PyException(Py.UnicodeDecodeError, exc); } else if (Py.isInstance(exc, Py.UnicodeEncodeError)) { throw new PyException(Py.UnicodeEncodeError, exc); } else if (Py.isInstance(exc, Py.UnicodeTranslateError)) { throw new PyException(Py.UnicodeTranslateError, exc); } throw wrong_exception_type(exc); } public static PyObject ignore_errors(PyObject[] args, String[] kws) { ArgParser ap = new ArgParser("ignore_errors", args, kws, "exc"); PyObject exc = ap.getPyObject(0); if (!isUnicodeError(exc)) { throw wrong_exception_type(exc); } PyObject end = exc.__getattr__("end"); return new PyTuple(Py.java2py(""), end); } private static boolean isUnicodeError(PyObject exc) { return Py.isInstance(exc, Py.UnicodeDecodeError) || Py.isInstance(exc, Py.UnicodeEncodeError) || Py.isInstance(exc, Py.UnicodeTranslateError); } public static PyObject replace_errors(PyObject[] args, String[] kws) { ArgParser ap = new ArgParser("replace_errors", args, kws, "exc"); PyObject exc = ap.getPyObject(0); if (Py.isInstance(exc, Py.UnicodeDecodeError)) { PyObject object = exc.__getattr__("object"); if (!Py.isInstance(object, PyString.TYPE) || Py.isInstance(object, PyUnicode.TYPE)) { throw Py.TypeError("object attribute must be str"); } PyObject end = exc.__getattr__("end"); return new PyTuple(new PyUnicode(Py_UNICODE_REPLACEMENT_CHARACTER), end); } else if (Py.isInstance(exc, Py.UnicodeEncodeError)) { PyObject object = exc.__getattr__("object"); if (!Py.isInstance(object, PyUnicode.TYPE)) { throw Py.TypeError("object attribute must be unicode"); } PyObject end = exc.__getattr__("end"); return new PyTuple(Py.java2py("?"), end); } else if (Py.isInstance(exc, Py.UnicodeTranslateError)) { PyObject end = exc.__getattr__("end"); return new PyTuple(new PyUnicode(Py_UNICODE_REPLACEMENT_CHARACTER), end); } throw wrong_exception_type(exc); } public static PyObject xmlcharrefreplace_errors(PyObject[] args, String[] kws) { ArgParser ap = new ArgParser("xmlcharrefreplace_errors", args, kws, "exc"); PyObject exc = ap.getPyObject(0); if (!Py.isInstance(exc, Py.UnicodeEncodeError)) { throw wrong_exception_type(exc); } int start = ((PyInteger) exc.__getattr__("start")).getValue(); int end = ((PyInteger) exc.__getattr__("end")).getValue(); String object = exc.__getattr__("object").toString(); StringBuilder replacement = new StringBuilder(); xmlcharrefreplace_internal(start, end, object, replacement); return new PyTuple(Py.java2py(replacement.toString()), exc.__getattr__("end")); } public static StringBuilder xmlcharrefreplace(int start, int end, String toReplace) { StringBuilder replacement = new StringBuilder(); xmlcharrefreplace_internal(start, end, toReplace, replacement); return replacement; } private static void xmlcharrefreplace_internal(int start, int end, String object, StringBuilder replacement) { for (int i = start; i < end; i++) { replacement.append("&#"); char cur = object.charAt(i); int digits; int base; if (cur < 10) { digits = 1; base = 1; } else if (cur < 100) { digits = 2; base = 10; } else if (cur < 1000) { digits = 3; base = 100; } else if (cur < 10000) { digits = 4; base = 1000; } else if (cur < 100000) { digits = 5; base = 10000; } else if (cur < 1000000) { digits = 6; base = 100000; } else { digits = 7; base = 1000000; } while (digits-- > 0) { replacement.append((char) ('0' + cur / base)); cur %= base; base /= 10; } replacement.append(';'); } } private static PyException wrong_exception_type(PyObject exc) { PyObject excClass = exc.__getattr__("__class__"); PyObject className = excClass.__getattr__("__name__"); return new PyException(Py.TypeError, "Don't know how to handle " + className + " in error callback"); } static char hexdigits[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; public static PyObject backslashreplace_errors(PyObject[] args, String[] kws) { ArgParser ap = new ArgParser("backslashreplace_errors", args, kws, "exc"); PyObject exc = ap.getPyObject(0); if (!Py.isInstance(exc, Py.UnicodeEncodeError)) { throw wrong_exception_type(exc); } int start = ((PyInteger) exc.__getattr__("start")).getValue(); int end = ((PyInteger) exc.__getattr__("end")).getValue(); String object = exc.__getattr__("object").toString(); StringBuilder replacement = new StringBuilder(); backslashreplace_internal(start, end, object, replacement); return new PyTuple(Py.java2py(replacement.toString()), exc.__getattr__("end")); } public static StringBuilder backslashreplace(int start, int end, String toReplace) { StringBuilder replacement = new StringBuilder(); backslashreplace_internal(start, end, toReplace, replacement); return replacement; } private static void backslashreplace_internal(int start, int end, String object, StringBuilder replacement) { for (Iterator iter = new StringSubsequenceIterator(object, start, end, 1); iter.hasNext();) { int c = iter.next(); replacement.append('\\'); if (c >= 0x00010000) { replacement.append('U'); replacement.append(hexdigits[(c >> 28) & 0xf]); replacement.append(hexdigits[(c >> 24) & 0xf]); replacement.append(hexdigits[(c >> 20) & 0xf]); replacement.append(hexdigits[(c >> 16) & 0xf]); replacement.append(hexdigits[(c >> 12) & 0xf]); replacement.append(hexdigits[(c >> 8) & 0xf]); } else if (c >= 0x100) { replacement.append('u'); replacement.append(hexdigits[(c >> 12) & 0xf]); replacement.append(hexdigits[(c >> 8) & 0xf]); } else { replacement.append('x'); } replacement.append(hexdigits[(c >> 4) & 0xf]); replacement.append(hexdigits[c & 0xf]); } } private static void registry_init() { synchronized (INIT_LOCK) { if (searchPath != null) { return; } searchPath = new PyList(); searchCache = new PyStringMap(); errorHandlers = new PyStringMap(); String[] builtinErrorHandlers = new String[]{"strict", IGNORE, REPLACE, XMLCHARREFREPLACE, BACKSLASHREPLACE }; for (String builtinErrorHandler : builtinErrorHandlers) { register_error(builtinErrorHandler, Py.newJavaFunc(codecs.class, builtinErrorHandler + "_errors")); } import_encodings(); } } /* --- UTF-7 Codec -------------------------------------------------------- */ /* see RFC2152 for details */ public static char utf7_special[] = { /* * indicate whether a UTF-7 character is special i.e. cannot be directly * encoded: 0 - not special 1 - special 2 - whitespace (optional) 3 - * RFC2152 Set O (optional) */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1 , }; private static boolean SPECIAL(char c, boolean encodeO, boolean encodeWS){ return (c>127 || utf7_special[(c)] == 1) || (encodeWS && (utf7_special[(c)] == 2)) || (encodeO && (utf7_special[(c)] == 3)); } private static final String B64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; private static char B64(int n) { return B64_CHARS.charAt(n & 0x3f); } private static boolean B64CHAR(char c) { return B64_CHARS.indexOf(c) != -1; } private static int UB64(char c) { return ((c) == '+' ? 62 : (c) == '/' ? 63 : (c) >= 'a' ? (c) - 71 : (c) >= 'A' ? (c) - 65 : (c) + 4); } // note that we follow CPython 2.5 exactly here - it does not support surrogates, // but has to process as-if they are there for replacement purposes // fortunately no one really cares about utf-7 public static String PyUnicode_DecodeUTF7(String str, String errors) { int s = 0; int e = str.length(); boolean inShift = false; int bitsInCharsleft = 0; long charsleft = 0; boolean surrogate = false; StringBuilder unicode = new StringBuilder(e); while (s < e) { // restart: char ch = str.charAt(s); if (inShift) { if ((ch == '-') || !B64CHAR(ch)) { inShift = false; s++; while (bitsInCharsleft >= 16) { bitsInCharsleft -= 16; char outCh = (char) ((charsleft >> bitsInCharsleft) & 0xffff); if (surrogate) { s = codecs.insertReplacementAndGetResume(unicode, errors, "utf-7", str, s, s + 1, "code pairs are not supported"); surrogate = false; } else if (0xDC00 <= outCh && outCh <= 0xDFFF) { surrogate = true; } else { unicode.append(outCh); } } if (bitsInCharsleft >= 6) { /* * The shift sequence has a partial character in it. If * bitsleft < 6 then we could just classify it as * padding but that is not the case here */ s = insertReplacementAndGetResume(unicode, errors, "utf-7", str, s, s + 1, "partial character in shift sequence"); } /* * According to RFC2152 the remaining bits should be zero. * We choose to signal an error/insert a replacement * character here so indicate the potential of a misencoded * character. */ if (bitsInCharsleft > 0 && ((charsleft << 5 - bitsInCharsleft) & 0x1f) > 0) { s = insertReplacementAndGetResume(unicode, errors, "utf-7", str, s, s + 1, "non-zero padding bits in shift sequence"); } if (ch == '-') { if ((s < e) && (str.charAt(s) == '-')) { unicode.append('-'); inShift = true; } } else if (SPECIAL(ch, false, false)) { s = insertReplacementAndGetResume(unicode, errors, "utf-7", str, s, s + 1, "unexpected special character"); } else { unicode.append(ch); } } else { charsleft = (charsleft << 6) | UB64(ch); bitsInCharsleft += 6; s++; while (bitsInCharsleft >= 16) { bitsInCharsleft -= 16; char outCh = (char) ((charsleft >> bitsInCharsleft) & 0xffff); if (surrogate) { s = codecs.insertReplacementAndGetResume(unicode, errors, "utf-7", str, s, s + 1, "code pairs are not supported"); } else if (0xDC00 <= outCh && outCh <= 0xDFFF) { surrogate = true; } else { unicode.append(outCh); } } } } else if (ch == '+') { s++; if (s < e && str.charAt(s) == '-') { s++; unicode.append('+'); } else { inShift = true; bitsInCharsleft = 0; } } else if (SPECIAL(ch, false, false)) { s = insertReplacementAndGetResume(unicode, errors, "utf-7", str, s, s + 1, "unexpected special character"); } else { unicode.append(ch); s++; } if (inShift && s == e) { s = insertReplacementAndGetResume(unicode, errors, "utf-7", str, s, s, "unterminated shift sequence"); } } return unicode.toString(); } public static String PyUnicode_EncodeUTF7(String str, boolean encodeSetO, boolean encodeWhiteSpace, String errors) { int size = str.length(); if (size == 0) { return ""; } boolean inShift = false; int bitsleft = 0; int charsleft = 0; StringBuilder v = new StringBuilder(); for (int i = 0; i < size; ++i) { char ch = str.charAt(i); if (!inShift) { if (ch == '+') { v.append('+'); v.append('-'); } else if (SPECIAL(ch, encodeSetO, encodeWhiteSpace)) { charsleft = ch; bitsleft = 16; v.append('+'); while (bitsleft >= 6) { v.append(B64(charsleft >> (bitsleft - 6))); bitsleft -= 6; } inShift = bitsleft > 0; } else { v.append(ch); } } else { if (!SPECIAL(ch, encodeSetO, encodeWhiteSpace)) { v.append(B64(charsleft << (6 - bitsleft))); charsleft = 0; bitsleft = 0; /* Characters not in the BASE64 set implicitly unshift the sequence so no '-' is required, except if the character is itself a '-' */ if (B64CHAR(ch) || ch == '-') { v.append('-'); } inShift = false; v.append(ch); } else { bitsleft += 16; charsleft = (charsleft << 16) | ch; while (bitsleft >= 6) { v.append(B64(charsleft >> (bitsleft - 6))); bitsleft -= 6; } /* If the next character is special then we dont' need to terminate the shift sequence. If the next character is not a BASE64 character or '-' then the shift sequence will be terminated implicitly and we don't have to insert a '-'. */ if (bitsleft == 0) { if (i + 1 < size) { char ch2 = str.charAt(i + 1); if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)) { } else if (B64CHAR(ch2) || ch2 == '-') { v.append('-'); inShift = false; } else { inShift = false; } } else { v.append('-'); inShift = false; } } } } } if (bitsleft > 0) { v.append(B64(charsleft << (6 - bitsleft))); v.append('-'); } return v.toString(); } /* --- UTF-8 Codec ---------------------------------------------------- */ private static byte utf8_code_length[] = { /* Map UTF-8 encoded prefix byte to sequence length. zero means illegal prefix. see RFC 2279 for details */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 }; // TODO: need to modify to use a codepoint approach (which is almost the case now, // ch is an public static String PyUnicode_DecodeUTF8(String str, String errors) { return PyUnicode_DecodeUTF8Stateful(str, errors, null); } public static String PyUnicode_DecodeUTF8Stateful(String str, String errors, int[] consumed) { int size = str.length(); StringBuilder unicode = new StringBuilder(size); /* Unpack UTF-8 encoded data */ int i; for (i = 0; i < size;) { int ch = str.charAt(i); if (ch < 0x80) { unicode.append((char) ch); i++; continue; } if (ch > 0xFF) { i = insertReplacementAndGetResume(unicode, errors, "utf-8", str, i, i + 1, "ordinal not in range(255)"); continue; } int n = utf8_code_length[ch]; if (i + n > size) { if (consumed != null) { break; } i = insertReplacementAndGetResume(unicode, errors, "utf-8", str, i, i + 1, "unexpected end of data"); continue; } switch (n) { case 0: i = insertReplacementAndGetResume(unicode, errors, "utf-8", str, i, i + 1, "unexpected code byte"); continue; case 1: i = insertReplacementAndGetResume(unicode, errors, "utf-8", str, i, i + 1, "internal error"); continue; case 2: char ch1 = str.charAt(i + 1); if ((ch1 & 0xc0) != 0x80) { i = insertReplacementAndGetResume(unicode, errors, "utf-8", str, i, i + 2, "invalid data"); continue; } ch = ((ch & 0x1f) << 6) + (ch1 & 0x3f); if (ch < 0x80) { i = insertReplacementAndGetResume(unicode, errors, "utf-8", str, i, i + 2, "illegal encoding"); continue; } else { unicode.appendCodePoint(ch); } break; case 3: ch1 = str.charAt(i + 1); char ch2 = str.charAt(i + 2); if ((ch1 & 0xc0) != 0x80 || (ch2 & 0xc0) != 0x80) { i = insertReplacementAndGetResume(unicode, errors, "utf-8", str, i, i + 3, "invalid data"); continue; } ch = ((ch & 0x0f) << 12) + ((ch1 & 0x3f) << 6) + (ch2 & 0x3f); if (ch < 0x800 || (ch >= 0xd800 && ch < 0xe000)) { i = insertReplacementAndGetResume(unicode, errors, "utf-8", str, i, i + 3, "illegal encoding"); continue; } else { unicode.appendCodePoint(ch); } break; case 4: ch1 = str.charAt(i + 1); ch2 = str.charAt(i + 2); char ch3 = str.charAt(i + 3); if ((ch1 & 0xc0) != 0x80 || (ch2 & 0xc0) != 0x80 || (ch3 & 0xc0) != 0x80) { i = insertReplacementAndGetResume(unicode, errors, "utf-8", str, i, i + 4, "invalid data"); continue; } ch = ((ch & 0x7) << 18) + ((ch1 & 0x3f) << 12) + ((ch2 & 0x3f) << 6) + (ch3 & 0x3f); /* validate and convert to UTF-16 */ if ((ch < 0x10000) || /* minimum value allowed for 4 byte encoding */ (ch > 0x10ffff)) { /* maximum value allowed for UTF-16 */ i = insertReplacementAndGetResume(unicode, errors, "utf-8", str, i, i + 4, "illegal encoding"); continue; } unicode.appendCodePoint(ch); break; default: // TODO: support /* Other sizes are only needed for UCS-4 */ i = insertReplacementAndGetResume(unicode, errors, "utf-8", str, i, i + n, "unsupported Unicode code range"); continue; } i += n; } if (consumed != null) { consumed[0] = i; } return unicode.toString(); } public static String PyUnicode_EncodeUTF8(String str, String errors) { return StringUtil.fromBytes(Charset.forName("UTF-8").encode(str)); } public static String PyUnicode_DecodeASCII(String str, int size, String errors) { return PyUnicode_DecodeIntLimited(str, size, errors, "ascii", 128); } public static String PyUnicode_DecodeLatin1(String str, int size, String errors) { return PyUnicode_DecodeIntLimited(str, size, errors, "latin-1", 256); } private static String PyUnicode_DecodeIntLimited(String str, int size, String errors, String encoding, int limit) { StringBuilder v = new StringBuilder(size); String reason = "ordinal not in range(" + limit + ")"; for (int i = 0; i < size; i++) { char ch = str.charAt(i); if (ch < limit) { v.append(ch); } else { i = insertReplacementAndGetResume(v, errors, encoding, str, i, i + 1, reason) - 1; } } return v.toString(); } public static String PyUnicode_EncodeASCII(String str, int size, String errors) { return PyUnicode_EncodeIntLimited(str, size, errors, "ascii", 128); } public static String PyUnicode_EncodeLatin1(String str, int size, String errors) { return PyUnicode_EncodeIntLimited(str, size, errors, "latin-1", 256); } private static String PyUnicode_EncodeIntLimited(String str, int size, String errors, String encoding, int limit) { String reason = "ordinal not in range(" + limit + ")"; StringBuilder v = new StringBuilder(size); for (int i = 0; i < size; i++) { char ch = str.charAt(i); if (ch >= limit) { int nextGood = i + 1; for (; nextGood < size; nextGood++) { if (str.charAt(nextGood) < limit) { break; } } if (errors != null) { if (errors.equals(IGNORE)) { i = nextGood - 1; continue; } else if (errors.equals(REPLACE)) { for (int j = i; j < nextGood; j++) { v.append('?'); } i = nextGood - 1; continue; } else if (errors.equals(XMLCHARREFREPLACE)) { v.append(xmlcharrefreplace(i, nextGood, str)); i = nextGood - 1; continue; } else if (errors.equals(BACKSLASHREPLACE)) { v.append(backslashreplace(i, nextGood, str)); i = nextGood - 1; continue; } } PyObject replacement = encoding_error(errors, encoding, str, i, nextGood, reason); String replStr = replacement.__getitem__(0).toString(); for (int j = 0; j < replStr.length(); j++) { if (replStr.charAt(j) >= limit) { throw Py.UnicodeEncodeError(encoding, str, i + j, i + j + 1, reason); } } v.append(replStr); i = calcNewPosition(size, replacement) - 1; } else { v.append(ch); } } return v.toString(); } public static int calcNewPosition(int size, PyObject errorTuple) { int newPosition = ((PyInteger) errorTuple.__getitem__(1)).getValue(); if (newPosition < 0) { newPosition = size + newPosition; } if (newPosition > size || newPosition < 0) { throw Py.IndexError(newPosition + " out of bounds of encoded string"); } return newPosition; } /* --- RawUnicodeEscape Codec ---------------------------------------- */ private static char[] hexdigit = "0123456789ABCDEF".toCharArray(); // The modified flag is used by cPickle. public static String PyUnicode_EncodeRawUnicodeEscape(String str, String errors, boolean modifed) { StringBuilder v = new StringBuilder(str.length()); for (Iterator iter = new PyUnicode(str).newSubsequenceIterator(); iter.hasNext();) { int codePoint = iter.next(); if (codePoint >= Character.MIN_SUPPLEMENTARY_CODE_POINT) { // Map 32-bit characters to '\\Uxxxxxxxx' v.append("\\U"); v.append(hexdigit[(codePoint >> 28) & 0xF]); v.append(hexdigit[(codePoint >> 24) & 0xF]); v.append(hexdigit[(codePoint >> 20) & 0xF]); v.append(hexdigit[(codePoint >> 16) & 0xF]); v.append(hexdigit[(codePoint >> 12) & 0xF]); v.append(hexdigit[(codePoint >> 8) & 0xF]); v.append(hexdigit[(codePoint >> 4) & 0xF]); v.append(hexdigit[codePoint & 0xF]); } else if (codePoint >= 256 || (modifed && (codePoint == '\\' || codePoint == '\n'))) { // Map 16-bit chararacters to '\\uxxxx' v.append("\\u"); v.append(hexdigit[(codePoint >> 12) & 0xF]); v.append(hexdigit[(codePoint >> 8) & 0xF]); v.append(hexdigit[(codePoint >> 4) & 0xF]); v.append(hexdigit[codePoint & 0xF]); } else { v.append((char)codePoint); } } return v.toString(); } public static String PyUnicode_DecodeRawUnicodeEscape(String str, String errors) { int size = str.length(); StringBuilder v = new StringBuilder(size); for (int i = 0; i < size;) { char ch = str.charAt(i); // Non-escape characters are interpreted as Unicode ordinals if (ch != '\\') { v.append(ch); i++; continue; } // \\u-escapes are only interpreted if the number of leading backslashes is // odd int bs = i; while (i < size) { ch = str.charAt(i); if (ch != '\\') { break; } v.append(ch); i++; } if (((i - bs) & 1) == 0 || i >= size || (ch != 'u' && ch != 'U')) { continue; } v.setLength(v.length() - 1); int count = ch == 'u' ? 4 : 8; i++; // \\uXXXX with 4 hex digits, \Uxxxxxxxx with 8 int codePoint = 0, asDigit = -1; for (int j = 0; j < count; i++, j++) { if (i == size) { // EOF in a truncated escape asDigit = -1; break; } ch = str.charAt(i); asDigit = Character.digit(ch, 16); if (asDigit == -1) { break; } codePoint = ((codePoint << 4) & ~0xF) + asDigit; } if (asDigit == -1) { i = codecs.insertReplacementAndGetResume(v, errors, "rawunicodeescape", str, bs, i, "truncated \\uXXXX"); } else { v.appendCodePoint(codePoint); } } return v.toString(); } private static class Punycode { // specified by punycode, http://www.ietf.org/rfc/rfc3492.txt private static final int BASE = 36; private static final int TMIN = 1; private static final int TMAX = 26; private static final int SKEW = 38; private static final int DAMP = 700; private static final int INITIAL_BIAS = 72; private static final int INITIAL_N = 128; private static final int BASIC = 0x80; private Punycode() { } private static int adapt(int delta, int numpoints, boolean firsttime) { delta = firsttime ? delta / DAMP : delta >> 1; delta += delta / numpoints; int k = 0; while (delta > (((BASE - TMIN) * TMAX) / 2)) { delta /= BASE - TMIN; k += BASE; } return k + (((BASE - TMIN + 1) * delta) / (delta + SKEW)); } private static boolean isBasic(int codePoint) { return codePoint < BASIC; } } public static String PyUnicode_EncodePunycode(PyUnicode input, String errors) { int n = Punycode.INITIAL_N; int delta = 0; long guard_delta; int bias = Punycode.INITIAL_BIAS; int b = 0; final StringBuilder buffer = new StringBuilder(); for (Iterator iter = input.iterator(); iter.hasNext();) { int c = iter.next(); if (Punycode.isBasic(c)) { buffer.appendCodePoint(c); b++; } } if (b > 0) { buffer.appendCodePoint('-'); } int h = b; int size = input.getCodePointCount(); while (h < size) { int m = Integer.MAX_VALUE; int i = 0; int codePointIndex = 0; for (Iterator iter = input.iterator(); iter.hasNext(); i++) { int c = iter.next(); if (c > n && c < m) { m = c; codePointIndex = i; } } guard_delta = delta + ((m - n) * (h + 1)); if (guard_delta > Integer.MAX_VALUE) { throw Py.UnicodeEncodeError("punycode", input.getString(), codePointIndex, codePointIndex + 1, "overflow"); } delta = (int) guard_delta; n = m; i = 0; for (Iterator iter = input.iterator(); iter.hasNext(); i++) { int c = iter.next(); if (c < n) { guard_delta = delta + 1; if (guard_delta > Integer.MAX_VALUE) { throw Py.UnicodeEncodeError("punycode", input.getString(), i, i + 1, "overflow"); } delta = (int) guard_delta; } if (c == n) { int q = delta; for (int k = Punycode.BASE;; k += Punycode.BASE) { int t = k <= bias ? Punycode.TMIN : (k >= bias + Punycode.TMAX ? Punycode.TMAX : k - bias); if (q < t) { break; } buffer.appendCodePoint(t + ((q - t) % (Punycode.BASE - t))); q = (q - t) / (Punycode.BASE - t); } buffer.appendCodePoint(q); bias = Punycode.adapt(delta, h + 1, h == b); delta = 0; h++; } } delta++; n++; } return buffer.toString(); } public static PyUnicode PyUnicode_DecodePunycode(String input, String errors) { int input_size = input.length(); int output_size = 0; ArrayList ucs4 = new ArrayList(input_size); int j = 0; for (; j < input_size; j++) { int c = input.charAt(j); if (!Punycode.isBasic(c)) { throw Py.UnicodeDecodeError("punycode", input, j, j + 1, "not basic"); } else if (c == '-') { break; } else { ucs4.add(c); output_size++; } } int n = Punycode.INITIAL_N; int i = 0; int bias = Punycode.INITIAL_BIAS; while (j < input_size) { int old_i = i; int w = 1; for (int k = Punycode.BASE;; k += Punycode.BASE) { int c = input.charAt(j++); int digit = c - '0'; long guard_i = i + digit * w; if (guard_i > Integer.MAX_VALUE) { throw Py.UnicodeDecodeError("punycode", input, j, j + 1, "overflow"); } i = (int) guard_i; int t = k <= bias ? Punycode.TMIN : (k >= bias + Punycode.TMAX ? Punycode.TMAX : k - bias); if (digit < t) { break; } long guard_w = w * Punycode.BASE - t; if (guard_w > Integer.MAX_VALUE) { throw Py.UnicodeDecodeError("punycode", input, j, j + 1, "overflow"); } } bias = Punycode.adapt(i - old_i, output_size + 1, old_i == 0); n += i / (output_size + 1); i %= output_size + 1; ucs4.add(i, n); } return new PyUnicode(ucs4); } public static String PyUnicode_EncodeIDNA(PyUnicode input, String errors) { throw new UnsupportedOperationException(); // 1. If the sequence contains any code points outside the ASCII range // (0..7F) then proceed to step 2, otherwise skip to step 3. // // 2. Perform the steps specified in [NAMEPREP] and fail if there is an // error. The AllowUnassigned flag is used in [NAMEPREP]. // this basically enails changing out space, etc. // // 3. If the UseSTD3ASCIIRules flag is set, then perform these checks: // // (a) Verify the absence of non-LDH ASCII code points; that is, the // absence of 0..2C, 2E..2F, 3A..40, 5B..60, and 7B..7F. // // (b) Verify the absence of leading and trailing hyphen-minus; that // is, the absence of U+002D at the beginning and end of the // sequence. // // 4. If the sequence contains any code points outside the ASCII range // (0..7F) then proceed to step 5, otherwise skip to step 8. // // 5. Verify that the sequence does NOT begin with the ACE prefix. // // 6. Encode the sequence using the encoding algorithm in [PUNYCODE] and // fail if there is an error. // // 7. Prepend the ACE prefix. // // 8. Verify that the number of code points is in the range 1 to 63 // inclusive. } public static PyUnicode PyUnicode_DecodeIDNA(String input, String errors) { throw new UnsupportedOperationException(); } /* --- Utility methods -------------------------------------------- */ public static PyObject encoding_error(String errors, String encoding, String toEncode, int start, int end, String reason) { PyObject errorHandler = lookup_error(errors); PyException exc = Py.UnicodeEncodeError(encoding, toEncode, start, end, reason); exc.normalize(); PyObject replacement = errorHandler.__call__(new PyObject[]{exc.value}); checkErrorHandlerReturn(errors, replacement); return replacement; } public static int insertReplacementAndGetResume(StringBuilder partialDecode, String errors, String encoding, String toDecode, int start, int end, String reason) { if (errors != null) { if (errors.equals(IGNORE)) { return end; } else if (errors.equals(REPLACE)) { while (start < end) { partialDecode.appendCodePoint(Py_UNICODE_REPLACEMENT_CHARACTER); start++; } return end; } } PyObject replacement = decoding_error(errors, encoding, toDecode, start, end, reason); checkErrorHandlerReturn(errors, replacement); partialDecode.append(replacement.__getitem__(0).toString()); return calcNewPosition(toDecode.length(), replacement); } public static PyObject decoding_error(String errors, String encoding, String toEncode, int start, int end, String reason) { PyObject errorHandler = lookup_error(errors); PyException exc = Py.UnicodeDecodeError(encoding, toEncode, start, end, reason); exc.normalize(); return errorHandler.__call__(new PyObject[]{exc.value}); } private static void checkErrorHandlerReturn(String errors, PyObject replacement) { if (!(replacement instanceof PyTuple) || replacement.__len__() != 2 || !(replacement.__getitem__(0) instanceof PyBaseString) || !(replacement.__getitem__(1) instanceof PyInteger)) { throw new PyException(Py.TypeError, "error_handler " + errors + " must return a tuple of (replacement, new position)"); } } } class StringSubsequenceIterator implements Iterator { private final String s; private int current, k, start, stop, step; StringSubsequenceIterator(String s, int start, int stop, int step) { // System.out.println("s=" + s.length() + ",start=" + start + ",stop=" + stop); this.s = s; k = 0; current = start; this.start = start; this.stop = stop; this.step = step; // this bounds checking is necessary to convert between use of code units elsewhere, and codepoints here // it would be nice if it were unnecessary! int count = getCodePointCount(s); if (start >= count) { this.stop = -1; } else if (stop >= count) { this.stop = count; } for (int i = 0; i < start; i++) { nextCodePoint(); } } StringSubsequenceIterator(String s) { this(s, 0, getCodePointCount(s), 1); } private static int getCodePointCount(String s) { return s.codePointCount(0, s.length()); } public boolean hasNext() { return current < stop; } public Object next() { int codePoint = nextCodePoint(); current += 1; for (int j = 1; j < step && hasNext(); j++) { nextCodePoint(); current += 1; } return codePoint; } private int nextCodePoint() { int U; // System.out.println("k=" + k); int W1 = s.charAt(k); if (W1 >= 0xD800 && W1 < 0xDC00) { int W2 = s.charAt(k + 1); U = (((W1 & 0x3FF) << 10) | (W2 & 0x3FF)) + 0x10000; k += 2; } else { U = W1; k += 1; } return U; } public void remove() { throw new UnsupportedOperationException("Not supported on String objects (immutable)"); } }