import sys class AppTestCodecs: spaceconfig = { "usemodules": ['unicodedata', 'struct', 'binascii'], } def test_register_noncallable(self): import _codecs raises(TypeError, _codecs.register, 1) def test_bigU_codecs(self): u = u'\U00010001\U00020002\U00030003\U00040004\U00050005' for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', 'utf-32', 'utf-32-le', 'utf-32-be', 'raw_unicode_escape', 'unicode_escape', 'unicode_internal'): assert str(u.encode(encoding),encoding) == u def test_ucs4(self): x = u'\U00100000' y = x.encode("raw-unicode-escape").decode("raw-unicode-escape") assert x == y def test_named_unicode(self): assert str(b'\\N{SPACE}','unicode-escape') == u" " raises( UnicodeDecodeError, str,b'\\N{SPACE','unicode-escape') raises( UnicodeDecodeError, str,b'\\NSPACE}','unicode-escape') raises( UnicodeDecodeError, str,b'\\NSPACE','unicode-escape') raises( UnicodeDecodeError, str,b'\\N','unicode-escape') assert str(b'\\N{SPACE}\\N{SPACE}','unicode-escape') == u" " assert str(b'\\N{SPACE}a\\N{SPACE}','unicode-escape') == u" a " assert b"\\N{foo}xx".decode("unicode-escape", "ignore") == u"xx" assert 1 <= len(u"\N{CJK UNIFIED IDEOGRAPH-20000}") <= 2 def test_literals(self): raises(SyntaxError, eval, 'u\'\\Uffffffff\'') def test_insecure_pickle(self): import pickle insecure = [b"abc", b"2 + 2", # not quoted #"'abc' + 'def'", # not a single quoted string b"'abc", # quote is not closed b"'abc\"", # open quote and close quote don't match b"'abc' ?", # junk after close quote b"'\\'", # trailing backslash # some tests of the quoting rules #"'abc\"\''", #"'\\\\a\'\'\'\\\'\\\\\''", ] for s in insecure: buf = b"S" + s + b"\012p0\012." raises ((ValueError, pickle.UnpicklingError), pickle.loads, buf) def test_unicodedecodeerror(self): assert str(UnicodeDecodeError( "ascii", b"g\xfcrk", 1, 2, "ouch")) == "'ascii' codec can't decode byte 0xfc in position 1: ouch" assert str(UnicodeDecodeError( "ascii", b"g\xfcrk", 1, 3, "ouch")) == "'ascii' codec can't decode bytes in position 1-2: ouch" def test_unicodetranslateerror(self): import sys assert str(UnicodeTranslateError( "g\xfcrk", 1, 2, "ouch"))== "can't translate character '\\xfc' in position 1: ouch" assert str(UnicodeTranslateError( "g\u0100rk", 1, 2, "ouch"))== "can't translate character '\\u0100' in position 1: ouch" assert str(UnicodeTranslateError( "g\uffffrk", 1, 2, "ouch"))== "can't translate character '\\uffff' in position 1: ouch" if sys.maxunicode > 0xffff and len(chr(0x10000)) == 1: assert str(UnicodeTranslateError( "g\U00010000rk", 1, 2, "ouch"))== "can't translate character '\\U00010000' in position 1: ouch" assert str(UnicodeTranslateError( "g\xfcrk", 1, 3, "ouch"))=="can't translate characters in position 1-2: ouch" def test_unicodeencodeerror(self): import sys assert str(UnicodeEncodeError( "ascii", "g\xfcrk", 1, 2, "ouch"))=="'ascii' codec can't encode character '\\xfc' in position 1: ouch" assert str(UnicodeEncodeError( "ascii", "g\xfcrk", 1, 4, "ouch"))== "'ascii' codec can't encode characters in position 1-3: ouch" assert str(UnicodeEncodeError( "ascii", "\xfcx", 0, 1, "ouch"))=="'ascii' codec can't encode character '\\xfc' in position 0: ouch" assert str(UnicodeEncodeError( "ascii", "\u0100x", 0, 1, "ouch"))=="'ascii' codec can't encode character '\\u0100' in position 0: ouch" assert str(UnicodeEncodeError( "ascii", "\uffffx", 0, 1, "ouch"))=="'ascii' codec can't encode character '\\uffff' in position 0: ouch" if sys.maxunicode > 0xffff and len(chr(0x10000)) == 1: assert str(UnicodeEncodeError( "ascii", "\U00010000x", 0, 1, "ouch")) =="'ascii' codec can't encode character '\\U00010000' in position 0: ouch" def test_indexerror(self): import _codecs test = b"\\" # trailing backslash raises (ValueError, _codecs.escape_decode, test) def test_charmap_decode(self): from _codecs import charmap_decode import sys assert charmap_decode(b'', 'strict', 'blablabla') == ('', 0) assert charmap_decode(b'xxx') == ('xxx', 3) assert charmap_decode(b'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 3) map = tuple([chr(i) for i in range(256)]) assert charmap_decode(b'xxx\xff', 'strict', map) == ('xxx\xff', 4) exc = raises(TypeError, charmap_decode, b'\xff', "strict", {0xff: b'a'}) assert str(exc.value) == "character mapping must return integer, None or str" raises(TypeError, charmap_decode, b'\xff', "strict", {0xff: 0x110000}) assert (charmap_decode(b"\x00\x01\x02", "strict", {0: 0x10FFFF, 1: ord('b'), 2: ord('c')}) == (u"\U0010FFFFbc", 3)) assert (charmap_decode(b"\x00\x01\x02", "strict", {0: u'\U0010FFFF', 1: u'b', 2: u'c'}) == (u"\U0010FFFFbc", 3)) def test_escape_decode_errors(self): from _codecs import escape_decode as decode raises(ValueError, decode, br"\x") raises(ValueError, decode, br"[\x]") assert decode(br"[\x]\x", "ignore") == (b"[]", 6) assert decode(br"[\x]\x", "replace") == (b"[?]?", 6) raises(ValueError, decode, br"\x0") raises(ValueError, decode, br"[\x0]") assert decode(br"[\x0]\x0", "ignore") == (b"[]", 8) assert decode(br"[\x0]\x0", "replace") == (b"[?]?", 8) def test_unicode_escape(self): from _codecs import unicode_escape_encode, unicode_escape_decode assert unicode_escape_encode('abc') == ('abc'.encode('unicode_escape'), 3) assert unicode_escape_decode(b'abc') == (b'abc'.decode('unicode_escape'), 3) assert unicode_escape_decode(b'\\x61\\x62\\x63') == ('abc', 12) class AppTestPartialEvaluation: spaceconfig = dict(usemodules=['array',]) def test_partial_utf8(self): import _codecs encoding = 'utf-8' check_partial = [ u"\x00", u"\x00", u"\x00\xff", u"\x00\xff", u"\x00\xff\u07ff", u"\x00\xff\u07ff", u"\x00\xff\u07ff", u"\x00\xff\u07ff\u0800", u"\x00\xff\u07ff\u0800", u"\x00\xff\u07ff\u0800", u"\x00\xff\u07ff\u0800\uffff", u"\x00\xff\u07ff\u0800\uffff", u"\x00\xff\u07ff\u0800\uffff", u"\x00\xff\u07ff\u0800\uffff", u"\x00\xff\u07ff\u0800\uffff\U00010000", ] buffer = b'' result = "" for (c, partialresult) in zip("\x00\xff\u07ff\u0800\uffff\U00010000".encode(encoding), check_partial): buffer += bytes([c]) res = _codecs.utf_8_decode(buffer,'strict',False) if res[1] >0 : buffer = b'' result += res[0] assert result == partialresult def test_partial_utf16(self): import _codecs encoding = 'utf-16' check_partial = [ u"", # first byte of BOM read u"", # second byte of BOM read => byteorder known u"", u"\x00", u"\x00", u"\x00\xff", u"\x00\xff", u"\x00\xff\u0100", u"\x00\xff\u0100", u"\x00\xff\u0100\uffff", u"\x00\xff\u0100\uffff", u"\x00\xff\u0100\uffff", u"\x00\xff\u0100\uffff", u"\x00\xff\u0100\uffff\U00010000", ] buffer = b'' result = "" for (c, partialresult) in zip("\x00\xff\u0100\uffff\U00010000".encode(encoding), check_partial): buffer += bytes([c]) res = _codecs.utf_16_decode(buffer,'strict',False) if res[1] >0 : buffer = b'' result += res[0] assert result == partialresult def test_bug1098990_a(self): import codecs, io self.encoding = 'utf-8' s1 = u"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n" s2 = u"offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n" s3 = u"next line.\r\n" s = (s1+s2+s3).encode(self.encoding) stream = io.BytesIO(s) reader = codecs.getreader(self.encoding)(stream) assert reader.readline() == s1 assert reader.readline() == s2 assert reader.readline() == s3 assert reader.readline() == u"" def test_bug1098990_b(self): import codecs, io self.encoding = 'utf-8' s1 = u"aaaaaaaaaaaaaaaaaaaaaaaa\r\n" s2 = u"bbbbbbbbbbbbbbbbbbbbbbbb\r\n" s3 = u"stillokay:bbbbxx\r\n" s4 = u"broken!!!!badbad\r\n" s5 = u"againokay.\r\n" s = (s1+s2+s3+s4+s5).encode(self.encoding) stream = io.BytesIO(s) reader = codecs.getreader(self.encoding)(stream) assert reader.readline() == s1 assert reader.readline() == s2 assert reader.readline() == s3 assert reader.readline() == s4 assert reader.readline() == s5 assert reader.readline() == u"" def test_seek_utf16le(self): # all codecs should be able to encode these import codecs, io encoding = 'utf-16-le' s = "%s\n%s\n" % (10*"abc123", 10*"def456") reader = codecs.getreader(encoding)(io.BytesIO(s.encode(encoding))) for t in range(5): # Test that calling seek resets the internal codec state and buffers reader.seek(0, 0) line = reader.readline() assert s[:len(line)] == line def test_unicode_internal_encode(self): import sys class U(str): pass enc = U("a").encode("unicode_internal") if sys.maxunicode == 65535: # UCS2 build if sys.byteorder == "big": assert enc == b"\x00a" else: assert enc == b"a\x00" elif len("\U00010098") == 1: # UCS4 build on a UCS4 CPython enc2 = "\U00010098".encode("unicode_internal") if sys.byteorder == "big": assert enc == b"\x00\x00\x00a" assert enc2 == b"\x00\x01\x00\x98" else: assert enc == b"a\x00\x00\x00" assert enc2 == b"\x98\x00\x01\x00" else: # UCS4 build on a UCS2 CPython if sys.byteorder == "big": assert enc == b"\x00\x00\x00a" else: assert enc == b"a\x00\x00\x00" def test_unicode_internal_decode(self): import sys, _codecs, array if sys.maxunicode == 65535: # UCS2 build if sys.byteorder == "big": bytes = b"\x00a" else: bytes = b"a\x00" else: # UCS4 build if sys.byteorder == "big": bytes = b"\x00\x00\x00a" bytes2 = b"\x00\x01\x00\x98" else: bytes = b"a\x00\x00\x00" bytes2 = b"\x98\x00\x01\x00" assert bytes2.decode("unicode_internal") == "\U00010098" assert bytes.decode("unicode_internal") == "a" assert _codecs.unicode_internal_decode(array.array('b', bytes))[0] == u"a" assert _codecs.unicode_internal_decode(memoryview(bytes))[0] == u"a" # This codec accepts bytes and unicode on both sides _codecs.unicode_internal_decode(u'\0\0\0\0') _codecs.unicode_internal_decode(b'\0\0\0\0') _codecs.unicode_internal_encode(u'\0\0\0\0') _codecs.unicode_internal_encode(b'\0\0\0\0') def test_raw_unicode_escape(self): import _codecs assert str(b"\u0663", "raw-unicode-escape") == "\u0663" assert "\u0663".encode("raw-unicode-escape") == b"\u0663" assert _codecs.raw_unicode_escape_decode(r"\u1234") == ("\u1234", 6) def test_escape_decode(self): import _codecs test = _codecs.escape_encode(b'a\n\\b\x00c\td\u2045')[0] assert _codecs.escape_decode(test)[0] == b'a\n\\b\x00c\td\u2045' assert _codecs.escape_decode(b'\\077')[0] == b'?' assert _codecs.escape_decode(b'\\100')[0] == b'@' assert _codecs.escape_decode(b'\\253')[0] == bytes([0o253]) assert _codecs.escape_decode(b'\\312')[0] == bytes([0o312]) def test_escape_decode_wrap_around(self): import _codecs assert _codecs.escape_decode(b'\\400')[0] == b'\0' def test_escape_decode_ignore_invalid(self): import _codecs assert _codecs.escape_decode(b'\\9')[0] == b'\\9' assert _codecs.escape_decode(b'\\01')[0] == b'\x01' assert _codecs.escape_decode(b'\\0f')[0] == b'\0' + b'f' assert _codecs.escape_decode(b'\\08')[0] == b'\0' + b'8' def test_escape_decode_errors(self): import _codecs raises(ValueError, _codecs.escape_decode, br"\x") raises(ValueError, _codecs.escape_decode, br"[\x]") raises(ValueError, _codecs.escape_decode, br"\x0") raises(ValueError, _codecs.escape_decode, br"[\x0]") def test_unicode_escape_decode_errors(self): from _codecs import unicode_escape_decode, raw_unicode_escape_decode for decode in [unicode_escape_decode, raw_unicode_escape_decode]: for c, d in ('u', 4), ('U', 4): for i in range(d): raises(UnicodeDecodeError, decode, "\\" + c + "0"*i) raises(UnicodeDecodeError, decode, "[\\" + c + "0"*i + "]") data = "[\\" + c + "0"*i + "]\\" + c + "0"*i assert decode(data, "ignore") == (u"[]", len(data)) assert decode(data, "replace") == (u"[\ufffd]\ufffd", len(data)) raises(UnicodeDecodeError, decode, r"\U00110000") assert decode(r"\U00110000", "ignore") == (u"", 10) assert decode(r"\U00110000", "replace") == (u"\ufffd", 10) exc = raises(UnicodeDecodeError, unicode_escape_decode, b"\u1z32z3", 'strict') assert str(exc.value) == r"'unicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX escape" exc = raises(UnicodeDecodeError, raw_unicode_escape_decode, b"\u1z32z3", 'strict') assert str(exc.value) == r"'rawunicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX" exc = raises(UnicodeDecodeError, raw_unicode_escape_decode, b"\U1z32z3", 'strict') assert str(exc.value) == r"'rawunicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX" def test_escape_encode(self): import _codecs assert _codecs.escape_encode(b'"')[0] == b'"' assert _codecs.escape_encode(b"'")[0] == b"\\'" def test_decode_utf8_different_case(self): constant = "a" assert constant.encode("utf-8") == constant.encode("UTF-8") def test_codec_wrong_result(self): import _codecs def search_function(encoding): def f(input, errors="strict"): return 42 print(encoding) if encoding == 'test.mytestenc': return (f, f, None, None) return None _codecs.register(search_function) raises(TypeError, b"hello".decode, "test.mytestenc") raises(TypeError, "hello".encode, "test.mytestenc") def test_codec_wrapped_exception(self): import _codecs def search_function(encoding): def f(input, errors="strict"): raise to_raise if encoding == 'test.failingenc': return (f, f, None, None) return None _codecs.register(search_function) to_raise = RuntimeError('should be wrapped') exc = raises(RuntimeError, b"hello".decode, "test.failingenc") assert str(exc.value) == ( "decoding with 'test.failingenc' codec failed " "(RuntimeError: should be wrapped)") exc = raises(RuntimeError, u"hello".encode, "test.failingenc") assert str(exc.value) == ( "encoding with 'test.failingenc' codec failed " "(RuntimeError: should be wrapped)") # to_raise.attr = "don't wrap" exc = raises(RuntimeError, u"hello".encode, "test.failingenc") assert exc.value == to_raise # to_raise = RuntimeError("Should", "Not", "Wrap") exc = raises(RuntimeError, u"hello".encode, "test.failingenc") assert exc.value == to_raise def test_cpytest_decode(self): import codecs assert codecs.decode(b'\xe4\xf6\xfc', 'latin-1') == '\xe4\xf6\xfc' raises(TypeError, codecs.decode) assert codecs.decode(b'abc') == 'abc' exc = raises(UnicodeDecodeError, codecs.decode, b'\xff', 'ascii') def test_bad_errorhandler_return(self): import codecs def baddecodereturn1(exc): return 42 codecs.register_error("test.baddecodereturn1", baddecodereturn1) raises(TypeError, b"\xff".decode, "ascii", "test.baddecodereturn1") raises(TypeError, b"\\".decode, "unicode-escape", "test.baddecodereturn1") raises(TypeError, b"\\x0".decode, "unicode-escape", "test.baddecodereturn1") raises(TypeError, b"\\x0y".decode, "unicode-escape", "test.baddecodereturn1") raises(TypeError, b"\\Uffffeeee".decode, "unicode-escape", "test.baddecodereturn1") raises(TypeError, b"\\uyyyy".decode, "raw-unicode-escape", "test.baddecodereturn1") def test_cpy_bug1175396(self): import codecs, io s = [ '<%!--===================================================\r\n', ' BLOG index page: show recent articles,\r\n', ' today\'s articles, or articles of a specific date.\r\n', '========================================================--%>\r\n', '<%@inputencoding="ISO-8859-1"%>\r\n', '<%@pagetemplate=TEMPLATE.y%>\r\n', '<%@import=import frog.util, frog%>\r\n', '<%@import=import frog.objects%>\r\n', '<%@import=from frog.storageerrors import StorageError%>\r\n', '<%\r\n', '\r\n', 'import logging\r\n', 'log=logging.getLogger("Snakelets.logger")\r\n', '\r\n', '\r\n', 'user=self.SessionCtx.user\r\n', 'storageEngine=self.SessionCtx.storageEngine\r\n', '\r\n', '\r\n', 'def readArticlesFromDate(date, count=None):\r\n', ' entryids=storageEngine.listBlogEntries(date)\r\n', ' entryids.reverse() # descending\r\n', ' if count:\r\n', ' entryids=entryids[:count]\r\n', ' try:\r\n', ' return [ frog.objects.BlogEntry.load(storageEngine, date, Id) for Id in entryids ]\r\n', ' except StorageError,x:\r\n', ' log.error("Error loading articles: "+str(x))\r\n', ' self.abort("cannot load articles")\r\n', ] stream = io.BytesIO("".join(s).encode("utf7")) assert b"aborrt" not in stream.getvalue() reader = codecs.getreader("utf7")(stream) for (i, line) in enumerate(reader): assert line == s[i] def test_buffer_encode(self): import _codecs, array assert (_codecs.readbuffer_encode(array.array('b', b'spam')) == (b'spam', 4)) assert _codecs.readbuffer_encode(u"test") == (b'test', 4) assert _codecs.readbuffer_encode("") == (b"", 0) def test_utf8sig(self): import codecs d = codecs.getincrementaldecoder("utf-8-sig")() s = "spam" assert d.decode(s.encode("utf-8-sig")) == s def test_decoder_state(self): import codecs encoding = 'utf16' u = 'abc123' s = u.encode(encoding) for i in range(len(u) + 1): d = codecs.getincrementalencoder(encoding)() part1 = d.encode(u[:i]) state = d.getstate() d = codecs.getincrementalencoder(encoding)() d.setstate(state) part2 = d.encode(u[i:], True) assert s == part1 + part2 def test_escape_decode_escaped_newline(self): import _codecs s = b'\\\n' decoded = _codecs.unicode_escape_decode(s)[0] assert decoded == '' def test_charmap_decode_1(self): import codecs assert codecs.charmap_encode('xxx') == (b'xxx', 3) assert codecs.charmap_encode('xxx', 'strict', {ord('x'): b'XX'}) == (b'XXXXXX', 3) res = codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab") assert res == ("ab\ufffd", 3) res = codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab\ufffe") assert res == ('ab\ufffd', 3) def test_decode_errors(self): import sys if sys.maxunicode > 0xffff: try: b"\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal") except UnicodeDecodeError as ex: assert "unicode_internal" == ex.encoding assert b"\x00\x00\x00\x00\x00\x11\x11\x00" == ex.object assert ex.start == 4 assert ex.end == 8 else: raise Exception("DID NOT RAISE") def test_errors(self): import codecs assert codecs.replace_errors(UnicodeEncodeError( "ascii", "\u3042", 0, 1, "ouch")) == ("?", 1) assert codecs.replace_errors(UnicodeDecodeError( "ascii", b"\xff", 0, 1, "ouch")) == ("\ufffd", 1) assert codecs.replace_errors(UnicodeTranslateError( "\u3042", 0, 1, "ouch")) == ("\ufffd", 1) assert codecs.replace_errors(UnicodeEncodeError( "ascii", "\u3042\u3042", 0, 2, "ouch")) == ("??", 2) assert codecs.replace_errors(UnicodeDecodeError( "ascii", b"\xff\xff", 0, 2, "ouch")) == ("\ufffd", 2) assert codecs.replace_errors(UnicodeTranslateError( "\u3042\u3042", 0, 2, "ouch")) == ("\ufffd\ufffd", 2) class BadStartUnicodeEncodeError(UnicodeEncodeError): def __init__(self): UnicodeEncodeError.__init__(self, "ascii", "", 0, 1, "bad") self.start = [] # A UnicodeEncodeError object with a bad object attribute class BadObjectUnicodeEncodeError(UnicodeEncodeError): def __init__(self): UnicodeEncodeError.__init__(self, "ascii", "", 0, 1, "bad") self.object = [] # A UnicodeDecodeError object without an end attribute class NoEndUnicodeDecodeError(UnicodeDecodeError): def __init__(self): UnicodeDecodeError.__init__(self, "ascii", b"", 0, 1, "bad") del self.end # A UnicodeDecodeError object with a bad object attribute class BadObjectUnicodeDecodeError(UnicodeDecodeError): def __init__(self): UnicodeDecodeError.__init__(self, "ascii", b"", 0, 1, "bad") self.object = [] # A UnicodeTranslateError object without a start attribute class NoStartUnicodeTranslateError(UnicodeTranslateError): def __init__(self): UnicodeTranslateError.__init__(self, "", 0, 1, "bad") del self.start # A UnicodeTranslateError object without an end attribute class NoEndUnicodeTranslateError(UnicodeTranslateError): def __init__(self): UnicodeTranslateError.__init__(self, "", 0, 1, "bad") del self.end # A UnicodeTranslateError object without an object attribute class NoObjectUnicodeTranslateError(UnicodeTranslateError): def __init__(self): UnicodeTranslateError.__init__(self, "", 0, 1, "bad") del self.object import codecs raises(TypeError, codecs.replace_errors, BadObjectUnicodeEncodeError()) raises(TypeError, codecs.replace_errors, 42) # "replace" complains about the wrong exception type raises(TypeError, codecs.replace_errors, UnicodeError("ouch")) raises(TypeError, codecs.replace_errors, BadObjectUnicodeEncodeError()) raises(TypeError, codecs.replace_errors, BadObjectUnicodeDecodeError() ) # With the correct exception, "replace" returns an "?" or "\ufffd" replacement def test_decode_ignore(self): assert b'\xff'.decode('utf-7', 'ignore') == '' assert b'\x00'.decode('unicode-internal', 'ignore') == '' def test_backslashreplace(self): import sys import codecs sin = u"a\xac\u1234\u20ac\u8000\U0010ffff" if sys.maxunicode > 65535: expected_ascii = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" expected_8859 = b"a\xac\\u1234\xa4\\u8000\\U0010ffff" else: expected_ascii = b"a\\xac\\u1234\\u20ac\\u8000\\udbff\\udfff" expected_8859 = b"a\xac\\u1234\xa4\\u8000\\udbff\\udfff" assert sin.encode('ascii', 'backslashreplace') == expected_ascii assert sin.encode("iso-8859-15", "backslashreplace") == expected_8859 assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == b'a\\xac\u1234\u20ac\u8000' assert b'\x00\x60\x80'.decode( 'ascii', 'backslashreplace') == u'\x00\x60\\x80' assert codecs.charmap_decode( b"\x00\x01\x02", "backslashreplace", "ab") == ("ab\\x02", 3) def test_namereplace(self): assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'namereplace') == ( b'a\\N{NOT SIGN}\\N{ETHIOPIC SYLLABLE SEE}\\N{EURO SIGN}' b'\\N{CJK UNIFIED IDEOGRAPH-8000}') assert '[\uDC80]'.encode('utf-8', 'namereplace') == b'[\\udc80]' def test_surrogateescape(self): assert b'a\x80b'.decode('utf-8', 'surrogateescape') == 'a\udc80b' assert 'a\udc80b'.encode('utf-8', 'surrogateescape') == b'a\x80b' for enc in ('utf-8', 'ascii', 'latin-1', 'charmap'): assert '\udcc3'.encode(enc, 'surrogateescape') == b'\xc3' def test_surrogatepass_handler(self): import _codecs assert _codecs.lookup_error("surrogatepass") assert ("abc\ud800def".encode("utf-8", "surrogatepass") == b"abc\xed\xa0\x80def") assert (b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass") == "abc\ud800def") assert ('surrogate:\udcff'.encode("utf-8", "surrogatepass") == b'surrogate:\xed\xb3\xbf') assert (b'surrogate:\xed\xb3\xbf'.decode("utf-8", "surrogatepass") == 'surrogate:\udcff') raises(UnicodeDecodeError, b"abc\xed\xa0".decode, "utf-8", "surrogatepass") raises(UnicodeDecodeError, b"abc\xed\xa0z".decode, "utf-8", "surrogatepass") assert u'\ud8ae'.encode('utf_16_be', 'surrogatepass') == b'\xd8\xae' assert (u'\U0000d8ae'.encode('utf-32-be', 'surrogatepass') == b'\x00\x00\xd8\xae') def test_badandgoodsurrogatepassexceptions(self): import codecs surrogatepass_errors = codecs.lookup_error('surrogatepass') # "surrogatepass" complains about a non-exception passed in raises(TypeError, surrogatepass_errors, 42) # "surrogatepass" complains about the wrong exception types raises(TypeError, surrogatepass_errors, UnicodeError("ouch")) # "surrogatepass" can not be used for translating raises(TypeError, surrogatepass_errors, UnicodeTranslateError("\ud800", 0, 1, "ouch")) # Use the correct exception for enc in ("utf-8", "utf-16le", "utf-16be", "utf-32le", "utf-32be"): raises(UnicodeEncodeError, surrogatepass_errors, UnicodeEncodeError(enc, "a", 0, 1, "ouch")) raises(UnicodeDecodeError, surrogatepass_errors, UnicodeDecodeError(enc, "a".encode(enc), 0, 1, "ouch")) for s in ("\ud800", "\udfff", "\ud800\udfff"): raises(UnicodeEncodeError, surrogatepass_errors, UnicodeEncodeError("ascii", s, 0, len(s), "ouch")) tests = [ ("utf-8", "\ud800", b'\xed\xa0\x80', 3), ("utf-16le", "\ud800", b'\x00\xd8', 2), ("utf-16be", "\ud800", b'\xd8\x00', 2), ("utf-32le", "\ud800", b'\x00\xd8\x00\x00', 4), ("utf-32be", "\ud800", b'\x00\x00\xd8\x00', 4), ("utf-8", "\udfff", b'\xed\xbf\xbf', 3), ("utf-16le", "\udfff", b'\xff\xdf', 2), ("utf-16be", "\udfff", b'\xdf\xff', 2), ("utf-32le", "\udfff", b'\xff\xdf\x00\x00', 4), ("utf-32be", "\udfff", b'\x00\x00\xdf\xff', 4), ("utf-8", "\ud800\udfff", b'\xed\xa0\x80\xed\xbf\xbf', 3), ("utf-16le", "\ud800\udfff", b'\x00\xd8\xff\xdf', 2), ("utf-16be", "\ud800\udfff", b'\xd8\x00\xdf\xff', 2), ("utf-32le", "\ud800\udfff", b'\x00\xd8\x00\x00\xff\xdf\x00\x00', 4), ("utf-32be", "\ud800\udfff", b'\x00\x00\xd8\x00\x00\x00\xdf\xff', 4), ] for enc, s, b, n in tests: assert surrogatepass_errors( UnicodeEncodeError(enc, "a" + s + "b", 1, 1 + len(s), "ouch") ) == (b, 1 + len(s)) assert surrogatepass_errors( UnicodeDecodeError(enc, bytearray(b"a" + b[:n] + b"b"), 1, 1 + n, "ouch") ) == (s[:1], 1 + n) def test_badhandler(self): import codecs results = ( 42, "foo", (1,2,3), ("foo", 1, 3), ("foo", None), ("foo",), ("foo", 1, 3), ("foo", None), ("foo",) ) encs = ("ascii", "latin-1", "iso-8859-1", "iso-8859-15") for res in results: codecs.register_error("test.badhandler", lambda x: res) for enc in encs: raises( TypeError, "\u3042".encode, enc, "test.badhandler" ) for (enc, bytes) in ( ("utf-8", b"\xff"), ("ascii", b"\xff"), ("utf-7", b"+x-"), ("unicode-internal", b"\x00"), ): raises( TypeError, bytes.decode, enc, "test.badhandler" ) def test_badhandler_longindex(self): import codecs import sys errors = 'test.badhandler_longindex' codecs.register_error(errors, lambda x: ('', sys.maxsize + 1)) # CPython raises OverflowError here raises((IndexError, OverflowError), b'apple\x92ham\x93spam'.decode, 'utf-8', errors) def test_unicode_internal(self): import codecs import sys try: b'\x00'.decode('unicode-internal') except UnicodeDecodeError: pass else: raise Exception("DID NOT RAISE") res = b"\x00\x00\x00\x00\x00".decode("unicode-internal", "replace") if sys.maxunicode > 65535: assert res == "\u0000\ufffd" # UCS4 build else: assert res == "\x00\x00\ufffd" # UCS2 build res = b"\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore") if sys.maxunicode > 65535: assert res == "\u0000" # UCS4 build else: assert res == "\x00\x00" # UCS2 build def handler_unicodeinternal(exc): if not isinstance(exc, UnicodeDecodeError): raise TypeError("don't know how to handle %r" % exc) return (u"\x01", 1) codecs.register_error("test.hui", handler_unicodeinternal) res = b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui") if sys.maxunicode > 65535: assert res == "\u0000\u0001\u0000" # UCS4 build else: assert res == "\x00\x00\x01\x00\x00" # UCS2 build def handler1(exc): if not isinstance(exc, UnicodeEncodeError) \ and not isinstance(exc, UnicodeDecodeError): raise TypeError("don't know how to handle %r" % exc) l = [u"<%d>" % exc.object[pos] for pos in range(exc.start, exc.end)] return (u"[%s]" % u"".join(l), exc.end) codecs.register_error("test.handler1", handler1) assert b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1") == \ u"\u3042[<92><117><51>]xxx" def test_unicode_internal_error_handler_infinite_loop(self): import codecs class MyException(Exception): pass seen = [0] def handler_unicodeinternal(exc): if not isinstance(exc, UnicodeDecodeError): raise TypeError("don't know how to handle %r" % exc) seen[0] += 1 if seen[0] == 20: # stop the 20th time this is called raise MyException return (u"\x01", 4) # 4 < len(input), so will try and fail again codecs.register_error("test.inf", handler_unicodeinternal) try: b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.inf") except MyException: pass else: raise AssertionError("should have gone into infinite loop") def test_encode_error_bad_handler(self): import codecs codecs.register_error("test.bad_handler", lambda e: (repl, 1)) assert "xyz".encode("latin-1", "test.bad_handler") == b"xyz" repl = "\u1234" raises(UnicodeEncodeError, "\u5678".encode, "latin-1", "test.bad_handler") repl = "\u00E9" s = "\u5678".encode("latin-1", "test.bad_handler") assert s == b'\xe9' def test_lone_surrogates(self): encodings = ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', 'utf-32', 'utf-32-le', 'utf-32-be') for encoding in encodings: raises(UnicodeEncodeError, u'\ud800'.encode, encoding) assert (u'[\udc80]'.encode(encoding, "backslashreplace") == '[\\udc80]'.encode(encoding)) assert (u'[\udc80]'.encode(encoding, "ignore") == '[]'.encode(encoding)) assert (u'[\udc80]'.encode(encoding, "replace") == '[?]'.encode(encoding)) for encoding, ill_surrogate in [('utf-8', b'\xed\xb2\x80'), ('utf-16-le', b'\x80\xdc'), ('utf-16-be', b'\xdc\x80'), ('utf-32-le', b'\x80\xdc\x00\x00'), ('utf-32-be', b'\x00\x00\xdc\x80')]: print(encoding) before, after = "[", "]" before_sequence = before.encode(encoding) after_sequence = after.encode(encoding) test_string = before + "\uDC80" + after test_sequence = before_sequence + ill_surrogate + after_sequence raises(UnicodeDecodeError, test_sequence.decode, encoding) def test_charmap_encode(self): assert 'xxx'.encode('charmap') == b'xxx' import codecs exc = raises(TypeError, codecs.charmap_encode, u'\xff', "replace", {0xff: 300}) assert str(exc.value) == 'character mapping must be in range(256)' exc = raises(TypeError, codecs.charmap_encode, u'\xff', "replace", {0xff: u'a'}) assert str(exc.value) == 'character mapping must return integer, bytes or None, not str' raises(UnicodeError, codecs.charmap_encode, u"\xff", "replace", {0xff: None}) def test_charmap_encode_replace(self): charmap = dict([(c, bytes([c, c]).upper()) for c in b"abcdefgh"]) charmap[ord("?")] = b"XYZ" import codecs sin = "abcDEF" sout = codecs.charmap_encode(sin, "replace", charmap)[0] assert sout == b"AABBCCXYZXYZXYZ" def test_charmap_decode_2(self): assert b'foo'.decode('charmap') == 'foo' def test_charmap_build(self): import codecs assert codecs.charmap_build('123456') == {49: 0, 50: 1, 51: 2, 52: 3, 53: 4, 54: 5} def test_utf7_start_end_in_exception(self): try: b'+IC'.decode('utf-7') except UnicodeDecodeError as exc: assert exc.start == 0 assert exc.end == 3 def test_utf7_surrogate(self): assert b'+3ADYAA-'.decode('utf-7') == '\udc00\ud800' def test_utf7_errors(self): import codecs tests = [ (b'a\xffb', u'a\ufffdb'), (b'a+IK', u'a\ufffd'), (b'a+IK-b', u'a\ufffdb'), (b'a+IK,b', u'a\ufffdb'), (b'a+IKx', u'a\u20ac\ufffd'), (b'a+IKx-b', u'a\u20ac\ufffdb'), (b'a+IKwgr', u'a\u20ac\ufffd'), (b'a+IKwgr-b', u'a\u20ac\ufffdb'), (b'a+IKwgr,', u'a\u20ac\ufffd'), (b'a+IKwgr,-b', u'a\u20ac\ufffd-b'), (b'a+IKwgrB', u'a\u20ac\u20ac\ufffd'), (b'a+IKwgrB-b', u'a\u20ac\u20ac\ufffdb'), (b'a+/,+IKw-b', u'a\ufffd\u20acb'), (b'a+//,+IKw-b', u'a\ufffd\u20acb'), (b'a+///,+IKw-b', u'a\uffff\ufffd\u20acb'), (b'a+////,+IKw-b', u'a\uffff\ufffd\u20acb'), (b'a+2AE\xe1b', u'a\ufffdb'), (b'a+2AEA-b', u'a\ufffdb'), (b'a+2AH-b', u'a\ufffdb'), ] for raw, expected in tests: raises(UnicodeDecodeError, codecs.utf_7_decode, raw, 'strict', True) assert raw.decode('utf-7', 'replace') == expected def test_utf_16_encode_decode(self): import codecs, sys x = '123abc' if sys.byteorder == 'big': assert codecs.getencoder('utf-16')(x) == ( b'\xfe\xff\x001\x002\x003\x00a\x00b\x00c', 6) assert codecs.getdecoder('utf-16')( b'\xfe\xff\x001\x002\x003\x00a\x00b\x00c') == (x, 14) else: assert codecs.getencoder('utf-16')(x) == ( b'\xff\xfe1\x002\x003\x00a\x00b\x00c\x00', 6) assert codecs.getdecoder('utf-16')( b'\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14) def test_unicode_escape(self): import _codecs assert '\\'.encode('unicode-escape') == b'\\\\' assert b'\\\\'.decode('unicode-escape') == '\\' assert '\ud801'.encode('unicode-escape') == b'\\ud801' assert '\u0013'.encode('unicode-escape') == b'\\x13' assert _codecs.unicode_escape_decode(r"\u1234") == ("\u1234", 6) def test_mbcs(self): import sys if sys.platform != 'win32': return toencode = u'caf\xe9', b'caf\xe9' try: # test for non-latin1 codepage, more general test needed import winreg key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r'System\CurrentControlSet\Control\Nls\CodePage') if winreg.QueryValueEx(key, 'ACP')[0] == u'1255': # non-latin1 toencode = u'caf\xbf',b'caf\xbf' except: assert False, 'cannot test mbcs on this windows system, check code page' assert u'test'.encode('mbcs') == b'test' assert toencode[0].encode('mbcs') == toencode[1] raises(UnicodeEncodeError, u'\u040a'.encode, 'mbcs') assert b'cafx\e9'.decode('mbcs') == u'cafx\e9' def test_handler_string_result(self): import _codecs def f(exc): return (b'foo', exc.end) _codecs.register_error("test.test_codecs_not_a_string", f) result = '\u1234'.encode('ascii', 'test.test_codecs_not_a_string') assert result == b'foo' def test_decode_bytearray(self): import _codecs b = bytearray() assert _codecs.ascii_decode(b) == (u'', 0) assert _codecs.latin_1_decode(b) == (u'', 0) assert _codecs.utf_7_decode(b) == (u'', 0) assert _codecs.utf_8_decode(b) == (u'', 0) assert _codecs.utf_16_be_decode(b) == (u'', 0) assert _codecs.utf_16_decode(b) == (u'', 0) assert _codecs.utf_16_le_decode(b) == (u'', 0) assert _codecs.utf_16_ex_decode(b) == (u'', 0, 0) assert _codecs.utf_32_decode(b) == (u'', 0) assert _codecs.utf_32_be_decode(b) == (u'', 0) assert _codecs.utf_32_le_decode(b) == (u'', 0) assert _codecs.utf_32_ex_decode(b) == (u'', 0, 0) assert _codecs.charmap_decode(b) == (u'', 0) assert _codecs.unicode_escape_decode(b) == (u'', 0) assert _codecs.raw_unicode_escape_decode(b) == (u'', 0) assert _codecs.unicode_internal_decode(b) == (u'', 0) def test_unicode_internal_warnings(self): import codecs, warnings warnings.simplefilter("always") encoder = codecs.getencoder("unicode_internal") decoder = codecs.getdecoder("unicode_internal") warning_msg = "unicode_internal codec has been deprecated" with warnings.catch_warnings(record=True) as w: try: encoder(42) except TypeError: pass assert len(w) == 1 assert str(w[0].message) == warning_msg assert w[0].category == DeprecationWarning with warnings.catch_warnings(record=True) as w: try: decoder(42) except TypeError: pass assert len(w) == 0 with warnings.catch_warnings(record=True) as w: encoded_abc = encoder("abc")[0] assert len(w) == 1 assert str(w[0].message)== warning_msg assert w[0].category == DeprecationWarning with warnings.catch_warnings(record=True) as w: print(type(encoded_abc)) decoder(encoded_abc) assert len(w) == 1 assert str(w[0].message) == warning_msg assert w[0].category == DeprecationWarning def test_xmlcharrefreplace(self): r = u'\u1234\u0080\u2345\u0079\u00AB'.encode('latin1', 'xmlcharrefreplace') assert r == b'ሴ\x80⍅y\xab' r = u'\u1234\u0080\u2345\u0079\u00AB'.encode('ascii', 'xmlcharrefreplace') assert r == b'ሴ€⍅y«' def test_errorhandler_collection(self): import _codecs errors = [] def record_error(exc): if not isinstance(exc, UnicodeEncodeError): raise TypeError("don't know how to handle %r" % exc) errors.append(exc.object[exc.start:exc.end]) return (u'', exc.end) _codecs.register_error("test.record", record_error) sin = u"\xac\u1234\u1234\u20ac\u8000" assert sin.encode("ascii", "test.record") == b"" assert errors == [sin] errors = [] assert sin.encode("latin-1", "test.record") == b"\xac" assert errors == [u'\u1234\u1234\u20ac\u8000'] errors = [] assert sin.encode("iso-8859-15", "test.record") == b"\xac\xa4" assert errors == [u'\u1234\u1234', u'\u8000']