win codecs

Author: youknowone

Lib/_pycodecs.py

@@ -1109,7 +1109,7 @@ def unicode_call_errorhandler(errors,  encoding,
     else:
         exceptionObject = UnicodeEncodeError(encoding, input, startinpos, endinpos, reason)
     res = errorHandler(exceptionObject)
-    if isinstance(res, tuple) and isinstance(res[0], str) and isinstance(res[1], int):
+    if isinstance(res, tuple) and isinstance(res[0], (str, bytes)) and isinstance(res[1], int):
         newpos = res[1]
         if (newpos < 0):
             newpos = len(input) + newpos
@@ -1159,7 +1159,11 @@ def unicode_encode_ucs1(p, size, errors, limit):
             while collend < len(p) and ord(p[collend]) >= limit:
                 collend += 1
             x = unicode_call_errorhandler(errors, encoding, reason, p, collstart, collend, False)
-            res += x[0].encode()
+            replacement = x[0]
+            if isinstance(replacement, bytes):
+                res += replacement
+            else:
+                res += replacement.encode()
             pos = x[1]
 
     return res
@@ -1376,12 +1380,16 @@ def PyUnicode_EncodeCharmap(p, size, mapping='latin-1', errors='strict'):
         except KeyError:
             x = unicode_call_errorhandler(errors, "charmap",
             "character maps to <undefined>", p, inpos, inpos+1, False)
-            try:
-                for y in x[0]:
-                    res += charmapencode_output(ord(y), mapping)
-            except KeyError:
-                raise UnicodeEncodeError("charmap", p, inpos, inpos+1,
-                                        "character maps to <undefined>")
+            replacement = x[0]
+            if isinstance(replacement, bytes):
+                res += list(replacement)
+            else:
+                try:
+                    for y in replacement:
+                        res += charmapencode_output(ord(y), mapping)
+                except KeyError:
+                    raise UnicodeEncodeError("charmap", p, inpos, inpos+1,
+                                            "character maps to <undefined>")
         inpos += 1
     return res
 

Lib/encodings/__init__.py

@@ -172,3 +172,23 @@ def _alias_mbcs(encoding):
             pass
 
     codecs.register(_alias_mbcs)
+
+    from ._win_cp_codecs import create_win32_code_page_codec
+
+    def win32_code_page_search_function(encoding):
+        encoding = encoding.lower()
+        if not encoding.startswith('cp'):
+            return None
+        try:
+            cp = int(encoding[2:])
+        except ValueError:
+            return None
+        # Test if the code page is supported
+        try:
+            codecs.code_page_encode(cp, 'x')
+        except (OverflowError, OSError):
+            return None
+
+        return create_win32_code_page_codec(cp)
+
+    codecs.register(win32_code_page_search_function)

Lib/encodings/_win_cp_codecs.py

@@ -0,0 +1,36 @@
+import codecs
+
+def create_win32_code_page_codec(cp):
+    from codecs import code_page_encode, code_page_decode
+
+    def encode(input, errors='strict'):
+        return code_page_encode(cp, input, errors)
+
+    def decode(input, errors='strict'):
+        return code_page_decode(cp, input, errors, True)
+
+    class IncrementalEncoder(codecs.IncrementalEncoder):
+        def encode(self, input, final=False):
+            return code_page_encode(cp, input, self.errors)[0]
+
+    class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+        def _buffer_decode(self, input, errors, final):
+            return code_page_decode(cp, input, errors, final)
+
+    class StreamWriter(codecs.StreamWriter):
+        def encode(self, input, errors='strict'):
+            return code_page_encode(cp, input, errors)
+
+    class StreamReader(codecs.StreamReader):
+        def decode(self, input, errors, final):
+            return code_page_decode(cp, input, errors, final)
+
+    return codecs.CodecInfo(
+        name=f'cp{cp}',
+        encode=encode,
+        decode=decode,
+        incrementalencoder=IncrementalEncoder,
+        incrementaldecoder=IncrementalDecoder,
+        streamreader=StreamReader,
+        streamwriter=StreamWriter,
+    )

Lib/test/test_codecs.py

@@ -3399,7 +3399,6 @@ def test_invalid_code_page(self):
         self.assertRaises(OSError, codecs.code_page_encode, 123, 'a')
         self.assertRaises(OSError, codecs.code_page_decode, 123, b'a')
 
-    @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
     def test_code_page_name(self):
         self.assertRaisesRegex(UnicodeEncodeError, 'cp932',
             codecs.code_page_encode, 932, '\xff')
@@ -3501,7 +3500,6 @@ def test_cp932(self):
             (b'\x81\x00abc', 'backslashreplace', '\\x81\x00abc'),
         ))
 
-    @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
     def test_cp1252(self):
         self.check_encode(1252, (
             ('abc', 'strict', b'abc'),
@@ -3520,7 +3518,6 @@ def test_cp1252(self):
             (b'\xff', 'strict', '\xff'),
         ))
 
-    @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
     def test_cp708(self):
         self.check_encode(708, (
             ('abc2%', 'strict', b'abc2%'),
@@ -3550,7 +3547,6 @@ def test_cp708(self):
             (b'[\xa0]', 'surrogatepass', None),
         ))
 
-    @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
     def test_cp20106(self):
         self.check_encode(20106, (
             ('abc', 'strict', b'abc'),
@@ -3596,7 +3592,6 @@ def test_cp_utf7(self):
             (b'[\xff]', 'strict', '[\xff]'),
         ))
 
-    @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
     def test_multibyte_encoding(self):
         self.check_decode(932, (
             (b'\x84\xe9\x80', 'ignore', '\u9a3e'),
@@ -3630,7 +3625,6 @@ def test_code_page_decode_flags(self):
         self.assertEqual(codecs.code_page_decode(42, b'abc'),
                          ('\uf061\uf062\uf063', 3))
 
-    @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
     def test_incremental(self):
         decoded = codecs.code_page_decode(932, b'\x82', 'strict', False)
         self.assertEqual(decoded, ('', 0))