Port CPython 3.6.5 test cases

tests/cpython/__init__.py

@@ -0,0 +1,77 @@
+import os
+import json
+import doctest
+import unittest
+
+from test import support
+
+# import json with and without accelerations
+cjson = support.import_fresh_module('json', fresh=['_json'])
+pyjson = support.import_fresh_module('json', blocked=['_json'])
+hyperjson = support.import_fresh_module('hyperjson')
+# JSONDecodeError is cached inside the _json module
+cjson.JSONDecodeError = cjson.decoder.JSONDecodeError = json.JSONDecodeError
+
+# create two base classes that will be used by the other tests
+class PyTest(unittest.TestCase):
+    json = pyjson
+    loads = staticmethod(pyjson.loads)
+    dumps = staticmethod(pyjson.dumps)
+    JSONDecodeError = staticmethod(pyjson.JSONDecodeError)
+
+@unittest.skipUnless(cjson, 'requires _json')
+class CTest(unittest.TestCase):
+    if cjson is not None:
+        json = cjson
+        loads = staticmethod(cjson.loads)
+        dumps = staticmethod(cjson.dumps)
+        JSONDecodeError = staticmethod(cjson.JSONDecodeError)
+
+@unittest.skipUnless(hyperjson, 'requires hyperjson')
+class RustTest(unittest.TestCase):
+    if hyperjson is not None:
+        json = hyperjson
+        loads = staticmethod(hyperjson.loads)
+        dumps = staticmethod(hyperjson.dumps)
+        # FIXME: hyperjson does not have this
+        JSONDecodeError = staticmethod(pyjson.JSONDecodeError)
+        # JSONDecodeError = staticmethod(hyperjson.JSONDecodeError)
+
+# test PyTest and CTest checking if the functions come from the right module
+class TestPyTest(PyTest):
+    def test_pyjson(self):
+        self.assertEqual(self.json.scanner.make_scanner.__module__,
+                         'json.scanner')
+        self.assertEqual(self.json.decoder.scanstring.__module__,
+                         'json.decoder')
+        self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__,
+                         'json.encoder')
+
+class TestCTest(CTest):
+    def test_cjson(self):
+        self.assertEqual(self.json.scanner.make_scanner.__module__, '_json')
+        self.assertEqual(self.json.decoder.scanstring.__module__, '_json')
+        self.assertEqual(self.json.encoder.c_make_encoder.__module__, '_json')
+        self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__,
+                         '_json')
+
+class TestRustTest(RustTest):
+    def test_hyperjson(self):
+        # FIXME: hyperjson does not have any of this
+        pass
+        # self.assertEqual(self.json.scanner.make_scanner.__module__, 'hyperjson')
+        # self.assertEqual(self.json.decoder.scanstring.__module__, 'hyperjson')
+        # self.assertEqual(self.json.encoder.c_make_encoder.__module__, 'hyperjson')
+        # self.assertEqual(self.json.encoder.encode_basestring_ascii.__module__,
+        #                  'hyperjson')
+
+
+def load_tests(loader, _, pattern):
+    suite = unittest.TestSuite()
+    for mod in (json, json.encoder, json.decoder):
+        suite.addTest(doctest.DocTestSuite(mod))
+    suite.addTest(TestPyTest('test_pyjson'))
+    suite.addTest(TestCTest('test_cjson'))
+
+    pkg_dir = os.path.dirname(__file__)
+    return support.load_package_tests(pkg_dir, loader, suite, pattern)

tests/cpython/__main__.py

@@ -0,0 +1,4 @@
+import unittest
+from cpython import load_tests
+
+unittest.main()

tests/cpython/test_decode.py

@@ -0,0 +1,98 @@
+import decimal
+from io import StringIO, BytesIO
+from collections import OrderedDict
+from cpython import PyTest, CTest, RustTest
+
+
+class _TestDecode:
+    def test_decimal(self):
+        rval = self.loads('1.1', parse_float=decimal.Decimal)
+        self.assertTrue(isinstance(rval, decimal.Decimal))
+        self.assertEqual(rval, decimal.Decimal('1.1'))
+
+    def test_float(self):
+        rval = self.loads('1', parse_int=float)
+        self.assertTrue(isinstance(rval, float))
+        self.assertEqual(rval, 1.0)
+
+    def test_empty_objects(self):
+        self.assertEqual(self.loads('{}'), {})
+        self.assertEqual(self.loads('[]'), [])
+        self.assertEqual(self.loads('""'), "")
+
+    def test_object_pairs_hook(self):
+        s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}'
+        p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4),
+             ("qrt", 5), ("pad", 6), ("hoy", 7)]
+        self.assertEqual(self.loads(s), eval(s))
+        self.assertEqual(self.loads(s, object_pairs_hook=lambda x: x), p)
+        self.assertEqual(self.json.load(StringIO(s),
+                                        object_pairs_hook=lambda x: x), p)
+        od = self.loads(s, object_pairs_hook=OrderedDict)
+        self.assertEqual(od, OrderedDict(p))
+        self.assertEqual(type(od), OrderedDict)
+        # the object_pairs_hook takes priority over the object_hook
+        self.assertEqual(self.loads(s, object_pairs_hook=OrderedDict,
+                                    object_hook=lambda x: None),
+                         OrderedDict(p))
+        # check that empty object literals work (see #17368)
+        self.assertEqual(self.loads('{}', object_pairs_hook=OrderedDict),
+                         OrderedDict())
+        self.assertEqual(self.loads('{"empty": {}}',
+                                    object_pairs_hook=OrderedDict),
+                         OrderedDict([('empty', OrderedDict())]))
+
+    def test_decoder_optimizations(self):
+        # Several optimizations were made that skip over calls to
+        # the whitespace regex, so this test is designed to try and
+        # exercise the uncommon cases. The array cases are already covered.
+        rval = self.loads('{   "key"    :    "value"    ,  "k":"v"    }')
+        self.assertEqual(rval, {"key":"value", "k":"v"})
+
+    def check_keys_reuse(self, source, loads):
+        rval = loads(source)
+        (a, b), (c, d) = sorted(rval[0]), sorted(rval[1])
+        self.assertIs(a, c)
+        self.assertIs(b, d)
+
+    def test_keys_reuse(self):
+        s = '[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'
+        self.check_keys_reuse(s, self.loads)
+        self.check_keys_reuse(s, self.json.decoder.JSONDecoder().decode)
+
+    def test_extra_data(self):
+        s = '[1, 2, 3]5'
+        msg = 'Extra data'
+        self.assertRaisesRegex(self.JSONDecodeError, msg, self.loads, s)
+
+    def test_invalid_escape(self):
+        s = '["abc\\y"]'
+        msg = 'escape'
+        self.assertRaisesRegex(self.JSONDecodeError, msg, self.loads, s)
+
+    def test_invalid_input_type(self):
+        msg = 'the JSON object must be str'
+        for value in [1, 3.14, [], {}, None]:
+            self.assertRaisesRegex(TypeError, msg, self.loads, value)
+
+    def test_string_with_utf8_bom(self):
+        # see #18958
+        bom_json = "[1,2,3]".encode('utf-8-sig').decode('utf-8')
+        with self.assertRaises(self.JSONDecodeError) as cm:
+            self.loads(bom_json)
+        self.assertIn('BOM', str(cm.exception))
+        with self.assertRaises(self.JSONDecodeError) as cm:
+            self.json.load(StringIO(bom_json))
+        self.assertIn('BOM', str(cm.exception))
+        # make sure that the BOM is not detected in the middle of a string
+        bom_in_str = '"{}"'.format(''.encode('utf-8-sig').decode('utf-8'))
+        self.assertEqual(self.loads(bom_in_str), '\ufeff')
+        self.assertEqual(self.json.load(StringIO(bom_in_str)), '\ufeff')
+
+    def test_negative_index(self):
+        d = self.json.JSONDecoder()
+        self.assertRaises(ValueError, d.raw_decode, 'a'*42, -50000)
+
+# class TestPyDecode(_TestDecode, PyTest): pass
+# class TestCDecode(_TestDecode, CTest): pass
+class TestRustDecode(_TestDecode, RustTest): pass

tests/cpython/test_default.py

@@ -0,0 +1,13 @@
+from cpython import PyTest, CTest, RustTest
+
+
+class _TestDefault:
+    def test_default(self):
+        self.assertEqual(
+            self.dumps(type, default=repr),
+            self.dumps(repr(type)))
+
+
+# class TestPyDefault(_TestDefault, PyTest): pass
+# class TestCDefault(_TestDefault, CTest): pass
+class TestRustDefault(_TestDefault, RustTest): pass

tests/cpython/test_dump.py

@@ -0,0 +1,70 @@
+from io import StringIO
+from cpython import PyTest, CTest, RustTest
+
+from test.support import bigmemtest, _1G
+
+class _TestDump:
+    def test_dump(self):
+        sio = StringIO()
+        self.json.dump({}, sio)
+        self.assertEqual(sio.getvalue(), '{}')
+
+    def test_dumps(self):
+        self.assertEqual(self.dumps({}), '{}')
+
+    def test_encode_truefalse(self):
+        self.assertEqual(self.dumps(
+                 {True: False, False: True}, sort_keys=True),
+                 '{"false": true, "true": false}')
+        self.assertEqual(self.dumps(
+                {2: 3.0, 4.0: 5, False: 1, 6: True}, sort_keys=True),
+                '{"false": 1, "2": 3.0, "4.0": 5, "6": true}')
+
+    # Issue 16228: Crash on encoding resized list
+    def test_encode_mutated(self):
+        a = [object()] * 10
+        def crasher(obj):
+            del a[-1]
+        self.assertEqual(self.dumps(a, default=crasher),
+                 '[null, null, null, null, null]')
+
+    # Issue 24094
+    def test_encode_evil_dict(self):
+        class D(dict):
+            def keys(self):
+                return L
+
+        class X:
+            def __hash__(self):
+                del L[0]
+                return 1337
+
+            def __lt__(self, o):
+                return 0
+
+        L = [X() for i in range(1122)]
+        d = D()
+        d[1337] = "true.dat"
+        self.assertEqual(self.dumps(d, sort_keys=True), '{"1337": "true.dat"}')
+
+
+# class TestPyDump(_TestDump, PyTest): pass
+
+# class TestCDump(_TestDump, CTest):
+
+#     # The size requirement here is hopefully over-estimated (actual
+#     # memory consumption depending on implementation details, and also
+#     # system memory management, since this may allocate a lot of
+#     # small objects).
+
+#     @bigmemtest(size=_1G, memuse=1)
+#     def test_large_list(self, size):
+#         N = int(30 * 1024 * 1024 * (size / _1G))
+#         l = [1] * N
+#         encoded = self.dumps(l)
+#         self.assertEqual(len(encoded), N * 3)
+#         self.assertEqual(encoded[:1], "[")
+#         self.assertEqual(encoded[-2:], "1]")
+#         self.assertEqual(encoded[1:-2], "1, " * (N - 1))
+
+class TestRustDump(_TestDump, RustTest): pass

tests/cpython/test_encode_basestring_ascii.py

@@ -0,0 +1,50 @@
+from collections import OrderedDict
+from cpython import PyTest, CTest, RustTest
+from test.support import bigaddrspacetest
+
+
+CASES = [
+    ('/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'),
+    ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'),
+    ('controls', '"controls"'),
+    ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'),
+    ('{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'),
+    (' s p a c e d ', '" s p a c e d "'),
+    ('\U0001d120', '"\\ud834\\udd20"'),
+    ('\u03b1\u03a9', '"\\u03b1\\u03a9"'),
+    ("`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'),
+    ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'),
+    ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'),
+]
+
+class _TestEncodeBasestringAscii:
+    def test_encode_basestring_ascii(self):
+        fname = self.json.encoder.encode_basestring_ascii.__name__
+        for input_string, expect in CASES:
+            result = self.json.encoder.encode_basestring_ascii(input_string)
+            self.assertEqual(result, expect,
+                '{0!r} != {1!r} for {2}({3!r})'.format(
+                    result, expect, fname, input_string))
+
+    def test_ordered_dict(self):
+        # See issue 6105
+        items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)]
+        s = self.dumps(OrderedDict(items))
+        self.assertEqual(s, '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}')
+
+    def test_sorted_dict(self):
+        items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)]
+        s = self.dumps(dict(items), sort_keys=True)
+        self.assertEqual(s, '{"five": 5, "four": 4, "one": 1, "three": 3, "two": 2}')
+
+
+# class TestPyEncodeBasestringAscii(_TestEncodeBasestringAscii, PyTest): pass
+# class TestCEncodeBasestringAscii(_TestEncodeBasestringAscii, CTest):
+#     @bigaddrspacetest
+#     def test_overflow(self):
+#         size = (2**32)//6 + 1
+#         s = "\x00"*size
+#         with self.assertRaises(OverflowError):
+#             self.json.encoder.encode_basestring_ascii(s)
+
+class TestRustEncodeBasestringAscii(_TestEncodeBasestringAscii, RustTest): pass