diff --git a/ijson-2.5.1.tar.gz b/ijson-2.5.1.tar.gz deleted file mode 100644 index e6f1532..0000000 --- a/ijson-2.5.1.tar.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:19ec46a2f7991004e5202ecee56c569616b8a7f95686ad7fd0a9ec81cac00269 -size 19338 diff --git a/ijson-3.0.tar.gz b/ijson-3.0.tar.gz new file mode 100644 index 0000000..289fcee --- /dev/null +++ b/ijson-3.0.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13507564c2798a2af9250211b0be246ca75bbf499fd7375803d3f91dccc7c2c1 +size 43273 diff --git a/python-ijson.changes b/python-ijson.changes index a222817..6537d9b 100644 --- a/python-ijson.changes +++ b/python-ijson.changes @@ -1,3 +1,33 @@ +------------------------------------------------------------------- +Thu Apr 9 13:11:22 UTC 2020 - Marketa Calabkova + +- update to 3.0 + * Exposing backend's name under ``.backend``, + and default backend's name under ``ijson.backend``. + * Exposing ``ijson.sendable_list`` to users in case it comes in handy. + * Improved the protocol for user-facing coroutines, + where instead of having to send a final, empty bytes string + to finish the parsing process + users can simply call ``.close()`` on the coroutine. + * Including C code in coverage measurements, + and increased overall code coverage up to 99%. + * Full re-design of ijson. + * Initial support for ``asyncio`` in python 3.5+. + * Exposure of underlying infrastructure implementing the push model. + * C extension broken down into separate source files + for easier understanding and maintenance. + * Fixed a deprecation warning in the C backend + present in python 3.8 when parsing Decimal values. + * New `kvitems` method in all backends. + Like `items`, it takes a prefix, + and iterates over the key/value pairs of matching objects + (instead of iterating over objects themselves, like in `items`). + * When using python 2, all backends now return + `map_key` values as `unicode` objects, not `str`. + * Including more files in source distributions (#14). + * Adjusting python backend to avoid reading off the input stream + too eagerly (#15). + ------------------------------------------------------------------- Fri Oct 18 09:57:10 UTC 2019 - John Vandenberg diff --git a/python-ijson.spec b/python-ijson.spec index 4ed537a..1afd337 100644 --- a/python-ijson.spec +++ b/python-ijson.spec @@ -1,7 +1,7 @@ # # spec file for package python-ijson # -# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2020 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -18,20 +18,21 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-ijson -Version: 2.5.1 +Version: 3.0 Release: 0 Summary: Iterative JSON parser with a standard Python iterator interface License: BSD-3-Clause Group: Development/Languages/Python URL: https://github.com/ICRAR/ijson Source: https://files.pythonhosted.org/packages/source/i/ijson/ijson-%{version}.tar.gz -# https://github.com/isagalaev/ijson/pull/74 -Source1: https://raw.githubusercontent.com/ICRAR/ijson/master/tests.py +# https://github.com/ICRAR/ijson/pull/26 +Source1: https://raw.githubusercontent.com/ICRAR/ijson/master/tests_asyncio.py BuildRequires: %{python_module devel} BuildRequires: %{python_module setuptools} -BuildRequires: pkgconfig(yajl) BuildRequires: fdupes BuildRequires: python-rpm-macros +BuildRequires: python3-asyncio +BuildRequires: pkgconfig(yajl) %python_subpackages %description diff --git a/tests.py b/tests.py deleted file mode 100644 index 3eb882a..0000000 --- a/tests.py +++ /dev/null @@ -1,346 +0,0 @@ -# -*- coding:utf-8 -*- -from __future__ import unicode_literals -import collections -import unittest -from decimal import Decimal -import threading -from importlib import import_module - -from ijson import common -from ijson.compat import BytesIO, StringIO, b2s, IS_PY2 -from ijson.backends.python import basic_parse, Lexer -import warnings - - -JSON = b''' -{ - "docs": [ - { - "null": null, - "boolean": false, - "true": true, - "integer": 0, - "double": 0.5, - "exponent": 1.0e+2, - "long": 10000000000, - "string": "\\u0441\\u0442\\u0440\\u043e\\u043a\\u0430 - \xd1\x82\xd0\xb5\xd1\x81\xd1\x82" - }, - { - "meta": [[1], {}] - }, - { - "meta": {"key": "value"} - }, - { - "meta": null - } - ] -} -''' -JSON_OBJECT = { - "docs": [ - { - "null": None, - "boolean": False, - "true": True, - "integer": 0, - "double": Decimal("0.5"), - "exponent": 1e+2, - "long": 10000000000, - "string": "строка - тест" - }, - { - "meta": [[1], {}] - }, - { - "meta": { - "key": "value" - } - }, - { - "meta": None - } - ] -} -JSON_EVENTS = [ - ('start_map', None), - ('map_key', 'docs'), - ('start_array', None), - ('start_map', None), - ('map_key', 'null'), - ('null', None), - ('map_key', 'boolean'), - ('boolean', False), - ('map_key', 'true'), - ('boolean', True), - ('map_key', 'integer'), - ('number', 0), - ('map_key', 'double'), - ('number', Decimal('0.5')), - ('map_key', 'exponent'), - ('number', 100), - ('map_key', 'long'), - ('number', 10000000000), - ('map_key', 'string'), - ('string', 'строка - тест'), - ('end_map', None), - ('start_map', None), - ('map_key', 'meta'), - ('start_array', None), - ('start_array', None), - ('number', 1), - ('end_array', None), - ('start_map', None), - ('end_map', None), - ('end_array', None), - ('end_map', None), - ('start_map', None), - ('map_key', 'meta'), - ('start_map', None), - ('map_key', 'key'), - ('string', 'value'), - ('end_map', None), - ('end_map', None), - ('start_map', None), - ('map_key', 'meta'), - ('null', None), - ('end_map', None), - ('end_array', None), - ('end_map', None), -] -SCALAR_JSON = b'0' -INVALID_JSONS = [ - b'["key", "value",]', # trailing comma - b'["key" "value"]', # no comma - b'{"key": "value",}', # trailing comma - b'{"key": "value" "key"}', # no comma - b'{"key" "value"}', # no colon - b'invalid', # unknown lexeme - b'[1, 2] dangling junk' # dangling junk -] -YAJL1_PASSING_INVALID = INVALID_JSONS[6] -INCOMPLETE_JSONS = [ - b'', - b'"test', - b'[', - b'[1', - b'[1,', - b'{', - b'{"key"', - b'{"key":', - b'{"key": "value"', - b'{"key": "value",', -] -STRINGS_JSON = br''' -{ - "str1": "", - "str2": "\"", - "str3": "\\", - "str4": "\\\\", - "special\t": "\b\f\n\r\t" -} -''' -NUMBERS_JSON = b'[1, 1.0, 1E2]' -SURROGATE_PAIRS_JSON = br'"\uD83D\uDCA9"' - - -class Parse(object): - ''' - Base class for parsing tests that is used to create test cases for each - available backends. - ''' - def test_basic_parse(self): - events = list(self.backend.basic_parse(BytesIO(JSON))) - self.assertEqual(events, JSON_EVENTS) - - def test_basic_parse_threaded(self): - thread = threading.Thread(target=self.test_basic_parse) - thread.start() - thread.join() - - def test_scalar(self): - events = list(self.backend.basic_parse(BytesIO(SCALAR_JSON))) - self.assertEqual(events, [('number', 0)]) - - def test_strings(self): - events = list(self.backend.basic_parse(BytesIO(STRINGS_JSON))) - strings = [value for event, value in events if event == 'string'] - self.assertEqual(strings, ['', '"', '\\', '\\\\', '\b\f\n\r\t']) - self.assertTrue(('map_key', 'special\t') in events) - - def test_surrogate_pairs(self): - event = next(self.backend.basic_parse(BytesIO(SURROGATE_PAIRS_JSON))) - parsed_string = event[1] - self.assertEqual(parsed_string, '💩') - - def test_numbers(self): - events = list(self.backend.basic_parse(BytesIO(NUMBERS_JSON))) - types = [type(value) for event, value in events if event == 'number'] - self.assertEqual(types, [int, Decimal, Decimal]) - - def test_invalid(self): - for json in INVALID_JSONS: - # Yajl1 doesn't complain about additional data after the end - # of a parsed object. Skipping this test. - if self.__class__.__name__ == 'YajlParse' and json == YAJL1_PASSING_INVALID: - continue - with self.assertRaises(common.JSONError) as cm: - list(self.backend.basic_parse(BytesIO(json))) - - def test_incomplete(self): - for json in INCOMPLETE_JSONS: - with self.assertRaises(common.IncompleteJSONError): - list(self.backend.basic_parse(BytesIO(json))) - - def test_utf8_split(self): - buf_size = JSON.index(b'\xd1') + 1 - try: - events = list(self.backend.basic_parse(BytesIO(JSON), buf_size=buf_size)) - except UnicodeDecodeError: - self.fail('UnicodeDecodeError raised') - - def test_lazy(self): - # shouldn't fail since iterator is not exhausted - self.backend.basic_parse(BytesIO(INVALID_JSONS[0])) - self.assertTrue(True) - - def test_boundary_lexeme(self): - buf_size = JSON.index(b'false') + 1 - events = list(self.backend.basic_parse(BytesIO(JSON), buf_size=buf_size)) - self.assertEqual(events, JSON_EVENTS) - - def test_boundary_whitespace(self): - buf_size = JSON.index(b' ') + 1 - events = list(self.backend.basic_parse(BytesIO(JSON), buf_size=buf_size)) - self.assertEqual(events, JSON_EVENTS) - - def test_api(self): - self.assertTrue(list(self.backend.items(BytesIO(JSON), ''))) - self.assertTrue(list(self.backend.parse(BytesIO(JSON)))) - - def test_items_twodictlevels(self): - f = BytesIO(b'{"meta":{"view":{"columns":[{"id": -1}, {"id": -2}]}}}') - ids = list(self.backend.items(f, 'meta.view.columns.item.id')) - self.assertEqual(2, len(ids)) - self.assertListEqual([-2,-1], sorted(ids)) - - def test_multiple_values(self): - if not self.supports_multiple_values: - return - basic_parse = self.backend.basic_parse - items = lambda x, **kwargs: self.backend.items(x, '', **kwargs) - multiple_values = JSON + JSON + JSON - for func in (basic_parse, items): - generator = func(BytesIO(multiple_values)) - self.assertRaises(common.JSONError, list, generator) - generator = func(BytesIO(multiple_values), multiple_values=False) - self.assertRaises(common.JSONError, list, generator) - generator = func(BytesIO(multiple_values), multiple_values=True) - result = list(generator) - if func == basic_parse: - self.assertEqual(result, JSON_EVENTS + JSON_EVENTS + JSON_EVENTS) - else: - self.assertEqual(result, [JSON_OBJECT, JSON_OBJECT, JSON_OBJECT]) - - def test_map_type(self): - obj = next(self.backend.items(BytesIO(JSON), '')) - self.assertTrue(isinstance(obj, dict)) - obj = next(self.backend.items(BytesIO(JSON), '', map_type=collections.OrderedDict)) - self.assertTrue(isinstance(obj, collections.OrderedDict)) - - def test_string_stream(self): - with warnings.catch_warnings(record=True) as warns: - events = list(self.backend.basic_parse(StringIO(b2s(JSON)))) - self.assertEqual(events, JSON_EVENTS) - if self.warn_on_string_stream: - self.assertEqual(len(warns), 1) - self.assertEqual(DeprecationWarning, warns[0].category) - -# Generating real TestCase classes for each importable backend -for name in ['python', 'yajl', 'yajl2', 'yajl2_cffi', 'yajl2_c']: - try: - classname = '%sParse' % ''.join(p.capitalize() for p in name.split('_')) - if IS_PY2: - classname = classname.encode('ascii') - - locals()[classname] = type( - classname, - (unittest.TestCase, Parse), - { - 'backend': import_module('ijson.backends.%s' % name), - 'supports_multiple_values': name != 'yajl', - 'warn_on_string_stream': name != 'python' and not IS_PY2 - }, - ) - except ImportError: - pass - - -class Common(unittest.TestCase): - ''' - Backend independent tests. They all use basic_parse imported explicitly from - the python backend to generate parsing events. - ''' - def test_object_builder(self): - builder = common.ObjectBuilder() - for event, value in basic_parse(BytesIO(JSON)): - builder.event(event, value) - self.assertEqual(builder.value, { - 'docs': [ - { - 'string': 'строка - тест', - 'null': None, - 'boolean': False, - 'true': True, - 'integer': 0, - 'double': Decimal('0.5'), - 'exponent': 100, - 'long': 10000000000, - }, - { - 'meta': [[1], {}], - }, - { - 'meta': {'key': 'value'}, - }, - { - 'meta': None, - }, - ], - }) - - def test_scalar_builder(self): - builder = common.ObjectBuilder() - for event, value in basic_parse(BytesIO(SCALAR_JSON)): - builder.event(event, value) - self.assertEqual(builder.value, 0) - - def test_parse(self): - events = common.parse(basic_parse(BytesIO(JSON))) - events = [value - for prefix, event, value in events - if prefix == 'docs.item.meta.item.item' - ] - self.assertEqual(events, [1]) - - def test_items(self): - events = basic_parse(BytesIO(JSON)) - meta = list(common.items(common.parse(events), 'docs.item.meta')) - self.assertEqual(meta, [ - [[1], {}], - {'key': 'value'}, - None, - ]) - -class Stream(unittest.TestCase): - def test_bytes(self): - l = Lexer(BytesIO(JSON)) - self.assertEqual(next(l)[1], '{') - - def test_string(self): - l = Lexer(StringIO(JSON.decode('utf-8'))) - self.assertEqual(next(l)[1], '{') - - -if __name__ == '__main__': - unittest.main() diff --git a/tests_asyncio.py b/tests_asyncio.py new file mode 100644 index 0000000..fc68170 --- /dev/null +++ b/tests_asyncio.py @@ -0,0 +1,47 @@ +# -*- coding:utf-8 -*- + +import asyncio +import io + +from ijson import compat + + +class AsyncReader(object): + def __init__(self, data): + if type(data) == compat.bytetype: + self.data = io.BytesIO(data) + else: + self.data = io.StringIO(data) + + async def read(self, n=-1): + return self.data.read(n) + +class Async(object): + '''Test adaptation for async generators''' + + suffix = '_async' + + def _run(self, f): + loop = asyncio.new_event_loop() + try: + loop.run_until_complete(f) + finally: + loop.close() + + def all(self, routine, json_content, *args, **kwargs): + events = [] + async def run(): + async for event in routine(AsyncReader(json_content), *args, **kwargs): + events.append(event) + self._run(run()) + return events + + def first(self, routine, json_content, *args, **kwargs): + events = [] + async def run(): + async for event in routine(AsyncReader(json_content), *args, **kwargs): + events.append(event) + if events: + return + self._run(run()) + return events[0] \ No newline at end of file