diff --git a/pykwalify/core.py b/pykwalify/core.py index 25be852..13dfed6 100644 --- a/pykwalify/core.py +++ b/pykwalify/core.py @@ -133,9 +133,17 @@ def __init__(self, source_file=None, schema_files=None, source_data=None, schema if self.source is None: log.debug(u"No source file loaded, trying source data variable") self.source = source_data + if self.schema is None: log.debug(u"No schema file loaded, trying schema data variable") - self.schema = schema_data + + if isinstance(schema_data, list): + merged_schema = {} + for schema in schema_data: + merged_schema.update(schema) + self.schema = merged_schema + else: + self.schema = schema_data # Test if anything was loaded if self.source is None: @@ -256,9 +264,9 @@ def _validate(self, value, rule, path, done): return log.debug(u" ? ValidateRule: %s", rule) - if rule.include_name is not None: - self._validate_include(value, rule, path, done=None) - elif rule.sequence is not None: + # if rule.include_name is not None: + # self._validate_include(value, rule, path, done=None) + if rule.sequence is not None: self._validate_sequence(value, rule, path, done=None) elif rule.mapping is not None or rule.allowempty_map: self._validate_mapping(value, rule, path, done=None) @@ -303,29 +311,6 @@ def _handle_func(self, value, rule, path, done=None): if not found_method: raise CoreError(u"Did not find method '{0}' in any loaded extension file".format(func)) - def _validate_include(self, value, rule, path, done=None): - """ - """ - # TODO: It is difficult to get a good test case to trigger this if case - if rule.include_name is None: - self.errors.append(SchemaError.SchemaErrorEntry( - msg=u'Include name not valid', - path=path, - value=value.encode('unicode_escape'))) - return - include_name = rule.include_name - partial_schema_rule = pykwalify.partial_schemas.get(include_name) - if not partial_schema_rule: - self.errors.append(SchemaError.SchemaErrorEntry( - msg=u"Cannot find partial schema with name '{include_name}'. Existing partial schemas: '{existing_schemas}'. Path: '{path}'", - path=path, - value=value, - include_name=include_name, - existing_schemas=", ".join(sorted(pykwalify.partial_schemas.keys())))) - return - - self._validate(value, partial_schema_rule, path, done) - def _validate_sequence(self, value, rule, path, done=None): """ """ diff --git a/pykwalify/rule.py b/pykwalify/rule.py index 7ac2c9e..059abb6 100644 --- a/pykwalify/rule.py +++ b/pykwalify/rule.py @@ -372,9 +372,25 @@ def init(self, schema, path): # Check if this item is a include, overwrite schema with include schema and continue to parse if include: - log.debug(u"Found include tag...") - self.include_name = include - return + import pykwalify + + partial_schema_rule = pykwalify.partial_schemas.get(include) + + if not partial_schema_rule: + raise RuleError( + msg=u"Include key: {0} not defined in schema".format(include), + error_key=u"include.key.unknown", + path=path, + ) + + # schema = {key: value for (key, value) in (schema.items() + partial_schema_rule.schema.items())} + # schema = dict(schema.items() | partial_schema_rule.schema.items()) + for k, v in partial_schema_rule.schema.items(): + schema[k] = v + + # log.debug(u"Found include tag...") + # self.include_name = include + # return t = None rule = self @@ -419,6 +435,7 @@ def init(self, schema, path): "format": self.init_format_value, "func": self.init_func, "ident": self.init_ident_value, + "include": self.init_include, "length": self.init_length_value, "map": self.init_mapping_value, "mapping": self.init_mapping_value, @@ -438,17 +455,24 @@ def init(self, schema, path): "version": self.init_version, } + # Do a initial pass of all keys to look for the schema tagg. + # If we are on the root rule, this check will not be done. + # If we are in any child rules, check all keys for any schema and + # raise error if we find any as schemas can only be defined at root level + if self.parent: + for k, v in schema.items(): + if k.startswith("schema;"): + # Schema tag is only allowed on top level of data + log.debug(u"Found schema tag...") + raise RuleError( + msg=u"Schema is only allowed on top level of schema file", + error_key=u"schema.not.toplevel", + path=path, + ) + for k, v in schema.items(): if k in func_mapping: func_mapping[k](v, rule, path) - elif k.startswith("schema;"): - # Schema tag is only allowed on top level of data - log.debug(u"Found schema tag...") - raise RuleError( - msg=u"Schema is only allowed on top level of schema file", - error_key=u"schema.not.toplevel", - path=path, - ) else: raise RuleError( msg=u"Unknown key: {0} found".format(k), @@ -460,6 +484,9 @@ def init(self, schema, path): self.check_type_keywords(schema, rule, path) + def init_include(self, v, rule, path): + pass + def init_format_value(self, v, rule, path): log.debug(u"Init format value : %s", path) diff --git a/setup.py b/setup.py index acdba95..2766f60 100644 --- a/setup.py +++ b/setup.py @@ -27,9 +27,10 @@ }, install_requires=[ 'docopt>=0.6.2', - "ruamel.yaml>=0.16.0" + 'ruamel.yaml>=0.16.0', 'python-dateutil>=2.8.0', ], + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", classifiers=[ # 'Development Status :: 1 - Planning', # 'Development Status :: 2 - Pre-Alpha', diff --git a/tests/files/partial_schemas/10f.yaml b/tests/files/partial_schemas/10f.yaml new file mode 100644 index 0000000..9074228 --- /dev/null +++ b/tests/files/partial_schemas/10f.yaml @@ -0,0 +1,24 @@ +--- +data: + point: + x: 3 + +schema: + type: map + mapping: + point: + mapping: + x: + required: true + include: coordinate_value + y: + required: true + include: coordinate_value + +partial-files: + - schema;coordinate_value: + type: number + +fail-exception-class: "SchemaError" +fail-validation-errors: + - "Cannot find required key 'y'. Path: '/point'" diff --git a/tests/files/partial_schemas/1s-partials.yaml b/tests/files/partial_schemas/1s-partials.yaml deleted file mode 100644 index ef84657..0000000 --- a/tests/files/partial_schemas/1s-partials.yaml +++ /dev/null @@ -1,11 +0,0 @@ -schema;fooone: - type: map - mapping: - foo: - type: str - -schema;footwo: - type: map - mapping: - foo: - type: bool \ No newline at end of file diff --git a/tests/files/partial_schemas/1s-schema.yaml b/tests/files/partial_schemas/1s-schema.yaml index ea433b7..7a8cf30 100644 --- a/tests/files/partial_schemas/1s-schema.yaml +++ b/tests/files/partial_schemas/1s-schema.yaml @@ -1,3 +1,15 @@ type: seq sequence: - include: fooone + +schema;fooone: + type: map + mapping: + foo: + type: str + +schema;footwo: + type: map + mapping: + foo: + type: bool \ No newline at end of file diff --git a/tests/files/partial_schemas/2s-partials.yaml b/tests/files/partial_schemas/2s-partials.yaml deleted file mode 100644 index 238abb9..0000000 --- a/tests/files/partial_schemas/2s-partials.yaml +++ /dev/null @@ -1,16 +0,0 @@ -schema;footwo: - type: map - mapping: - bar: - include: foothree - -schema;fooone: - type: map - mapping: - foo: - include: footwo - -schema;foothree: - type: seq - sequence: - - type: bool diff --git a/tests/files/partial_schemas/2s-schema.yaml b/tests/files/partial_schemas/2s-schema.yaml index ea433b7..d9fc80a 100644 --- a/tests/files/partial_schemas/2s-schema.yaml +++ b/tests/files/partial_schemas/2s-schema.yaml @@ -1,3 +1,20 @@ type: seq sequence: - include: fooone + +schema;foothree: + type: seq + sequence: + - type: bool + +schema;footwo: + type: map + mapping: + bar: + include: foothree + +schema;fooone: + type: map + mapping: + foo: + include: footwo diff --git a/tests/files/partial_schemas/4f-schema.yaml b/tests/files/partial_schemas/4f-schema.yaml index d109e40..a6ac3d5 100644 --- a/tests/files/partial_schemas/4f-schema.yaml +++ b/tests/files/partial_schemas/4f-schema.yaml @@ -2,19 +2,19 @@ type: seq sequence: - include: fooone -schema;fooone: +schema;foothree: type: map mapping: - foo: - include: footwo + bar: + type: str schema;footwo: type: seq sequence: - include: foothree -schema;foothree: +schema;fooone: type: map mapping: - bar: - type: str + foo: + include: footwo diff --git a/tests/files/partial_schemas/5f-schema.yaml b/tests/files/partial_schemas/5f-schema.yaml index 788d906..da3f1ec 100644 --- a/tests/files/partial_schemas/5f-schema.yaml +++ b/tests/files/partial_schemas/5f-schema.yaml @@ -2,17 +2,17 @@ type: seq sequence: - include: fooone -schema;fooone: +schema;foothree: type: seq sequence: - - include: footwo + - type: str schema;footwo: type: seq sequence: - include: foothree -schema;foothree: +schema;fooone: type: seq sequence: - - type: str + - include: footwo diff --git a/tests/files/partial_schemas/6f-schema.yaml b/tests/files/partial_schemas/6f-schema.yaml index 00c0a91..475ccbd 100644 --- a/tests/files/partial_schemas/6f-schema.yaml +++ b/tests/files/partial_schemas/6f-schema.yaml @@ -3,11 +3,11 @@ mapping: foo: include: fooone -schema;fooone: +schema;foothree: type: map mapping: - bar: - include: footwo + ewq: + type: str schema;footwo: type: map @@ -15,8 +15,8 @@ schema;footwo: qwe: include: foothree -schema;foothree: +schema;fooone: type: map mapping: - ewq: - type: str + bar: + include: footwo diff --git a/tests/files/partial_schemas/7s-schema.yaml b/tests/files/partial_schemas/7s-schema.yaml index eaf0613..87275d3 100644 --- a/tests/files/partial_schemas/7s-schema.yaml +++ b/tests/files/partial_schemas/7s-schema.yaml @@ -5,6 +5,7 @@ mapping: required: True bar: include: bar + schema;bar: type: seq required: True diff --git a/tests/test_core.py b/tests/test_core.py index 7092d3d..aa5957c 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8,7 +8,7 @@ # pykwalify imports import pykwalify from pykwalify.core import Core -from pykwalify.errors import SchemaError, CoreError +from pykwalify.errors import SchemaError, CoreError, RuleError # 3rd party imports import pytest @@ -227,11 +227,10 @@ def test_multi_file_support(self): ( [ self.f("partial_schemas", "1s-schema.yaml"), - self.f("partial_schemas", "1s-partials.yaml"), ], self.f("partial_schemas", "1s-data.yaml"), { - 'sequence': [{'include': 'fooone'}], + 'sequence': [{'include': 'fooone', 'mapping': {'foo': {'type': 'str'}}, 'type': 'map'}], 'type': 'seq', } ), @@ -241,13 +240,25 @@ def test_multi_file_support(self): ( [ self.f("partial_schemas", "2s-schema.yaml"), - self.f("partial_schemas", "2s-partials.yaml"), ], self.f("partial_schemas", "2s-data.yaml"), + # { + # 'sequence': [{'include': 'fooone'}], + # 'type': 'seq', + # } + { - 'sequence': [{'include': 'fooone'}], - 'type': 'seq', - } + "type": "seq", + "sequence": + [{'include': 'fooone', + 'mapping': {'foo': {'include': 'footwo', + 'mapping': {'bar': {'include': 'foothree', + 'sequence': [{'type': 'bool'}], + 'type': 'seq'}}, + 'type': 'map'}}, + 'type': 'map'}] + }, + ), # This tests that you can include a partial schema alongside other rules in a map ( @@ -263,7 +274,10 @@ def test_multi_file_support(self): 'required': True }, 'bar': { - 'include': 'bar' + 'include': 'bar', + "required": True, + "sequence": [{'type': 'str'}], + "type": "seq", } } } @@ -278,8 +292,8 @@ def test_multi_file_support(self): self.f("partial_schemas", "1f-partials.yaml") ], self.f("partial_schemas", "1f-data.yaml"), - SchemaError, - ["Cannot find partial schema with name 'fooonez'. Existing partial schemas: 'bar, fooone, foothree, footwo'. Path: '/0'"] + RuleError, + ["Include key: None not defined in schema: Path: '/sequence/0'"], ), ( [ @@ -324,6 +338,8 @@ def test_multi_file_support(self): ] for passing_test in pass_tests: + print("Running testfile: {0}".format(passing_test)) + try: c = Core(source_file=passing_test[1], schema_files=passing_test[0]) c.validate() @@ -333,25 +349,64 @@ def test_multi_file_support(self): raise e # This serve as an extra schema validation that tests more complex structures then testrule.py do - compare(c.root_rule.schema_str, passing_test[2], prefix="Parsed rules is not correct, something have changed...") + compare(c.root_rule.schema_str, passing_test[2], prefix="Parsed rules is not correct, something have changed... {0}".format(passing_test)) for failing_test in failing_tests: - print("Test files: {0} : {1}".format(", ".join(failing_test[0]), failing_test[1])) + print("Running failing tests: {0}".format(failing_test)) with pytest.raises(failing_test[2]): c = Core(schema_files=failing_test[0], source_file=failing_test[1]) c.validate() - if not c.validation_errors: - raise AssertionError("No validation_errors was raised...") + if not c.validation_errors: + raise AssertionError("No validation_errors was raised...") + + compare( + sorted(c.validation_errors), + sorted(failing_test[3]), + prefix="Wrong validation errors when parsing files : {0} : {1}".format( + failing_test[0], + failing_test[1], + ), + ) + + fail_tests = [ + self.f('partial_schemas', '10f.yaml') + ] + + for fail_test in fail_tests: + with open(fail_test, 'r') as stream: + fail_test_raw_data = yaml.safe_load_all(stream) + + for document_index, document in enumerate(fail_test_raw_data): + fail_test_data = document + + data = fail_test_data['data'] + schema = fail_test_data['schema'] + partial_files = fail_test_data['partial-files'] + fail_exception_class = fail_test_data['fail-exception-class'] + fail_validation_errors = fail_test_data['fail-validation-errors'] + fail_except_class_instance = None + + if fail_exception_class == "SchemaError": + fail_except_class_instance = SchemaError + elif fail_exception_class == "RuleError": + fail_except_class_instance = RuleError + + with pytest.raises(fail_except_class_instance): + print("Running test file {0}".format(fail_test)) + c = Core( + source_data=data, + schema_data=[schema] + partial_files, + ) + c.validate() compare( sorted(c.validation_errors), - sorted(failing_test[3]), - prefix="Wrong validation errors when parsing files : {0} : {1}".format( - failing_test[0], - failing_test[1], - ), + sorted(fail_validation_errors), + prefix="Wrong validation errors when parsing file : {0}".format( + fail_test, + ) ) def test_python_obj_loading(self, tmp_path): @@ -369,10 +424,10 @@ def test_python_obj_loading(self, tmp_path): - default - goodbye """ - schema_path = os.path.join(tmp_path, 'schema.yaml') + schema_path = os.path.join(str(tmp_path), 'schema.yaml') with open(schema_path, 'w') as stream: stream.write(schema) - data_path = os.path.join(tmp_path, 'data.yaml') + data_path = os.path.join(str(tmp_path), 'data.yaml') with open(data_path, 'w') as stream: stream.write(data) diff --git a/tests/test_rule.py b/tests/test_rule.py index 4b2b7c9..af5f5c2 100644 --- a/tests/test_rule.py +++ b/tests/test_rule.py @@ -23,7 +23,23 @@ def setUp(self): def test_schema(self): # Test that when using both schema; and include tag that it throw an error because schema; tags should be parsed via Core() with pytest.raises(RuleError) as r: - Rule(schema={"schema;str": {"type": "map", "mapping": {"foo": {"type": "str"}}}, "type": "map", "mapping": {"foo": {"include": "str"}}}) + Rule( + schema={ + "schema;str": { + "type": "map", "mapping": { + "foo": { + "type": "str" + }, + }, + }, + "type": "map", + "mapping": { + "foo": { + "include": "str" + }, + }, + } + ) assert str(r.value) == "" assert r.value.error_key == 'schema.not.toplevel'