From 72712b4cb1d70bf57afdcf3a4d6d1eb3beb7188d Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 7 Jun 2014 20:11:09 +0200 Subject: [PATCH 01/12] - Directive map now no longer validates pattern key as a global regex for all keys inside the map. - This is done in favor for the new directive regex; that can be defined inside mapping: - Removed tests that tests for pattern: that broke after these changes. - If a passing_test fails it now outputs the failing files that was tested --- pykwalify/core.py | 8 +------- tests/files/27a.yaml | 1 - tests/files/27b.yaml | 6 ------ tests/files/29a.yaml | 6 ------ tests/files/29b.yaml | 9 --------- tests/testcore.py | 14 +++++++------- 6 files changed, 8 insertions(+), 36 deletions(-) delete mode 100644 tests/files/27a.yaml delete mode 100644 tests/files/27b.yaml delete mode 100644 tests/files/29a.yaml delete mode 100644 tests/files/29b.yaml diff --git a/pykwalify/core.py b/pykwalify/core.py index f763184..321e62c 100644 --- a/pykwalify/core.py +++ b/pykwalify/core.py @@ -233,13 +233,7 @@ def _validate_mapping(self, value, rule, path, errors=[], done=None): regex_mappings = [(regex_rule, re.match(regex_rule._map_regex_rule, str(k))) for regex_rule in rule._regex_mappings] Log.debug(" + Mapping Regex matches: {}".format(regex_mappings)) - if rule._pattern: - # This is the global regex pattern specefied at the same level as mapping: and type: map keys - res = re.match(rule._pattern, str(k)) - Log.debug("Matching regexPattern: {} with value: {}".format(rule._pattern, k)) - if res is None: # Not matching - errors.append("pattern.unmatch : {} --> {} : {}".format(rule._pattern, k, path)) - elif any(regex_mappings): + if any(regex_mappings): # Found atleast one that matches a mapping regex for mm in regex_mappings: if mm[1]: diff --git a/tests/files/27a.yaml b/tests/files/27a.yaml deleted file mode 100644 index 5f6e70a..0000000 --- a/tests/files/27a.yaml +++ /dev/null @@ -1 +0,0 @@ -na1me: foobar \ No newline at end of file diff --git a/tests/files/27b.yaml b/tests/files/27b.yaml deleted file mode 100644 index d629738..0000000 --- a/tests/files/27b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -type: map -pattern: "^[a-z]+$" -allowempty: True -mapping: - name: - type: str diff --git a/tests/files/29a.yaml b/tests/files/29a.yaml deleted file mode 100644 index 9587acf..0000000 --- a/tests/files/29a.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- mic: - name: input - bits: 16 -- media: - name: output - bits: 32 diff --git a/tests/files/29b.yaml b/tests/files/29b.yaml deleted file mode 100644 index 0525396..0000000 --- a/tests/files/29b.yaml +++ /dev/null @@ -1,9 +0,0 @@ -type: seq -sequence: - - type: map - pattern: ".+" - mapping: - name: - type: str - bits: - type: str diff --git a/tests/testcore.py b/tests/testcore.py index 225b200..440ad02 100644 --- a/tests/testcore.py +++ b/tests/testcore.py @@ -85,8 +85,6 @@ def testCore(self): # ("28a.yaml", "28b.yaml", {'allowempty': True, 'mapping': {'name': {'type': 'str'}}, 'pattern': '^[a-z0-9]+$', 'type': 'map'}), # - ("29a.yaml", "29b.yaml", {'sequence': [{'mapping': {'bits': {'type': 'str'}, 'name': {'type': 'str'}}, 'pattern': '.+', 'type': 'map'}], 'type': 'seq'}), - # ("30a.yaml", "30b.yaml", {'sequence': [{'mapping': {'foobar': {'mapping': {'opa': {'type': 'bool'}}, 'type': 'map'}, 'media': {'type': 'int'}, 'regex;[mi.+]': {'sequence': [{'type': 'str'}], 'type': 'seq'}, 'regex;[mo.+]': {'sequence': [{'type': 'bool'}], 'type': 'seq'}}, 'matching-rule': 'any', 'type': 'map'}], 'type': 'seq'}), # This test that a regex that will compile ("31a.yaml", "31b.yaml", {'mapping': {'regex;mi.+': {'sequence': [{'type': 'str'}], 'type': 'seq'}}, 'matching-rule': 'any', 'type': 'map'}), @@ -130,14 +128,16 @@ def testCore(self): ("22a.yaml", "22b.yaml", SchemaError, ["Value: abc is not of type 'number' : /2"]), # This test the text validation rule with wrong data ("24a.yaml", "24b.yaml", SchemaError, ["Value: True is not of type 'text' : /3"]), - # This tests pattern matching on keys in a map - ("27a.yaml", "27b.yaml", SchemaError, ['pattern.unmatch : ^[a-z]+$ --> na1me : ']), ] for passing_test in pass_tests: - c = Core(source_file=self.f(passing_test[0]), schema_file=self.f(passing_test[1])) - c.validate() - compare(c.validation_errors, [], prefix="No validation errors should exist...") + try: + c = Core(source_file=self.f(passing_test[0]), schema_file=self.f(passing_test[1])) + c.validate() + compare(c.validation_errors, [], prefix="No validation errors should exist...") + except Exception as e: + print("ERROR RUNNING FILE: {} : {}".format(passing_test[0], passing_test[1])) + raise e # This serve as an extra schema validation that tests more complex structures then testrule.py do compare(c.root_rule._schema_str, passing_test[2], prefix="Parsed rules is not correct, something have changed...") From 5a079193abca2183d2ca2ec0407ded2ca7da6087 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 7 Jun 2014 20:29:44 +0200 Subject: [PATCH 02/12] - keyword "pattern" is no longer allowed to be used when using "type: map". This is in favor of the new "regex;" rule - Removed test files 28a & 28b because of the above change - New rule test that tests that pattern is not allowed --- pykwalify/rule.py | 3 +++ tests/files/28a.yaml | 1 - tests/files/28b.yaml | 6 ------ tests/testcore.py | 3 +-- tests/testrule.py | 4 ++++ 5 files changed, 8 insertions(+), 9 deletions(-) delete mode 100644 tests/files/28a.yaml delete mode 100644 tests/files/28b.yaml diff --git a/pykwalify/rule.py b/pykwalify/rule.py index d15eecb..c0d0480 100644 --- a/pykwalify/rule.py +++ b/pykwalify/rule.py @@ -160,6 +160,9 @@ def initPatternValue(self, v, rule, path): self._pattern = v + if self._schema_str["type"] == "map": + raise RuleError("map.pattern : pattern not allowed inside map : {} : {}".format(v, path)) + # TODO: Some form of validation of the regexp? it exists in the source try: diff --git a/tests/files/28a.yaml b/tests/files/28a.yaml deleted file mode 100644 index 5f6e70a..0000000 --- a/tests/files/28a.yaml +++ /dev/null @@ -1 +0,0 @@ -na1me: foobar \ No newline at end of file diff --git a/tests/files/28b.yaml b/tests/files/28b.yaml deleted file mode 100644 index 0c2e051..0000000 --- a/tests/files/28b.yaml +++ /dev/null @@ -1,6 +0,0 @@ -type: map -pattern: "^[a-z0-9]+$" -allowempty: True -mapping: - name: - type: str diff --git a/tests/testcore.py b/tests/testcore.py index 440ad02..572c0c7 100644 --- a/tests/testcore.py +++ b/tests/testcore.py @@ -83,8 +83,6 @@ def testCore(self): # ("26a.yaml", "26b.yaml", {'type': 'any'}), # - ("28a.yaml", "28b.yaml", {'allowempty': True, 'mapping': {'name': {'type': 'str'}}, 'pattern': '^[a-z0-9]+$', 'type': 'map'}), - # ("30a.yaml", "30b.yaml", {'sequence': [{'mapping': {'foobar': {'mapping': {'opa': {'type': 'bool'}}, 'type': 'map'}, 'media': {'type': 'int'}, 'regex;[mi.+]': {'sequence': [{'type': 'str'}], 'type': 'seq'}, 'regex;[mo.+]': {'sequence': [{'type': 'bool'}], 'type': 'seq'}}, 'matching-rule': 'any', 'type': 'map'}], 'type': 'seq'}), # This test that a regex that will compile ("31a.yaml", "31b.yaml", {'mapping': {'regex;mi.+': {'sequence': [{'type': 'str'}], 'type': 'seq'}}, 'matching-rule': 'any', 'type': 'map'}), @@ -148,3 +146,4 @@ def testCore(self): c.validate() compare(sorted(c.validation_errors), sorted(failing_test[3]), prefix="Wrong validation errors when parsing files : {} : {}".format(failing_test[0], failing_test[1])) + \ No newline at end of file diff --git a/tests/testrule.py b/tests/testrule.py index 14dfb5e..ac09be3 100644 --- a/tests/testrule.py +++ b/tests/testrule.py @@ -102,3 +102,7 @@ def testRuleClass(self): # This will test that a invalid regex will throw error when parsing rules with self.assertRaises(RuleError): Rule(schema={"type": "map", "matching-rule": "any", "mapping": {"regex;(+": {"type": "seq", "sequence": [{"type": "str"}]}}}) + + # Test that pattern keyword is not allowed when using a map + with self.assertRaisesRegexp(RuleError, ".+map\.pattern.+"): + Rule(schema={"type": "map", "pattern": "^[a-z]+$", "allowempty": True, "mapping": {"name": {"type": "str"}}}) From 3dce4ac96e1e2ddda2180d441dab20bb85b48f00 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 7 Jun 2014 21:19:31 +0200 Subject: [PATCH 03/12] Updated Readme about not using "pattern:" in map but should use "regex;" --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c77eab3..fa932ec 100644 --- a/README.md +++ b/README.md @@ -96,11 +96,10 @@ enum: pattern: Specifies regular expression pattern of value. (Uses re.match() ) - pattern rule works in map to validate keys, it is usefull when allowempty is set to True. Pattern also works on all scalar types. - This will be matched against all keys in a map. + Pattern no longer works in map. Use regex; as keys in "mapping:" -regex;: +regex;: This is only implemented in map where a key inside the mapping keyword can implement this regex; pattern and all keys will be matched against the pattern. If a match is found then it will parsed the subrules on that key. A single key can be matched against multiple regex rules and the normal map rules. From 7106fd50e7b9bb51626d389d3d6c9d79e93fe554 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Mon, 9 Jun 2014 02:02:26 +0200 Subject: [PATCH 04/12] Implemented new feature "partial schemas" - New directive "schema;" that is used to define a partial schema. Only works at top level of schema. - New directive "include: " that can be used anywhere "type" directive can be used. It imports a partial defined schema where used. - Updated Readme with new directives - Added new tests for "schema;" and "include:" directives (Some not working right now) - Cli and Core() now can handle multiple schema files and merge them all before starting the parsing - Currently include cannot be used inside "schema;". This will be fixed at a later time. --- README.md | 34 +++++++++++++++++++++++++ ReleaseNotes.rst | 8 ++++++ pykwalify/__init__.py | 9 ++++++- pykwalify/cli.py | 4 +-- pykwalify/core.py | 59 ++++++++++++++++++++++++++++++++++--------- pykwalify/rule.py | 19 ++++++++++++-- tests/files/32a.yaml | 14 ++++++++++ tests/files/32b.yaml | 1 + tests/files/32c.yaml | 12 +++++++++ tests/files/33a.yaml | 3 +++ tests/files/33b.yaml | 11 ++++++++ tests/files/33c.yaml | 1 + tests/files/34a.yaml | 3 +++ tests/files/34b.yaml | 11 ++++++++ tests/files/34c.yaml | 1 + tests/testcore.py | 40 ++++++++++++++++++++++++++--- tests/testrule.py | 12 +++++++++ 17 files changed, 222 insertions(+), 20 deletions(-) create mode 100644 tests/files/32a.yaml create mode 100644 tests/files/32b.yaml create mode 100644 tests/files/32c.yaml create mode 100644 tests/files/33a.yaml create mode 100644 tests/files/33b.yaml create mode 100644 tests/files/33c.yaml create mode 100644 tests/files/34a.yaml create mode 100644 tests/files/34b.yaml create mode 100644 tests/files/34c.yaml diff --git a/README.md b/README.md index fa932ec..4a10ab7 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,40 @@ matching-rule: +## Partial schemas + +It is possible to create small partial schemas that can be included in other schemas. This feature do not use any built-in YAML or JSON linking. + +To define a partial schema use the keyword "schema;:". must be globally unique for the loaded schema partials. If collisions is detected then error will be raised. + +To use a partial schema use the keyword "include: :". This will work at any place you can specify the keyword "type". Include directive do not currently work inside a partial schema. + +It is possible to define any number of partial schemas in any schema file as long as they are defined at top level of the schema. + +For example, this schema contains one partial and the regular schema. + +```yaml +schema;fooone: + type: map + mapping: + foo: + type: str + + +type: seq +sequence: + - include: fooone + +``` + +And it can be used to validate the following data + +```yaml +- foo: "opa" +``` + + + ## License MIT [See LICENSE file] diff --git a/ReleaseNotes.rst b/ReleaseNotes.rst index f4496ff..cb40972 100644 --- a/ReleaseNotes.rst +++ b/ReleaseNotes.rst @@ -2,6 +2,14 @@ Release Notes ============= +v14.06.1 +======== + + - New feature "partial schema". Define a small schema with a ID that can be reused at other places in the schema. See readme for details. + - New directive "include" that is used to include a partial schema at the specefied location. + - Cli and Core() now can handle multiple schema files. + - Directive "pattern" can no longer be used with map to validate all keys against that regex. Use "regex;" inside "mapping:" + v14.06 ====== diff --git a/pykwalify/__init__.py b/pykwalify/__init__.py index c4dbd06..91694dc 100644 --- a/pykwalify/__init__.py +++ b/pykwalify/__init__.py @@ -4,7 +4,7 @@ __author__ = 'Grokzen ' #__version__ = '.'.join(map(str, __version_info__)) -__foobar__ = "0.1.2" +__foobar__ = "14.06" # Set to True to have revision from Version Control System in version string __devel__ = True @@ -30,3 +30,10 @@ def init_logging(): "formatters": {"simple": {"format": " {}".format(msg)}}} logging.config.dictConfig(logging_conf) + + +partial_schemas = {} + + +def add_partial_schema(schema_id, rule_object): + partial_schemas[schema_id] = rule_object diff --git a/pykwalify/cli.py b/pykwalify/cli.py index 6635b52..f7410b0 100644 --- a/pykwalify/cli.py +++ b/pykwalify/cli.py @@ -31,7 +31,7 @@ def main(): ##### __docopt__ = """ -usage: pykwalify -d DATAFILE -s SCHEMAFILE [-q] [-v ...] +usage: pykwalify -d DATAFILE -s SCHEMAFILE ... [-q] [-v ...] pykwalify -h | --help pykwalify -V | --version @@ -92,5 +92,5 @@ def main(): ##### 3. parse cli arguments ##### - c = Core(source_file=args["--data-file"], schema_file=args["--schema-file"]) + c = Core(source_file=args["--data-file"], schema_files=args["--schema-file"]) c.validate() diff --git a/pykwalify/core.py b/pykwalify/core.py index 321e62c..2d00adb 100644 --- a/pykwalify/core.py +++ b/pykwalify/core.py @@ -14,6 +14,7 @@ Log = logging.getLogger(__name__) # pyKwalify imports +import pykwalify from pykwalify.rule import Rule from pykwalify.types import isScalar, tt from pykwalify.errors import CoreError, SchemaError @@ -25,9 +26,9 @@ class Core(object): """ Core class of pyKwalify """ - def __init__(self, source_file=None, schema_file=None, source_data=None, schema_data=None): + def __init__(self, source_file=None, schema_files=[], source_data=None, schema_data=None): Log.debug("source_file: {}".format(source_file)) - Log.debug("schema_file: {}".format(schema_file)) + Log.debug("schema_file: {}".format(schema_files)) Log.debug("source_data: {}".format(source_data)) Log.debug("schema_data: {}".format(schema_data)) @@ -48,17 +49,35 @@ def __init__(self, source_file=None, schema_file=None, source_data=None, schema_ else: raise CoreError("Unable to load source_file. Unknown file format of specified file path: {}".format(source_file)) - if schema_file is not None: - if not os.path.exists(schema_file): - raise CoreError("Provided source_file do not exists on disk") + if not isinstance(schema_files, list): + raise CoreError("schema_files must be of list type") + + # Merge all schema files into one signel file for easy parsing + if len(schema_files) > 0: + schema_data = {} + for f in schema_files: + if not os.path.exists(f): + raise CoreError("Provided source_file do not exists on disk") + + with open(f, "r") as stream: + if f.endswith(".json"): + data = json.load(stream) + if not data: + raise CoreError("No data loaded from file : {}".format(f)) + elif f.endswith(".yaml") or f.endswith(".yml"): + data = yaml.load(stream) + if not data: + raise CoreError("No data loaded from file : {}".format(f)) + else: + raise CoreError("Unable to load file : {} : Unknown file format. Supported file endings is [.json, .yaml, .yml]") + + for key in data.keys(): + if key in schema_data.keys(): + raise CoreError("Parsed key : {} : two times in schema files...".format(key)) + + schema_data = dict(schema_data, **data) - with open(schema_file, "r") as stream: - if schema_file.endswith(".json"): - self.schema = json.load(stream) - elif schema_file.endswith(".yaml"): - self.schema = yaml.load(stream) - else: - raise CoreError("Unable to load source_file. Unknown file format of specified file path: {}".format(schema_file)) + self.schema = schema_data # Nothing was loaded so try the source_data variable if self.source is None: @@ -101,10 +120,26 @@ def _start_validate(self, value=None): errors = [] done = [] + s = {} + + # Look for schema; tags so they can be parsed before the root rule is parsed + for k, v in self.schema.items(): + if k.startswith("schema;"): + Log.debug("Found partial schema; : {}".format(v)) + r = Rule(schema=v) + Log.debug(" Partial schema : {}".format(r)) + pykwalify.partial_schemas[k.split(";", 1)[1]] = r + else: + # readd all items that is not schema; so they can be parsed + s[k] = v + + self.schema = s + Log.debug("Building root rule object") root_rule = Rule(schema=self.schema) self.root_rule = root_rule Log.debug("Done building root rule") + Log.debug("Root rule: {}".format(self.root_rule)) self._validate(value, root_rule, path, errors, done) diff --git a/pykwalify/rule.py b/pykwalify/rule.py index c0d0480..976803c 100644 --- a/pykwalify/rule.py +++ b/pykwalify/rule.py @@ -13,6 +13,7 @@ Log = logging.getLogger(__name__) # pyKwalify imports +import pykwalify from pykwalify.types import DEFAULT_TYPE, typeClass, isBuiltinType, isCollectionType from pykwalify.errors import SchemaConflict, RuleError @@ -56,9 +57,20 @@ def __str__(self): def init(self, schema, path): Log.debug("Init schema: {}".format(schema)) - if schema is not None: - # assert isinstance(schema, dict), "schema is not a dict : {}".format(path) + include = schema.get("include", None) + + # Check if this item is a include, overwrite schema with include schema and continue to parse + if include: + Log.debug("Found include tag...") + partial_schema = pykwalify.partial_schemas.get(include, None) + if not partial_schema: + raise RuleError("include partial schema id error : schema id '{}' not found : Available partial schemas [{}]".format(include, ", ".join(pykwalify.partial_schemas.keys()))) + # Partial schema found, overwrite this rule with the partial schema rule and continue to parse like normal... + schema = partial_schema._schema_str + Log.debug("Parsing partial schema rule : {}".format(schema)) + + if schema is not None: if "type" not in schema: raise RuleError("key 'type' not found in schema rule : {}".format(path)) else: @@ -96,6 +108,9 @@ def init(self, schema, path): for k, v in schema.items(): if k in func_mapping: func_mapping[k](v, rule, path) + elif k.startswith("schema;"): + Log.debug("Found schema tag...") + raise RuleError("Schema is only allowed on top level of schema file...") else: raise RuleError("Unknown key: {} found : {}".format(k, path)) diff --git a/tests/files/32a.yaml b/tests/files/32a.yaml new file mode 100644 index 0000000..6d9dc13 --- /dev/null +++ b/tests/files/32a.yaml @@ -0,0 +1,14 @@ +schema;fooone: + type: map + mapping: + foo: + type: str + +schema;footwo: + type: map + mapping: + foo: + type: bool + +schema;str: + type: str diff --git a/tests/files/32b.yaml b/tests/files/32b.yaml new file mode 100644 index 0000000..13e396c --- /dev/null +++ b/tests/files/32b.yaml @@ -0,0 +1 @@ +include: fooone diff --git a/tests/files/32c.yaml b/tests/files/32c.yaml new file mode 100644 index 0000000..4a80875 --- /dev/null +++ b/tests/files/32c.yaml @@ -0,0 +1,12 @@ + + +# 4 +# foo: "bar" + +# 3 +foo: "opa" + +# 2 +# - foo: "opa" + +# 1 diff --git a/tests/files/33a.yaml b/tests/files/33a.yaml new file mode 100644 index 0000000..ea433b7 --- /dev/null +++ b/tests/files/33a.yaml @@ -0,0 +1,3 @@ +type: seq +sequence: + - include: fooone diff --git a/tests/files/33b.yaml b/tests/files/33b.yaml new file mode 100644 index 0000000..ef84657 --- /dev/null +++ b/tests/files/33b.yaml @@ -0,0 +1,11 @@ +schema;fooone: + type: map + mapping: + foo: + type: str + +schema;footwo: + type: map + mapping: + foo: + type: bool \ No newline at end of file diff --git a/tests/files/33c.yaml b/tests/files/33c.yaml new file mode 100644 index 0000000..4444bad --- /dev/null +++ b/tests/files/33c.yaml @@ -0,0 +1 @@ +- foo: "opa" \ No newline at end of file diff --git a/tests/files/34a.yaml b/tests/files/34a.yaml new file mode 100644 index 0000000..ea433b7 --- /dev/null +++ b/tests/files/34a.yaml @@ -0,0 +1,3 @@ +type: seq +sequence: + - include: fooone diff --git a/tests/files/34b.yaml b/tests/files/34b.yaml new file mode 100644 index 0000000..b93127a --- /dev/null +++ b/tests/files/34b.yaml @@ -0,0 +1,11 @@ +schema;fooone: + type: map + mapping: + foo: + include: footwo + +schema;footwo: + type: map + mapping: + foo: + type: bool diff --git a/tests/files/34c.yaml b/tests/files/34c.yaml new file mode 100644 index 0000000..4444bad --- /dev/null +++ b/tests/files/34c.yaml @@ -0,0 +1 @@ +- foo: "opa" \ No newline at end of file diff --git a/tests/testcore.py b/tests/testcore.py index 572c0c7..25a3f47 100644 --- a/tests/testcore.py +++ b/tests/testcore.py @@ -49,6 +49,41 @@ def testCoreDataMode(self): with self.assertRaises(SchemaError): Core(source_data=dict, schema_data={"type": "any"}).validate() + def test_multi_file_support(self): + """ + This should test that multiple files is supported correctly + """ + pass_tests = [ + # Test that include directive can be used at top level of the schema + ([self.f("33a.yaml"), self.f("33b.yaml")], self.f("33c.yaml"), {'sequence': [{'include': 'fooone'}], 'type': 'seq'}), + # This test that include directive works inside sequence + ([self.f("33a.yaml"), self.f("33b.yaml")], self.f("33c.yaml"), {'sequence': [{'include': 'fooone'}], 'type': 'seq'}) + ] + + failing_tests = [ + # Test include inside partial schema # TODO: This test do not currently work correctly + # ([self.f("34a.yaml"), self.f("34b.yaml")], self.f("34c.yaml"), RuleError, []) + ] + + for passing_test in pass_tests: + try: + c = Core(source_file=passing_test[1], schema_files=passing_test[0]) + c.validate() + compare(c.validation_errors, [], prefix="No validation errors should exist...") + except Exception as e: + print("ERROR RUNNING FILE: {} : {}".format(passing_test[0], passing_test[1])) + raise e + + # This serve as an extra schema validation that tests more complex structures then testrule.py do + compare(c.root_rule._schema_str, passing_test[2], prefix="Parsed rules is not correct, something have changed...") + + for failing_test in failing_tests: + with self.assertRaises(failing_test[2], msg="Test files: {} : {}".format(", ".join(failing_test[0]), failing_test[1])): + c = Core(schema_files=failing_test[0], source_file=failing_test[1]) + c.validate() + + compare(sorted(c.validation_errors), sorted(failing_test[3]), prefix="Wrong validation errors when parsing files : {} : {}".format(failing_test[0], failing_test[1])) + def testCore(self): # These tests should pass with no exception raised pass_tests = [ @@ -130,7 +165,7 @@ def testCore(self): for passing_test in pass_tests: try: - c = Core(source_file=self.f(passing_test[0]), schema_file=self.f(passing_test[1])) + c = Core(source_file=self.f(passing_test[0]), schema_files=[self.f(passing_test[1])]) c.validate() compare(c.validation_errors, [], prefix="No validation errors should exist...") except Exception as e: @@ -142,8 +177,7 @@ def testCore(self): for failing_test in fail_tests: with self.assertRaises(failing_test[2], msg="Test file: {} : {}".format(failing_test[0], failing_test[1])): - c = Core(source_file=self.f(failing_test[0]), schema_file=self.f(failing_test[1])) + c = Core(source_file=self.f(failing_test[0]), schema_files=[self.f(failing_test[1])]) c.validate() compare(sorted(c.validation_errors), sorted(failing_test[3]), prefix="Wrong validation errors when parsing files : {} : {}".format(failing_test[0], failing_test[1])) - \ No newline at end of file diff --git a/tests/testrule.py b/tests/testrule.py index ac09be3..fc43aa0 100644 --- a/tests/testrule.py +++ b/tests/testrule.py @@ -106,3 +106,15 @@ def testRuleClass(self): # Test that pattern keyword is not allowed when using a map with self.assertRaisesRegexp(RuleError, ".+map\.pattern.+"): Rule(schema={"type": "map", "pattern": "^[a-z]+$", "allowempty": True, "mapping": {"name": {"type": "str"}}}) + + # Test that error is raised when using include tag but schema do not exists + with self.assertRaises(RuleError): + r = Rule(schema={"type": "map", "mapping": {"foo": {"include": "str"}}}) + + # Test that when only having a schema; rule it should throw error + with self.assertRaises(RuleError): + r = Rule(schema={"schema;fooone": {"type": "map", "mapping": {"foo": {"type": "str"}}}}) + + # Test that when using both schema; and include tag that it throw an error because schema; tags should be parsed via Core() + with self.assertRaises(RuleError): + r = Rule(schema={"schema;str": {"type": "map", "mapping": {"foo": {"type": "str"}}}, "type": "map", "mapping": {"foo": {"include": "str"}}}) From 1959367b1a7571da8f81dba7469a248238fa4cfe Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 21 Jun 2014 21:07:04 +0200 Subject: [PATCH 05/12] Added instructions how to run test suite --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 4a10ab7..e69a018 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,22 @@ Note: It is recomended allways to use a virtual-enviroment when using pyKwalify +# How to run tests + +Install test requirements with + +``` +pip install -r test-requirements.txt +``` + +Run tests with + +``` +nosetests +``` + + + # Implemented validation rules ``` From 1032660bb244682aed346e747e1fce015ed2fce6 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 21 Jun 2014 21:25:18 +0200 Subject: [PATCH 06/12] Fixed a bug with recursive includes inside defined schemas. Added new test file 35 that will test a working partial schema with a include to another partial schema. Uncommented test 34 to test broken partial schemas. --- pykwalify/core.py | 4 ++++ tests/files/35a.yaml | 3 +++ tests/files/35b.yaml | 11 +++++++++++ tests/files/35c.yaml | 2 ++ tests/testcore.py | 6 ++++-- 5 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/files/35a.yaml create mode 100644 tests/files/35b.yaml create mode 100644 tests/files/35c.yaml diff --git a/pykwalify/core.py b/pykwalify/core.py index 2d00adb..e8b093c 100644 --- a/pykwalify/core.py +++ b/pykwalify/core.py @@ -250,6 +250,10 @@ def _validate_mapping(self, value, rule, path, errors=[], done=None): Log.debug(" + Value is None, returning...") return + if not isinstance(value, dict): + errors.append("mapping.value.notdict : {} : {}".format(value, path)) + return + m = rule._mapping Log.debug(" + RuleMapping: {}".format(m)) diff --git a/tests/files/35a.yaml b/tests/files/35a.yaml new file mode 100644 index 0000000..ea433b7 --- /dev/null +++ b/tests/files/35a.yaml @@ -0,0 +1,3 @@ +type: seq +sequence: + - include: fooone diff --git a/tests/files/35b.yaml b/tests/files/35b.yaml new file mode 100644 index 0000000..003dec2 --- /dev/null +++ b/tests/files/35b.yaml @@ -0,0 +1,11 @@ +schema;fooone: + type: map + mapping: + foo: + include: footwo + +schema;footwo: + type: map + mapping: + bar: + type: bool diff --git a/tests/files/35c.yaml b/tests/files/35c.yaml new file mode 100644 index 0000000..c1bf711 --- /dev/null +++ b/tests/files/35c.yaml @@ -0,0 +1,2 @@ +- foo: + bar: true diff --git a/tests/testcore.py b/tests/testcore.py index 25a3f47..3dd9fb4 100644 --- a/tests/testcore.py +++ b/tests/testcore.py @@ -57,12 +57,14 @@ def test_multi_file_support(self): # Test that include directive can be used at top level of the schema ([self.f("33a.yaml"), self.f("33b.yaml")], self.f("33c.yaml"), {'sequence': [{'include': 'fooone'}], 'type': 'seq'}), # This test that include directive works inside sequence - ([self.f("33a.yaml"), self.f("33b.yaml")], self.f("33c.yaml"), {'sequence': [{'include': 'fooone'}], 'type': 'seq'}) + ([self.f("33a.yaml"), self.f("33b.yaml")], self.f("33c.yaml"), {'sequence': [{'include': 'fooone'}], 'type': 'seq'}), + # This test recursive schemas + ([self.f("35a.yaml"), self.f("35b.yaml")], self.f("35c.yaml"), {'sequence': [{'include': 'fooone'}], 'type': 'seq'}) ] failing_tests = [ # Test include inside partial schema # TODO: This test do not currently work correctly - # ([self.f("34a.yaml"), self.f("34b.yaml")], self.f("34c.yaml"), RuleError, []) + ([self.f("34a.yaml"), self.f("34b.yaml")], self.f("34c.yaml"), SchemaError, ['mapping.value.notdict : opa : /0/foo']) ] for passing_test in pass_tests: From 4e1a980ddc5227f335069791ed4d1b7d5bee5cf9 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 21 Jun 2014 21:37:45 +0200 Subject: [PATCH 07/12] Added a third partial schema to test file 35b. Moved them around some. --- tests/files/35b.yaml | 17 +++++++++++------ tests/files/35c.yaml | 3 ++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/files/35b.yaml b/tests/files/35b.yaml index 003dec2..a7873b1 100644 --- a/tests/files/35b.yaml +++ b/tests/files/35b.yaml @@ -1,11 +1,16 @@ -schema;fooone: - type: map - mapping: - foo: - include: footwo +schema;foothree: + type: seq + sequence: + - type: bool schema;footwo: type: map mapping: bar: - type: bool + include: foothree + +schema;fooone: + type: map + mapping: + foo: + include: footwo diff --git a/tests/files/35c.yaml b/tests/files/35c.yaml index c1bf711..c67a590 100644 --- a/tests/files/35c.yaml +++ b/tests/files/35c.yaml @@ -1,2 +1,3 @@ - foo: - bar: true + bar: + - true From 8f435676c3d016f3615dc9eaaf5c53fd9ee376e9 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Sat, 21 Jun 2014 22:24:38 +0200 Subject: [PATCH 08/12] Fixed partial schema implementation. Some tests on python 3.4 was broken but not on other python version. Moved include logic from Rule into Core for better handling. --- pykwalify/core.py | 17 ++++++++++++++++- pykwalify/rule.py | 11 +++-------- tests/files/34a.yaml | 2 +- tests/files/35b.yaml | 10 +++++----- tests/testcore.py | 15 +++++++++++---- tests/testrule.py | 10 +++++----- 6 files changed, 41 insertions(+), 24 deletions(-) diff --git a/pykwalify/core.py b/pykwalify/core.py index e8b093c..83c1dae 100644 --- a/pykwalify/core.py +++ b/pykwalify/core.py @@ -157,7 +157,9 @@ def _validate(self, value, rule, path, errors, done): Log.debug(" ? ValidateRule: {}".format(rule)) n = len(errors) - if rule._sequence is not None: + if rule._include_name is not None: + self._validate_include(value, rule, path, errors, done=None) + elif rule._sequence is not None: self._validate_sequence(value, rule, path, errors, done=None) elif rule._mapping is not None or rule._allowempty_map: self._validate_mapping(value, rule, path, errors, done=None) @@ -167,6 +169,19 @@ def _validate(self, value, rule, path, errors, done): if len(errors) != n: return + def _validate_include(self, value, rule, path, errors=[], done=None): + if rule._include_name is None: + errors.append("Include name not valid : {} : {}".format(path, value)) + return + + include_name = rule._include_name + partial_schema_rule = pykwalify.partial_schemas.get(include_name, None) + if not partial_schema_rule: + errors.append("No partial schema found for name : {} : Existing partial schemas: {}".format(include_name, ", ".join(sorted(pykwalify.partial_schemas.keys())))) + return + + self._validate(value, partial_schema_rule, path, errors, done) + def _validate_sequence(self, value, rule, path, errors=[], done=None): Log.debug("Core Validate sequence") Log.debug(" * Data: {}".format(value)) diff --git a/pykwalify/rule.py b/pykwalify/rule.py index 976803c..385bef3 100644 --- a/pykwalify/rule.py +++ b/pykwalify/rule.py @@ -13,7 +13,6 @@ Log = logging.getLogger(__name__) # pyKwalify imports -import pykwalify from pykwalify.types import DEFAULT_TYPE, typeClass, isBuiltinType, isCollectionType from pykwalify.errors import SchemaConflict, RuleError @@ -43,6 +42,7 @@ def __init__(self, schema=None, parent=None): self._matching_rule = None self._map_regex_rule = None self._regex_mappings = None + self._include_name = None self._parent = parent self._schema = schema @@ -62,13 +62,8 @@ def init(self, schema, path): # Check if this item is a include, overwrite schema with include schema and continue to parse if include: Log.debug("Found include tag...") - partial_schema = pykwalify.partial_schemas.get(include, None) - if not partial_schema: - raise RuleError("include partial schema id error : schema id '{}' not found : Available partial schemas [{}]".format(include, ", ".join(pykwalify.partial_schemas.keys()))) - - # Partial schema found, overwrite this rule with the partial schema rule and continue to parse like normal... - schema = partial_schema._schema_str - Log.debug("Parsing partial schema rule : {}".format(schema)) + self._include_name = include + return if schema is not None: if "type" not in schema: diff --git a/tests/files/34a.yaml b/tests/files/34a.yaml index ea433b7..462d391 100644 --- a/tests/files/34a.yaml +++ b/tests/files/34a.yaml @@ -1,3 +1,3 @@ type: seq sequence: - - include: fooone + - include: fooonez diff --git a/tests/files/35b.yaml b/tests/files/35b.yaml index a7873b1..238abb9 100644 --- a/tests/files/35b.yaml +++ b/tests/files/35b.yaml @@ -1,8 +1,3 @@ -schema;foothree: - type: seq - sequence: - - type: bool - schema;footwo: type: map mapping: @@ -14,3 +9,8 @@ schema;fooone: mapping: foo: include: footwo + +schema;foothree: + type: seq + sequence: + - type: bool diff --git a/tests/testcore.py b/tests/testcore.py index 3dd9fb4..6af5923 100644 --- a/tests/testcore.py +++ b/tests/testcore.py @@ -10,12 +10,16 @@ from testfixtures import compare # pyKwalify imports +import pykwalify from pykwalify.core import Core from pykwalify.errors import PyKwalifyExit, UnknownError, FileNotAccessible, OptionError, NotImplemented, ParseFailure, SchemaError, CoreError, RuleError class TestCore(unittest.TestCase): + def setUp(self): + pykwalify.partial_schemas = {} + def f(self, *args): return os.path.join(os.path.dirname(os.path.realpath(__file__)), "files", *args) @@ -56,15 +60,15 @@ def test_multi_file_support(self): pass_tests = [ # Test that include directive can be used at top level of the schema ([self.f("33a.yaml"), self.f("33b.yaml")], self.f("33c.yaml"), {'sequence': [{'include': 'fooone'}], 'type': 'seq'}), - # This test that include directive works inside sequence - ([self.f("33a.yaml"), self.f("33b.yaml")], self.f("33c.yaml"), {'sequence': [{'include': 'fooone'}], 'type': 'seq'}), + # # This test that include directive works inside sequence + # ([self.f("33a.yaml"), self.f("33b.yaml")], self.f("33c.yaml"), {'sequence': [{'include': 'fooone'}], 'type': 'seq'}), # This test recursive schemas ([self.f("35a.yaml"), self.f("35b.yaml")], self.f("35c.yaml"), {'sequence': [{'include': 'fooone'}], 'type': 'seq'}) ] failing_tests = [ - # Test include inside partial schema # TODO: This test do not currently work correctly - ([self.f("34a.yaml"), self.f("34b.yaml")], self.f("34c.yaml"), SchemaError, ['mapping.value.notdict : opa : /0/foo']) + # Test include inside partial schema + ([self.f("34a.yaml"), self.f("34b.yaml")], self.f("34c.yaml"), SchemaError, ['No partial schema found for name : fooonez : Existing partial schemas: fooone, foothree, footwo']) ] for passing_test in pass_tests: @@ -84,6 +88,9 @@ def test_multi_file_support(self): c = Core(schema_files=failing_test[0], source_file=failing_test[1]) c.validate() + if not c.validation_errors: + raise AssertionError("No validation_errors was raised...") + compare(sorted(c.validation_errors), sorted(failing_test[3]), prefix="Wrong validation errors when parsing files : {} : {}".format(failing_test[0], failing_test[1])) def testCore(self): diff --git a/tests/testrule.py b/tests/testrule.py index fc43aa0..8d10207 100644 --- a/tests/testrule.py +++ b/tests/testrule.py @@ -5,12 +5,16 @@ import unittest # pyKwalify imports +import pykwalify from pykwalify.rule import Rule -from pykwalify.errors import RuleError +from pykwalify.errors import RuleError, SchemaError class TestRule(unittest.TestCase): + def setUp(self): + pykwalify.partial_schemas = {} + def testRuleClass(self): # this tests seq type with a internal type of str r = Rule(schema={"type": "seq", "sequence": [{"type": "str"}]}) @@ -107,10 +111,6 @@ def testRuleClass(self): with self.assertRaisesRegexp(RuleError, ".+map\.pattern.+"): Rule(schema={"type": "map", "pattern": "^[a-z]+$", "allowempty": True, "mapping": {"name": {"type": "str"}}}) - # Test that error is raised when using include tag but schema do not exists - with self.assertRaises(RuleError): - r = Rule(schema={"type": "map", "mapping": {"foo": {"include": "str"}}}) - # Test that when only having a schema; rule it should throw error with self.assertRaises(RuleError): r = Rule(schema={"schema;fooone": {"type": "map", "mapping": {"foo": {"type": "str"}}}}) From db5f7f09cda84dad1e9daa77d57df6cfa577153e Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 24 Jun 2014 00:54:43 +0200 Subject: [PATCH 09/12] Validate scalar type first and not last because it will check if the value is None early and that will avoid the scalar type check. --- pykwalify/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pykwalify/core.py b/pykwalify/core.py index 321e62c..e9af7b0 100644 --- a/pykwalify/core.py +++ b/pykwalify/core.py @@ -273,6 +273,8 @@ def _validate_scalar(self, value, rule, path, errors=[], done=None): if rule._default and value is None: value = rule._default + self._validate_scalar_type(value, rule._type, errors, path) + if value is None: return @@ -351,7 +353,6 @@ def _validate_scalar(self, value, rule, path, errors=[], done=None): if l.get("min-ex", None) is not None and l["min-ex"] >= L: errors.append("length.tooshort-ex : {} >= {} : {}".format(l["min-ex"], L, path)) - self._validate_scalar_type(value, rule._type, errors, path) def _validate_scalar_type(self, value, t, errors, path): Log.debug("Core scalar: validating scalar type") From c41d6975f70e1ba4b9cbef9efb5c4c0c6a9a49ee Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 24 Jun 2014 01:24:19 +0200 Subject: [PATCH 10/12] Added support for 'none' as 'type'. Added 2 new tests that check this new support. --- README.md | 1 + pykwalify/types.py | 8 +++++++- tests/files/36a.yaml | 5 +++++ tests/files/36b.yaml | 16 ++++++++++++++++ tests/files/37a.yaml | 3 +++ tests/files/37b.yaml | 14 ++++++++++++++ tests/testcore.py | 4 ++++ 7 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 tests/files/36a.yaml create mode 100644 tests/files/36b.yaml create mode 100644 tests/files/37a.yaml create mode 100644 tests/files/37b.yaml diff --git a/README.md b/README.md index e69a018..6045db6 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ type: - timestamp [NYI] - seq - map + - none - scalar (all but seq and map) - any (means any implemented type of data) diff --git a/pykwalify/types.py b/pykwalify/types.py index a3f8c73..87148c7 100644 --- a/pykwalify/types.py +++ b/pykwalify/types.py @@ -24,7 +24,8 @@ "scalar": None, "text": None, "any": object, - "enum": str + "enum": str, + "none": None } @@ -88,6 +89,10 @@ def isEnum(obj): return isinstance(obj, str) +def isNone(obj): + return obj is None + + tt = {"str": isString, "int": isInt, "bool": isBool, @@ -96,4 +101,5 @@ def isEnum(obj): "text": isText, "any": isAny, "enum": isEnum, + "none": isNone } diff --git a/tests/files/36a.yaml b/tests/files/36a.yaml new file mode 100644 index 0000000..fba304b --- /dev/null +++ b/tests/files/36a.yaml @@ -0,0 +1,5 @@ +streams: + - name: + sampleRateMultiple: 1 + - name: media + sampleRateMultiple: 2 \ No newline at end of file diff --git a/tests/files/36b.yaml b/tests/files/36b.yaml new file mode 100644 index 0000000..54b4bfb --- /dev/null +++ b/tests/files/36b.yaml @@ -0,0 +1,16 @@ +type: map +mapping: + streams: + type: seq + required: True + sequence: + - type: map + mapping: + name: + type: str + length: + min: 1 + required: True + sampleRateMultiple: + type: int + required: True \ No newline at end of file diff --git a/tests/files/37a.yaml b/tests/files/37a.yaml new file mode 100644 index 0000000..a2a51ee --- /dev/null +++ b/tests/files/37a.yaml @@ -0,0 +1,3 @@ +streams: + - name: + sampleRateMultiple: 1 diff --git a/tests/files/37b.yaml b/tests/files/37b.yaml new file mode 100644 index 0000000..b5eb823 --- /dev/null +++ b/tests/files/37b.yaml @@ -0,0 +1,14 @@ +type: map +mapping: + streams: + type: seq + required: True + sequence: + - type: map + mapping: + name: + type: none + required: True + sampleRateMultiple: + type: int + required: True \ No newline at end of file diff --git a/tests/testcore.py b/tests/testcore.py index 6af5923..bbd5fd8 100644 --- a/tests/testcore.py +++ b/tests/testcore.py @@ -130,6 +130,8 @@ def testCore(self): ("30a.yaml", "30b.yaml", {'sequence': [{'mapping': {'foobar': {'mapping': {'opa': {'type': 'bool'}}, 'type': 'map'}, 'media': {'type': 'int'}, 'regex;[mi.+]': {'sequence': [{'type': 'str'}], 'type': 'seq'}, 'regex;[mo.+]': {'sequence': [{'type': 'bool'}], 'type': 'seq'}}, 'matching-rule': 'any', 'type': 'map'}], 'type': 'seq'}), # This test that a regex that will compile ("31a.yaml", "31b.yaml", {'mapping': {'regex;mi.+': {'sequence': [{'type': 'str'}], 'type': 'seq'}}, 'matching-rule': 'any', 'type': 'map'}), + # Test that type can be set to 'None' and it will validate ok + ("37a.yaml", "37b.yaml", {'mapping': {'streams': {'required': True, 'sequence': [{'mapping': {'name': {'required': True, 'type': 'none'}, 'sampleRateMultiple': {'required': True, 'type': 'int'}}, 'type': 'map'}], 'type': 'seq'}}, 'type': 'map'}), ] # These tests are designed to fail with some exception raised @@ -170,6 +172,8 @@ def testCore(self): ("22a.yaml", "22b.yaml", SchemaError, ["Value: abc is not of type 'number' : /2"]), # This test the text validation rule with wrong data ("24a.yaml", "24b.yaml", SchemaError, ["Value: True is not of type 'text' : /3"]), + # This test that typechecking works when value in map is None + ("36a.yaml", "36b.yaml", SchemaError, ["Value: None is not of type 'str' : /streams/0/name"]) ] for passing_test in pass_tests: From 0e57ac0e41df3f722e1decacecfe7b13b2d943eb Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 24 Jun 2014 01:27:13 +0200 Subject: [PATCH 11/12] Updates to release notes --- ReleaseNotes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ReleaseNotes.rst b/ReleaseNotes.rst index cb40972..acc041a 100644 --- a/ReleaseNotes.rst +++ b/ReleaseNotes.rst @@ -9,6 +9,8 @@ v14.06.1 - New directive "include" that is used to include a partial schema at the specefied location. - Cli and Core() now can handle multiple schema files. - Directive "pattern" can no longer be used with map to validate all keys against that regex. Use "regex;" inside "mapping:" + - 'none' can now be used as a type + - Many more tests added v14.06 ====== From 572a1b86c2ff82b44b429d876c3209f16dccb7e0 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 24 Jun 2014 01:47:53 +0200 Subject: [PATCH 12/12] Bump version number --- pykwalify/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pykwalify/__init__.py b/pykwalify/__init__.py index 91694dc..c28b2b6 100644 --- a/pykwalify/__init__.py +++ b/pykwalify/__init__.py @@ -4,7 +4,7 @@ __author__ = 'Grokzen ' #__version__ = '.'.join(map(str, __version_info__)) -__foobar__ = "14.06" +__foobar__ = "14.06.1" # Set to True to have revision from Version Control System in version string __devel__ = True diff --git a/setup.py b/setup.py index 919ad00..99832f7 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ settings.update( name="pykwalify", - version="14.06", + version="14.06.1", description='Python lib/cli for JSON/YAML schema validation', long_description='Python lib/cli for JSON/YAML schema validation', author="Grokzen",