Implement acceptance of custom file extensions for files to validate

Makefile

@@ -19,7 +19,6 @@ cleantox:
 	-rm -rf .tox/
 
 cleancov:
-	coverage combine
 	coverage erase
 	-rm -rf htmlcov/
 
@@ -38,7 +37,9 @@ cleanall: clean cleanegg cleanpy cleancov
 
 test:
 	coverage erase
-	coverage run --source pykwalify/ -m python py.test
+	coverage run --source pykwalify/ -m pytest
+	coverage report
+	coverage html
 
 sdist:
 	python setup.py sdist

pykwalify/cli.py

@@ -1,7 +1,3 @@
-# -*- coding: utf-8 -*-
-
-""" pyKwalify - cli.py """
-
 # python std lib
 import logging
 import logging.config
@@ -25,24 +21,40 @@ def parse_cli():
     #
 
     __docopt__ = """
-usage: pykwalify -d FILE -s FILE ... [-e FILE ...]
-       [--strict-rule-validation] [--fix-ruby-style-regex] [--allow-assertions] [-v ...] [-q]
-
-optional arguments:
-  -d FILE, --data-file FILE            the file to be tested
-  -e FILE, --extension FILE            file containing python extension
-  -s FILE, --schema-file FILE          schema definition file
-  --fix-ruby-style-regex               This flag fixes some of the quirks of ruby style regex
-                                       that is not compatible with python style regex
-  --strict-rule-validation             enables strict validation of all keywords for all
-                                       Rule objects to find unsupported keyword usage
-  --allow-assertions                   By default assertions is disabled due to security risk.
-                                       Error will be raised if assertion is used in schema
-                                       but this flag is not used. This option enables assert keyword.
-  -h, --help                           show this help message and exit
-  -q, --quiet                          suppress terminal output
-  -v, --verbose                        verbose terminal output (multiple -v increases verbosity)
-  --version                            display the version number and exit
+Usage: pykwalify -d FILE -s FILE ... [-e FILE ...]
+       [-y EXT] [-j EXT]
+       [--strict-rule-validation] [--fix-ruby-style-regex]
+       [--allow-assertions]
+       [-v ...] [-q]
+
+Options:
+  -d FILE, --data-file FILE            The file to be validated
+  -e FILE, --extension FILE            File containing python extension
+  -s FILE, --schema-file FILE          Schema definition file
+  -y EXT, --yaml-extension EXT         A custom YAML file extension to
+                                           accept for validation
+  -j EXT, --json-extension EXT         A custom JSON file extension to
+                                           accept for validation
+  --fix-ruby-style-regex               This flag fixes some of the
+                                           quirks of ruby style regex
+                                           that is not compatible with
+                                           python style regex
+  --strict-rule-validation             Enables strict validation of all
+                                           keywords for all Rule objects
+                                           to find unsupported keyword
+                                           usage
+  --allow-assertions                   By default assertions is disabled
+                                           due to security risk. An
+                                           error will be raised if
+                                           assertion is used in schema
+                                           but this flag is not used.
+                                           This option enables assert
+                                           keyword.
+  -h, --help                           Show this help message and exit
+  -q, --quiet                          Suppress terminal output
+  -v, --verbose                        Verbose terminal output (multiple
+                                           'v' increase verbosity)
+  --version                            Display the version number and exit
 """
 
     # Import pykwalify package
@@ -78,6 +90,8 @@ def run(cli_args):
         strict_rule_validation=cli_args['--strict-rule-validation'],
         fix_ruby_style_regex=cli_args['--fix-ruby-style-regex'],
         allow_assertions=cli_args['--allow-assertions'],
+        custom_yaml_ext=cli_args['--yaml-extension'],
+        custom_json_ext=cli_args['--json-extension'],
     )
     c.validate()
     return c
@@ -90,6 +104,7 @@ def cli_entrypoint():
     """
     # Check minimum version of Python
     if sys.version_info < (2, 7, 0):
-        sys.stderr.write(u"WARNING: pykwalify: It is recommended to run pykwalify on python version 2.7.x or later...\n\n")
+        sys.stderr.write(
+            u"WARNING: pykwalify should be run with Python >= 2.7!\n\n")
 
     run(parse_cli())

pykwalify/core.py

@@ -16,7 +16,8 @@
 # pyKwalify imports
 import pykwalify
 from pykwalify.compat import unicode, nativestr, basestring
-from pykwalify.errors import CoreError, SchemaError, NotMappingError, NotSequenceError
+from pykwalify.errors import CoreError, SchemaError
+from pykwalify.errors import NotMappingError, NotSequenceError
 from pykwalify.rule import Rule
 from pykwalify.types import is_scalar, is_string, tt
 
@@ -30,8 +31,11 @@
 class Core(object):
     """ Core class of pyKwalify """
 
-    def __init__(self, source_file=None, schema_files=None, source_data=None, schema_data=None, extensions=None, strict_rule_validation=False,
-                 fix_ruby_style_regex=False, allow_assertions=False,):
+    def __init__(self, source_file=None, schema_files=None,
+                 source_data=None, schema_data=None, extensions=None,
+                 strict_rule_validation=False,
+                 fix_ruby_style_regex=False, allow_assertions=False,
+                 custom_yaml_ext=None, custom_json_ext=None):
         """
         :param extensions:
             List of paths to python files that should be imported and available via 'func' keywork.
@@ -49,6 +53,8 @@ def __init__(self, source_file=None, schema_files=None, source_data=None, schema
         log.debug(u"source_data: %s", source_data)
         log.debug(u"schema_data: %s", schema_data)
         log.debug(u"extension files: %s", extensions)
+        log.debug(u"yaml extension: %s", custom_yaml_ext)
+        log.debug(u"json extension: %s", custom_json_ext)
 
         self.source = None
         self.schema = None
@@ -60,24 +66,36 @@ def __init__(self, source_file=None, schema_files=None, source_data=None, schema
         self.strict_rule_validation = strict_rule_validation
         self.fix_ruby_style_regex = fix_ruby_style_regex
         self.allow_assertions = allow_assertions
+        self.custom_yaml_ext = custom_yaml_ext
+        self.custom_json_ext = custom_json_ext
 
         if source_file is not None:
             if not os.path.exists(source_file):
-                raise CoreError(u"Provided source_file do not exists on disk: {0}".format(source_file))
+                raise CoreError(u"Provided source_file do not exists on "
+                                "disk: {0}".format(source_file))
 
             with open(source_file, "r") as stream:
-                if source_file.endswith(".json"):
+                if source_file.endswith(".json") or \
+                        (self.custom_json_ext is not None and
+                         source_file.endswith(self.custom_json_ext)):
                     try:
                         self.source = json.load(stream)
                     except Exception:
-                        raise CoreError(u"Unable to load any data from source json file")
-                elif source_file.endswith(".yaml") or source_file.endswith('.yml'):
+                        raise CoreError(u"Unable to load any data from "
+                                        "source json file")
+                elif source_file.endswith(".yaml") or \
+                        source_file.endswith('.yml') or \
+                        (self.custom_yaml_ext is not None and
+                         source_file.endswith(self.custom_yaml_ext)):
                     try:
                         self.source = yaml.load(stream)
                     except Exception:
-                        raise CoreError(u"Unable to load any data from source yaml file")
+                        raise CoreError(u"Unable to load any data from "
+                                        "source yaml file")
                 else:
-                    raise CoreError(u"Unable to load source_file. Unknown file format of specified file path: {0}".format(source_file))
+                    raise CoreError(u"Unable to load source_file. Unknown "
+                                    "file format of specified file path:"
+                                    " {0}".format(source_file))
 
         if not isinstance(schema_files, list):
             raise CoreError(u"schema_files must be of list type")

tests/files/cli/1a.yext

@@ -0,0 +1,3 @@
+- foo
+- bar
+- baz

tests/test_cli.py

@@ -38,6 +38,36 @@ def test_cli(self, tmpdir):
             assert k in cli_args
             assert cli_args[k] == expected[k]
 
+    def test_cli_custom_yaml_ext(self, tmpdir):
+        """
+        Test that when passing in certain arguments from commandline they
+        are handled correctly by docopt and correct args structure is returned.
+        """
+        input = tmpdir.join("cli/1a.yext")
+        schema_file = tmpdir.join("cli/1b.yaml")
+
+        sys.argv = [
+            'scripts/pykwalify',
+            '-d', str(input),
+            '-s', str(schema_file),
+            '-y', 'yext',
+            '-v'
+        ]
+
+        expected = {
+            '--data-file': str(input),
+            '--schema-file': [str(schema_file)],
+            '--quiet': False,
+            '--verbose': 1,
+            '--yaml-extension': 'yext',
+        }
+
+        cli_args = cli.parse_cli()
+
+        for k, v in expected.items():
+            assert k in cli_args
+            assert cli_args[k] == expected[k]
+
     def f(self, *args):
         """
         Returns abs path to test files inside tests/files/

tests/test_core.py

@@ -110,6 +110,32 @@ def test_load_unsupported_format(self, tmpdir):
             Core(schema_files=[str(schema_f)])
         assert "Unknown file format. Supported file endings is" in str(ex.value)
 
+    def test_load_custom_yaml_format(self, tmpdir):
+        """
+        Try to load some file extension with custom support.
+        """
+        source_f = tmpdir.join("foo.yext")
+        source_f.write("3.14159")
+
+        schema_f = tmpdir.join("bar.yaml")
+        schema_f.write("type: float")
+
+        Core(source_file=str(source_f), schema_files=[str(schema_f)],
+             custom_yaml_ext='yext')
+
+    def test_load_custom_json_format(self, tmpdir):
+        """
+        Load source & schema files that has json file ending.
+        """
+        source_f = tmpdir.join("bar.jext")
+        source_f.write("3.14159")
+
+        schema_f = tmpdir.join("foo.json")
+        schema_f.write('{"type": "float"}')
+
+        Core(source_file=str(source_f), schema_files=[str(schema_f)],
+             custom_json_ext='jext')
+
     def test_load_empty_json_file(self, tmpdir):
         """
         Loading an empty json files should raise an exception